From d6a8c2b2146ba0ab7e96d783cf1bf23e97b28e4a Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Tue, 28 Jul 2026 02:28:03 +0300 Subject: [PATCH 1/2] openapi3: keep a document's origin tree only when it can be read The loader retained every loaded document's origin tree for the whole life of the loader. On a 22 MB spec that was 150 MB, a third of everything retained, and on most documents it could never be read at all. The tree has exactly one reader, attachOriginToResolved, which runs only for a $ref that resolves through a generic map. That happens only where the document carries arbitrary top-level keys, since typed resolution keeps its own origins. So a document with no such keys can never consult its tree, and retaining it is pure cost. Keep the tree only for documents that have them. The arbitrary-key origin tests cover the retained path: forcing the tree to never be kept fails all three, so they guard this predicate rather than passing vacuously. Retained size, measured with the field-locations change already in: 5 MB spec: 105 MB -> 69 MB 22 MB spec: 432 MB -> 282 MB Peak heap during load is unchanged: it is dominated by the transient yaml parse tree, not by what the loader keeps. Co-Authored-By: Claude Opus 5 (1M context) --- openapi3/loader.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/openapi3/loader.go b/openapi3/loader.go index 2f2480c07..cf1c75382 100644 --- a/openapi3/loader.go +++ b/openapi3/loader.go @@ -149,8 +149,26 @@ func (loader *Loader) loadSingleElementFromURI(ref string, rootPath *url.URL, el // rememberOriginTree retains doc's origin tree for attachOriginToResolved. // tree is nil when IncludeOrigin is off or the data took the json path. +// +// The tree is kept only for a document a $ref can reach into untyped, which is +// what attachOriginToResolved exists to re-origin. In practice that means a +// file of shared fragments, whose top level is the fragment name itself rather +// than the fields of an OpenAPI Object: +// +// User: # a $ref to "./schemas.yaml#/User" lands here, untyped +// type: object +// +// Anything OpenAPI defines a field for resolves through typed structures and +// keeps its origins on the way, so its tree could never be read. That includes +// a referenced document that is itself an OpenAPI Object: a $ref to +// "#/components/schemas/User" needs no tree. (A top-level x- extension is +// undefined by the same rule, so a document carrying one keeps its tree too, +// whether or not anything ever points at it.) +// +// Worth the condition: on a 22 MB spec the retained tree was a third of +// everything the loader held. func (loader *Loader) rememberOriginTree(doc *T, tree *originTree) { - if tree == nil { + if tree == nil || len(doc.Extensions) == 0 { return } if loader.originTrees == nil { From 43392d225544cfae699579b6d445ce589a3cf099 Mon Sep 17 00:00:00 2001 From: Reuven Harrison Date: Thu, 30 Jul 2026 14:48:52 +0300 Subject: [PATCH 2/2] openapi3: test that the origin tree is kept only where it can be read The retention change had no test of its own; it was covered only indirectly, by the arbitrary-key tests continuing to pass. Three cases, asserting on loader.originTrees directly rather than on memory, so they are deterministic: - an ordinary spec retains no tree at all, while its document still carries origins - testdata/origin/arbitrary_key.yaml retains exactly one, and it is the referenced file rather than the root: the root has only standard fields, the referenced file has the arbitrary top-level key "User". One load, both sides of the condition - with IncludeOrigin off there is no tree either way The second also asserts that the resolved schema still carries the origin of its own file, so the saving and the capability are pinned together: a predicate that dropped the wrong tree would fail there rather than silently returning origin-less schemas. Measured on the loader's retained tree: 36 MB -> 0 on a 5 MB spec, 150 MB -> 0 on a 22 MB one. --- openapi3/origin_retention_test.go | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 openapi3/origin_retention_test.go diff --git a/openapi3/origin_retention_test.go b/openapi3/origin_retention_test.go new file mode 100644 index 000000000..0b188026f --- /dev/null +++ b/openapi3/origin_retention_test.go @@ -0,0 +1,71 @@ +package openapi3 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// An ordinary spec never consults its origin tree, so the loader does not keep +// one. Origins are still attached to the document itself; what goes away is the +// second copy the loader used to hold for the lifetime of the loader, which on +// a large spec was a third of everything it retained. +func TestOriginTree_NotRetainedForAnOrdinarySpec(t *testing.T) { + loader := NewLoader() + loader.IncludeOrigin = true + loader.Context = t.Context() + + doc, err := loader.LoadFromFile("testdata/origin/simple.yaml") + require.NoError(t, err) + + require.NotNil(t, doc.Origin, "origins are still attached to the document") + require.Empty(t, loader.originTrees, "no tree is retained: nothing could read it") +} + +// The tree is kept for exactly the documents that can consult it, and this OAD +// contains one of each: the entry document has only fields OpenAPI defines, +// while the file it $refs is a shared fragment whose top level is the schema +// name "User", which is what arrives untyped and what attachOriginToResolved +// walks the tree for. +// +// The retained tree still does its job here, so this pins the saving and the +// capability together: dropping the tree for the referenced file would leave +// the resolved schema without an origin. +func TestOriginTree_RetainedOnlyWhereItCanBeRead(t *testing.T) { + loader := NewLoader() + loader.IsExternalRefsAllowed = true + loader.IncludeOrigin = true + loader.Context = t.Context() + + doc, err := loader.LoadFromFile("testdata/origin/arbitrary_key.yaml") + require.NoError(t, err) + + require.NotContains(t, loader.originTrees, doc, + "the entry document has only fields OpenAPI defines, so no $ref reaches into it untyped") + require.Len(t, loader.originTrees, 1, "only the shared-fragment file keeps its tree") + for retained := range loader.originTrees { + require.NotEmpty(t, retained.Extensions, + "a tree is kept only for a document with top-level fields OpenAPI does not define") + } + + // The capability the retained tree exists for: the resolved schema carries + // the origin of its own file, not of the $ref site. + schema := doc.Paths.Find("/users").Get.Responses.Value("200").Value. + Content["application/json"].Schema.Value + require.NotNil(t, schema.Origin, "the resolved schema keeps its origin") + require.Contains(t, schema.Origin.Key.File, "arbitrary_key_schemas.yaml") +} + +// Without IncludeOrigin there is no tree to begin with, so the new condition +// cannot change anything here. +func TestOriginTree_NotRetainedWhenOriginsAreOff(t *testing.T) { + loader := NewLoader() + loader.IsExternalRefsAllowed = true + loader.Context = t.Context() + + doc, err := loader.LoadFromFile("testdata/origin/arbitrary_key.yaml") + require.NoError(t, err) + + require.Nil(t, doc.Origin) + require.Empty(t, loader.originTrees) +}