From f8b92a7c5ba651527c3b7dc2a3fd53f55b2e0457 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Jul 2026 00:19:01 -0700 Subject: [PATCH 1/4] Set uploaded asset metadata schemaKey to "Asset" Whatever the metadata was gathered as (e.g. a BareAsset), what is created on the server is an Asset. The server stores schemaKey verbatim and does not normalize it, so set it at the point of upload rather than relying on the caller. This matters because dandi-schema is changing BareAsset.schemaKey from "Asset" to "BareAsset". Metadata gathered via a BareAsset then serializes with schemaKey="BareAsset", so an asset created or updated from it would be stored with the wrong schemaKey. Pin it at every asset-endpoint write: iter_upload (LocalFileAsset, ZarrAsset) and set_raw_metadata (RemoteBlobAsset, RemoteZarrAsset); the latter is reached by the reextract command without going through iter_upload. A no-op against the current dandi-schema release (where BareAsset.schemaKey is still "Asset"), so it can ship first. Co-Authored-By: Claude Code 2.1.202 / Claude Opus 4.8 claude-opus-4-8 --- dandi/dandiapi.py | 10 ++++++++++ dandi/files/bases.py | 5 +++++ dandi/files/zarr.py | 5 +++++ dandi/tests/test_upload.py | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 61566425b..c0eb6b228 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1993,6 +1993,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteBlobAsset` in place. """ + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" data = self.client.put( self.api_path, json={"metadata": metadata, "blob_id": self.blob} ) @@ -2016,6 +2021,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteZarrAsset` in place. """ + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" data = self.client.put( self.api_path, json={"metadata": metadata, "zarr_id": self.zarr} ) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 381ee51cf..3d1859433 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -363,6 +363,11 @@ def iter_upload( from dandi.support.digests import get_dandietag asset_path = metadata.setdefault("path", self.path) + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" client = dandiset.client yield {"status": "calculating etag"} etagger = get_dandietag(self.filepath) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 14bac02c5..ff01f6f58 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -580,6 +580,11 @@ def iter_upload( `RemoteAsset`. """ asset_path = metadata.setdefault("path", self.path) + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" client = dandiset.client lgr.debug("%s: Producing asset", asset_path) yield {"status": "producing asset"} diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index 577d99d11..cda476474 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -50,6 +50,10 @@ def test_upload_download( (dspath / parent).mkdir() copyfile(nwb_file, dspath / parent / name) new_dandiset.upload() + # The uploaded asset's metadata is derived from a ``BareAsset``, but it must + # be uploaded and stored with a ``schemaKey`` of `"Asset"` + (asset,) = d.get_assets() + assert asset.get_raw_metadata()["schemaKey"] == "Asset" download(d.version_api_url, tmp_path) assert list_paths(tmp_path) == [ tmp_path / d.identifier / dandiset_metadata_file, From 17fa7522a63536239b7089f34c91491bf8711b0c Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Jul 2026 18:23:30 -0700 Subject: [PATCH 2/4] Drop redundant `schemaKey="Asset"` from `nwb2asset` assertions `test_nwb2asset` and `test_nwb2asset_remote_asset` compared the `nwb2asset()` result against a `BareAsset` built with an explicit `schemaKey="Asset"`, which just restates the field default. `model_construct()` fills defaults in, so dropping it is a no-op against the current dandischema and ensures the equalities continue to hold once dandi-schema#419 lands, where `BareAsset.schemaKey` becomes "BareAsset". --- dandi/tests/test_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 00520d08f..a0b2ab85b 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -1135,7 +1135,6 @@ def test_nwb2asset(simple2_nwb: Path) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() assert nwb2asset(simple2_nwb, digest=DUMMY_DANDI_ETAG) == BareAsset.model_construct( - schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], access=[ @@ -1218,7 +1217,6 @@ def test_nwb2asset_remote_asset(nwb_dandiset: SampleDandiset) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() assert nwb2asset(r, digest=digest) == BareAsset.model_construct( - schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], access=[ From 399da87268bc8a53a945cca8b80732e25c65a087 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 8 Jul 2026 13:28:37 -0700 Subject: [PATCH 3/4] Make `test_prepare_metadata` independent of the `BareAsset.schemaKey` value `prepare_metadata()` returns a `BareAsset`, whose top-level `schemaKey` is `"Asset"` with the current dandischema but `"BareAsset"` under dandi/dandi-schema#419 (which pins `BareAsset.schemaKey` to its class name). To keep this test passing against either dandischema version, the expected `schemaKey` is set to the default of the `schemaKey` field in `BareAsset` before the comparison, and the four `metadata2asset*.json` files now store a placeholder for the field. The later `validate()` call checks the data against the `Asset` class, so `data_as_dict` is promoted to an Asset (`schemaKey = "Asset"`) beforehand. --- dandi/tests/data/metadata/metadata2asset.json | 2 +- dandi/tests/data/metadata/metadata2asset_3.json | 2 +- .../data/metadata/metadata2asset_cellline.json | 2 +- .../data/metadata/metadata2asset_simple1.json | 2 +- dandi/tests/test_metadata.py | 17 ++++++++++++++--- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/dandi/tests/data/metadata/metadata2asset.json b/dandi/tests/data/metadata/metadata2asset.json index 60a3a9005..279020176 100644 --- a/dandi/tests/data/metadata/metadata2asset.json +++ b/dandi/tests/data/metadata/metadata2asset.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_3.json b/dandi/tests/data/metadata/metadata2asset_3.json index 44b809a69..1e9e40eb9 100644 --- a/dandi/tests/data/metadata/metadata2asset_3.json +++ b/dandi/tests/data/metadata/metadata2asset_3.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_cellline.json b/dandi/tests/data/metadata/metadata2asset_cellline.json index 286eb0b1f..964fc52ac 100644 --- a/dandi/tests/data/metadata/metadata2asset_cellline.json +++ b/dandi/tests/data/metadata/metadata2asset_cellline.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_simple1.json b/dandi/tests/data/metadata/metadata2asset_simple1.json index 931779db5..3d9bcbb16 100644 --- a/dandi/tests/data/metadata/metadata2asset_simple1.json +++ b/dandi/tests/data/metadata/metadata2asset_simple1.json @@ -1,6 +1,6 @@ { "id": "dandiasset:bfc23fb6192b41c083a7257e09a3702b", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "keyword1", diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index a0b2ab85b..c168af852 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -418,11 +418,22 @@ def test_prepare_metadata(filename: str, metadata: dict[str, Any]) -> None: with (METADATA_DIR / filename).open() as fp: data_as_dict = json.load(fp) data_as_dict["schemaVersion"] = DANDI_SCHEMA_VERSION + # `prepare_metadata()` returns a `BareAsset`, so the top-level `schemaKey` + # of `data` is the default of the `schemaKey` field of `BareAsset` of the + # installed dandischema: "Asset" historically, but "BareAsset" once + # https://github.com/dandi/dandi-schema/pull/419 lands, which pins + # `BareAsset.schemaKey` to its own class name. The JSON files store a + # placeholder for this field, so set it to the current default here to keep + # the comparison independent of the installed dandischema version. + data_as_dict["schemaKey"] = BareAsset.model_fields["schemaKey"].default assert data == data_as_dict + + # `data_as_dict` is a BareAsset; the following lines turn it into an Asset + # instance so that `validate()` below checks it against the Asset class: set + # its `schemaKey`, and add the Asset-only fields `identifier` and (as of + # schema-0.5.0, https://github.com/dandi/dandischema/pull/52) `contentUrl`. + data_as_dict["schemaKey"] = "Asset" data_as_dict["identifier"] = "0b0a1a0b-e3ea-4cf6-be94-e02c830d54be" - # as of schema-0.5.0 (https://github.com/dandi/dandischema/pull/52) - # contentUrl is required, and validate below would map into Asset, - # due to schemaKey if Version(DANDI_SCHEMA_VERSION) >= Version("0.5.0"): data_as_dict["contentUrl"] = ["http://example.com"] validate(data_as_dict) From 0cf9beb3d314d6be44c0fcf8a236cbeaf553a4b3 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Jul 2026 14:07:28 -0700 Subject: [PATCH 4/4] Test schemaKey handling against dandi-schema #419 branch Temporarily point the dandischema dependency at the consolidate-models branch (dandi/dandi-schema#419), where BareAsset.schemaKey is "BareAsset", so CI runs the dandi-cli client against the consolidated schema and checks that this branch still uploads schemaKey="Asset". For checking the test outcome only; not intended to be merged. Co-Authored-By: Claude Code 2.1.202 / Claude Opus 4.8 claude-opus-4-8 --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a7623e51c..0f1974b3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,10 @@ dependencies = [ "bids-validator-deno >= 2.0.5", "click >= 7.1", "click-didyoumean", - "dandischema ~= 0.12.0", + # Temporarily point at the dandi-schema #419 branch, where + # BareAsset.schemaKey is "BareAsset", so CI runs the client against the + # consolidated schema to check the schemaKey handling on this branch. + "dandischema @ git+https://github.com/dandi/dandi-schema.git@consolidate-models", "etelemetry >= 0.2.2", "fasteners", "fscacher >= 0.3.0",