From 052c4bda4c6d00e4ceb24224b83e189bafbcede7 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Jul 2026 00:19:01 -0700 Subject: [PATCH 1/5] Set uploaded asset metadata schemaKey to "Asset" Whatever the metadata was gathered as (e.g. a BareAsset), what is created on the server is an Asset. The server stores schemaKey verbatim and does not normalize it, so set it at the point of upload rather than relying on the caller. This matters because dandi-schema is changing BareAsset.schemaKey from "Asset" to "BareAsset". Metadata gathered via a BareAsset then serializes with schemaKey="BareAsset", so an asset created or updated from it would be stored with the wrong schemaKey. Pin it at every asset-endpoint write: iter_upload (LocalFileAsset, ZarrAsset) and set_raw_metadata (RemoteBlobAsset, RemoteZarrAsset); the latter is reached by the reextract command without going through iter_upload. A no-op against the current dandi-schema release (where BareAsset.schemaKey is still "Asset"), so it can ship first. Co-Authored-By: Claude Code 2.1.202 / Claude Opus 4.8 claude-opus-4-8 --- dandi/dandiapi.py | 10 ++++++++++ dandi/files/bases.py | 5 +++++ dandi/files/zarr.py | 5 +++++ dandi/tests/test_upload.py | 4 ++++ 4 files changed, 24 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 61566425b..c0eb6b228 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1993,6 +1993,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteBlobAsset` in place. """ + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" data = self.client.put( self.api_path, json={"metadata": metadata, "blob_id": self.blob} ) @@ -2016,6 +2021,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteZarrAsset` in place. """ + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" data = self.client.put( self.api_path, json={"metadata": metadata, "zarr_id": self.zarr} ) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 381ee51cf..3d1859433 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -363,6 +363,11 @@ def iter_upload( from dandi.support.digests import get_dandietag asset_path = metadata.setdefault("path", self.path) + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" client = dandiset.client yield {"status": "calculating etag"} etagger = get_dandietag(self.filepath) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 14bac02c5..ff01f6f58 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -580,6 +580,11 @@ def iter_upload( `RemoteAsset`. """ asset_path = metadata.setdefault("path", self.path) + # Whatever the metadata was gathered as (e.g. a ``BareAsset``), + # what is being created on the server is an ``Asset`` as result of the upload. + # The server stores ``schemaKey`` verbatim, so set it + # here, at the point of upload, rather than relying on the caller. + metadata["schemaKey"] = "Asset" client = dandiset.client lgr.debug("%s: Producing asset", asset_path) yield {"status": "producing asset"} diff --git a/dandi/tests/test_upload.py b/dandi/tests/test_upload.py index 577d99d11..cda476474 100644 --- a/dandi/tests/test_upload.py +++ b/dandi/tests/test_upload.py @@ -50,6 +50,10 @@ def test_upload_download( (dspath / parent).mkdir() copyfile(nwb_file, dspath / parent / name) new_dandiset.upload() + # The uploaded asset's metadata is derived from a ``BareAsset``, but it must + # be uploaded and stored with a ``schemaKey`` of `"Asset"` + (asset,) = d.get_assets() + assert asset.get_raw_metadata()["schemaKey"] == "Asset" download(d.version_api_url, tmp_path) assert list_paths(tmp_path) == [ tmp_path / d.identifier / dandiset_metadata_file, From 044e97b2ca32a574e589400c5c4fb9d6a1f19b49 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 7 Jul 2026 18:23:30 -0700 Subject: [PATCH 2/5] Drop redundant `schemaKey="Asset"` from `nwb2asset` assertions `test_nwb2asset` and `test_nwb2asset_remote_asset` compared the `nwb2asset()` result against a `BareAsset` built with an explicit `schemaKey="Asset"`, which just restates the field default. `model_construct()` fills defaults in, so dropping it is a no-op against the current dandischema and ensures the equalities continue to hold once dandi-schema#419 lands, where `BareAsset.schemaKey` becomes "BareAsset". --- dandi/tests/test_metadata.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 00520d08f..a0b2ab85b 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -1135,7 +1135,6 @@ def test_nwb2asset(simple2_nwb: Path) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() assert nwb2asset(simple2_nwb, digest=DUMMY_DANDI_ETAG) == BareAsset.model_construct( - schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], access=[ @@ -1218,7 +1217,6 @@ def test_nwb2asset_remote_asset(nwb_dandiset: SampleDandiset) -> None: # Classes with ANY_AWARE_DATETIME fields need to be constructed with # model_construct() assert nwb2asset(r, digest=digest) == BareAsset.model_construct( - schemaKey="Asset", schemaVersion=DANDI_SCHEMA_VERSION, keywords=["keyword1", "keyword 2"], access=[ From 6167ee34d5636a4d2a28dbe014742023c48160a6 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 8 Jul 2026 13:28:37 -0700 Subject: [PATCH 3/5] Make `test_prepare_metadata` independent of the `BareAsset.schemaKey` value `prepare_metadata()` returns a `BareAsset`, whose top-level `schemaKey` is `"Asset"` with the current dandischema but `"BareAsset"` under dandi/dandi-schema#419 (which pins `BareAsset.schemaKey` to its class name). To keep this test passing against either dandischema version, the expected `schemaKey` is set to the default of the `schemaKey` field in `BareAsset` before the comparison, and the four `metadata2asset*.json` files now store a placeholder for the field. The later `validate()` call checks the data against the `Asset` class, so `data_as_dict` is promoted to an Asset (`schemaKey = "Asset"`) beforehand. --- dandi/tests/data/metadata/metadata2asset.json | 2 +- dandi/tests/data/metadata/metadata2asset_3.json | 2 +- .../data/metadata/metadata2asset_cellline.json | 2 +- .../data/metadata/metadata2asset_simple1.json | 2 +- dandi/tests/test_metadata.py | 17 ++++++++++++++--- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/dandi/tests/data/metadata/metadata2asset.json b/dandi/tests/data/metadata/metadata2asset.json index 60a3a9005..279020176 100644 --- a/dandi/tests/data/metadata/metadata2asset.json +++ b/dandi/tests/data/metadata/metadata2asset.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_3.json b/dandi/tests/data/metadata/metadata2asset_3.json index 44b809a69..1e9e40eb9 100644 --- a/dandi/tests/data/metadata/metadata2asset_3.json +++ b/dandi/tests/data/metadata/metadata2asset_3.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_cellline.json b/dandi/tests/data/metadata/metadata2asset_cellline.json index 286eb0b1f..964fc52ac 100644 --- a/dandi/tests/data/metadata/metadata2asset_cellline.json +++ b/dandi/tests/data/metadata/metadata2asset_cellline.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_simple1.json b/dandi/tests/data/metadata/metadata2asset_simple1.json index 931779db5..3d9bcbb16 100644 --- a/dandi/tests/data/metadata/metadata2asset_simple1.json +++ b/dandi/tests/data/metadata/metadata2asset_simple1.json @@ -1,6 +1,6 @@ { "id": "dandiasset:bfc23fb6192b41c083a7257e09a3702b", - "schemaKey": "Asset", + "schemaKey": "TO BE SPECIFIED DYNAMICALLY", "schemaVersion": "0.4.1", "keywords": [ "keyword1", diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index a0b2ab85b..c168af852 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -418,11 +418,22 @@ def test_prepare_metadata(filename: str, metadata: dict[str, Any]) -> None: with (METADATA_DIR / filename).open() as fp: data_as_dict = json.load(fp) data_as_dict["schemaVersion"] = DANDI_SCHEMA_VERSION + # `prepare_metadata()` returns a `BareAsset`, so the top-level `schemaKey` + # of `data` is the default of the `schemaKey` field of `BareAsset` of the + # installed dandischema: "Asset" historically, but "BareAsset" once + # https://github.com/dandi/dandi-schema/pull/419 lands, which pins + # `BareAsset.schemaKey` to its own class name. The JSON files store a + # placeholder for this field, so set it to the current default here to keep + # the comparison independent of the installed dandischema version. + data_as_dict["schemaKey"] = BareAsset.model_fields["schemaKey"].default assert data == data_as_dict + + # `data_as_dict` is a BareAsset; the following lines turn it into an Asset + # instance so that `validate()` below checks it against the Asset class: set + # its `schemaKey`, and add the Asset-only fields `identifier` and (as of + # schema-0.5.0, https://github.com/dandi/dandischema/pull/52) `contentUrl`. + data_as_dict["schemaKey"] = "Asset" data_as_dict["identifier"] = "0b0a1a0b-e3ea-4cf6-be94-e02c830d54be" - # as of schema-0.5.0 (https://github.com/dandi/dandischema/pull/52) - # contentUrl is required, and validate below would map into Asset, - # due to schemaKey if Version(DANDI_SCHEMA_VERSION) >= Version("0.5.0"): data_as_dict["contentUrl"] = ["http://example.com"] validate(data_as_dict) From bec49df6c09687b4fd6b4e5e424621cad417c9db Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 9 Jul 2026 18:45:10 -0700 Subject: [PATCH 4/5] Concentrate asset `schemaKey` setting in `set_asset_schema_key()` The four sites that pin `schemaKey` to `"Asset"` for the server (the two `iter_upload` methods and the two `RemoteAsset.set_raw_metadata` methods) each repeated the assignment and an identical explanatory comment. Move both into a single `set_asset_schema_key()` helper in `dandi.dandiapi` and call it from each site, so the operation and its rationale live in one place. --- dandi/dandiapi.py | 26 ++++++++++++++++---------- dandi/files/bases.py | 13 +++++++------ dandi/files/zarr.py | 7 ++----- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index c0eb6b228..42e311c95 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -70,6 +70,20 @@ lgr = get_logger() +def set_asset_schema_key(metadata: dict[str, Any]) -> None: + """ + Set ``schemaKey`` to ``"Asset"`` on metadata bound for the server as asset metadata. + + However the metadata was gathered (e.g. as a ``BareAsset``), if it is to be uploaded + to the server to populate, in part, an ``Asset`` instance, set the ``schemaKey`` of + the metadata to ``"Asset"`` here. The server expects the uploaded metadata to carry + ``"Asset"`` as its ``schemaKey``. + + The metadata is modified in place. + """ + metadata["schemaKey"] = "Asset" + + class AssetType(Enum): """ .. versionadded:: 0.36.0 @@ -1993,11 +2007,7 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteBlobAsset` in place. """ - # Whatever the metadata was gathered as (e.g. a ``BareAsset``), - # what is being created on the server is an ``Asset`` as result of the upload. - # The server stores ``schemaKey`` verbatim, so set it - # here, at the point of upload, rather than relying on the caller. - metadata["schemaKey"] = "Asset" + set_asset_schema_key(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "blob_id": self.blob} ) @@ -2021,11 +2031,7 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: Set the metadata for the asset on the server to the given value and update the `RemoteZarrAsset` in place. """ - # Whatever the metadata was gathered as (e.g. a ``BareAsset``), - # what is being created on the server is an ``Asset`` as result of the upload. - # The server stores ``schemaKey`` verbatim, so set it - # here, at the point of upload, rather than relying on the caller. - metadata["schemaKey"] = "Asset" + set_asset_schema_key(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "zarr_id": self.zarr} ) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 3d1859433..3577e605f 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -25,7 +25,12 @@ import requests import dandi -from dandi.dandiapi import RemoteAsset, RemoteDandiset, RESTFullAPIClient +from dandi.dandiapi import ( + RemoteAsset, + RemoteDandiset, + RESTFullAPIClient, + set_asset_schema_key, +) from dandi.metadata.core import get_default_metadata from dandi.misctypes import DUMMY_DANDI_ETAG, Digest, LocalReadableFile, P from dandi.utils import post_upload_size_check, pre_upload_size_check, yaml_load @@ -363,11 +368,7 @@ def iter_upload( from dandi.support.digests import get_dandietag asset_path = metadata.setdefault("path", self.path) - # Whatever the metadata was gathered as (e.g. a ``BareAsset``), - # what is being created on the server is an ``Asset`` as result of the upload. - # The server stores ``schemaKey`` verbatim, so set it - # here, at the point of upload, rather than relying on the caller. - metadata["schemaKey"] = "Asset" + set_asset_schema_key(metadata) client = dandiset.client yield {"status": "calculating etag"} etagger = get_dandietag(self.filepath) diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index ff01f6f58..8f5a4aa0e 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -38,6 +38,7 @@ RemoteZarrAsset, RemoteZarrEntry, RESTFullAPIClient, + set_asset_schema_key, ) from dandi.exceptions import UploadError from dandi.metadata.core import get_default_metadata @@ -580,11 +581,7 @@ def iter_upload( `RemoteAsset`. """ asset_path = metadata.setdefault("path", self.path) - # Whatever the metadata was gathered as (e.g. a ``BareAsset``), - # what is being created on the server is an ``Asset`` as result of the upload. - # The server stores ``schemaKey`` verbatim, so set it - # here, at the point of upload, rather than relying on the caller. - metadata["schemaKey"] = "Asset" + set_asset_schema_key(metadata) client = dandiset.client lgr.debug("%s: Producing asset", asset_path) yield {"status": "producing asset"} From f98a949a38cae64942ca554bdd53cc520fc8efa9 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Thu, 9 Jul 2026 23:50:38 -0700 Subject: [PATCH 5/5] Clarify the placeholder `schemaKey` in the `metadata2asset*.json` files Spell out, in the placeholder value itself, that `test_prepare_metadata` adjusts the top-level `schemaKey` to the default of the `schemaKey` field in the Pydantic `BareAsset` model, so a reader of the file understands the value is a stand-in rather than the compared value. Co-Authored-By: Yaroslav Halchenko <39889+yarikoptic@users.noreply.github.com> Co-Authored-By: Claude Code 2.1.202 / Claude Opus 4.8 claude-opus-4-8 --- dandi/tests/data/metadata/metadata2asset.json | 2 +- dandi/tests/data/metadata/metadata2asset_3.json | 2 +- dandi/tests/data/metadata/metadata2asset_cellline.json | 2 +- dandi/tests/data/metadata/metadata2asset_simple1.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dandi/tests/data/metadata/metadata2asset.json b/dandi/tests/data/metadata/metadata2asset.json index 279020176..267b72c14 100644 --- a/dandi/tests/data/metadata/metadata2asset.json +++ b/dandi/tests/data/metadata/metadata2asset.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "TO BE SPECIFIED DYNAMICALLY", + "schemaKey": "ANYTHING GOOD NOW -- `schemaKey` of a `BareAsset` instance will be adjusted to the default value of the `schemaKey` field in the Pydantic `BareAsset` model", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_3.json b/dandi/tests/data/metadata/metadata2asset_3.json index 1e9e40eb9..abc20e3b7 100644 --- a/dandi/tests/data/metadata/metadata2asset_3.json +++ b/dandi/tests/data/metadata/metadata2asset_3.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "TO BE SPECIFIED DYNAMICALLY", + "schemaKey": "ANYTHING GOOD NOW -- `schemaKey` of a `BareAsset` instance will be adjusted to the default value of the `schemaKey` field in the Pydantic `BareAsset` model", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_cellline.json b/dandi/tests/data/metadata/metadata2asset_cellline.json index 964fc52ac..b777d27c3 100644 --- a/dandi/tests/data/metadata/metadata2asset_cellline.json +++ b/dandi/tests/data/metadata/metadata2asset_cellline.json @@ -1,6 +1,6 @@ { "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be", - "schemaKey": "TO BE SPECIFIED DYNAMICALLY", + "schemaKey": "ANYTHING GOOD NOW -- `schemaKey` of a `BareAsset` instance will be adjusted to the default value of the `schemaKey` field in the Pydantic `BareAsset` model", "schemaVersion": "0.4.1", "keywords": [ "test", diff --git a/dandi/tests/data/metadata/metadata2asset_simple1.json b/dandi/tests/data/metadata/metadata2asset_simple1.json index 3d9bcbb16..a28b4eb4b 100644 --- a/dandi/tests/data/metadata/metadata2asset_simple1.json +++ b/dandi/tests/data/metadata/metadata2asset_simple1.json @@ -1,6 +1,6 @@ { "id": "dandiasset:bfc23fb6192b41c083a7257e09a3702b", - "schemaKey": "TO BE SPECIFIED DYNAMICALLY", + "schemaKey": "ANYTHING GOOD NOW -- `schemaKey` of a `BareAsset` instance will be adjusted to the default value of the `schemaKey` field in the Pydantic `BareAsset` model", "schemaVersion": "0.4.1", "keywords": [ "keyword1",