DO NOT MERGE: check #1886 schemaKey handling against dandi-schema #419

dandi/dandiapi.py

@@ -1993,6 +1993,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None:
         Set the metadata for the asset on the server to the given value and
         update the `RemoteBlobAsset` in place.
         """
+        # Whatever the metadata was gathered as (e.g. a ``BareAsset``),
+        # what is being created on the server is an ``Asset`` as result of the upload.
+        # The server stores ``schemaKey`` verbatim, so set it
+        # here, at the point of upload, rather than relying on the caller.
+        metadata["schemaKey"] = "Asset"
         data = self.client.put(
             self.api_path, json={"metadata": metadata, "blob_id": self.blob}
         )
@@ -2016,6 +2021,11 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None:
         Set the metadata for the asset on the server to the given value and
         update the `RemoteZarrAsset` in place.
         """
+        # Whatever the metadata was gathered as (e.g. a ``BareAsset``),
+        # what is being created on the server is an ``Asset`` as result of the upload.
+        # The server stores ``schemaKey`` verbatim, so set it
+        # here, at the point of upload, rather than relying on the caller.
+        metadata["schemaKey"] = "Asset"
         data = self.client.put(
             self.api_path, json={"metadata": metadata, "zarr_id": self.zarr}
         )

dandi/files/bases.py

@@ -363,6 +363,11 @@ def iter_upload(
         from dandi.support.digests import get_dandietag
 
         asset_path = metadata.setdefault("path", self.path)
+        # Whatever the metadata was gathered as (e.g. a ``BareAsset``),
+        # what is being created on the server is an ``Asset`` as result of the upload.
+        # The server stores ``schemaKey`` verbatim, so set it
+        # here, at the point of upload, rather than relying on the caller.
+        metadata["schemaKey"] = "Asset"
         client = dandiset.client
         yield {"status": "calculating etag"}
         etagger = get_dandietag(self.filepath)

dandi/files/zarr.py

@@ -580,6 +580,11 @@ def iter_upload(
             `RemoteAsset`.
         """
         asset_path = metadata.setdefault("path", self.path)
+        # Whatever the metadata was gathered as (e.g. a ``BareAsset``),
+        # what is being created on the server is an ``Asset`` as result of the upload.
+        # The server stores ``schemaKey`` verbatim, so set it
+        # here, at the point of upload, rather than relying on the caller.
+        metadata["schemaKey"] = "Asset"
         client = dandiset.client
         lgr.debug("%s: Producing asset", asset_path)
         yield {"status": "producing asset"}

dandi/tests/data/metadata/metadata2asset.json

@@ -1,6 +1,6 @@
 {
   "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be",
-  "schemaKey": "Asset",
+  "schemaKey": "TO BE SPECIFIED DYNAMICALLY",
   "schemaVersion": "0.4.1",
   "keywords": [
     "test",

dandi/tests/data/metadata/metadata2asset_3.json

@@ -1,6 +1,6 @@
 {
   "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be",
-  "schemaKey": "Asset",
+  "schemaKey": "TO BE SPECIFIED DYNAMICALLY",
   "schemaVersion": "0.4.1",
   "keywords": [
     "test",

dandi/tests/data/metadata/metadata2asset_cellline.json

@@ -1,6 +1,6 @@
 {
   "id": "dandiasset:0b0a1a0b-e3ea-4cf6-be94-e02c830d54be",
-  "schemaKey": "Asset",
+  "schemaKey": "TO BE SPECIFIED DYNAMICALLY",
   "schemaVersion": "0.4.1",
   "keywords": [
     "test",

dandi/tests/data/metadata/metadata2asset_simple1.json

@@ -1,6 +1,6 @@
 {
   "id": "dandiasset:bfc23fb6192b41c083a7257e09a3702b",
-  "schemaKey": "Asset",
+  "schemaKey": "TO BE SPECIFIED DYNAMICALLY",
   "schemaVersion": "0.4.1",
   "keywords": [
     "keyword1",

dandi/tests/test_metadata.py

@@ -418,11 +418,22 @@ def test_prepare_metadata(filename: str, metadata: dict[str, Any]) -> None:
     with (METADATA_DIR / filename).open() as fp:
         data_as_dict = json.load(fp)
     data_as_dict["schemaVersion"] = DANDI_SCHEMA_VERSION
+    # `prepare_metadata()` returns a `BareAsset`, so the top-level `schemaKey`
+    # of `data` is the default of the `schemaKey` field of `BareAsset` of the
+    # installed dandischema: "Asset" historically, but "BareAsset" once
+    # https://github.com/dandi/dandi-schema/pull/419 lands, which pins
+    # `BareAsset.schemaKey` to its own class name.  The JSON files store a
+    # placeholder for this field, so set it to the current default here to keep
+    # the comparison independent of the installed dandischema version.
+    data_as_dict["schemaKey"] = BareAsset.model_fields["schemaKey"].default
     assert data == data_as_dict
+
+    # `data_as_dict` is a BareAsset; the following lines turn it into an Asset
+    # instance so that `validate()` below checks it against the Asset class: set
+    # its `schemaKey`, and add the Asset-only fields `identifier` and (as of
+    # schema-0.5.0, https://github.com/dandi/dandischema/pull/52) `contentUrl`.
+    data_as_dict["schemaKey"] = "Asset"
     data_as_dict["identifier"] = "0b0a1a0b-e3ea-4cf6-be94-e02c830d54be"
-    # as of schema-0.5.0 (https://github.com/dandi/dandischema/pull/52)
-    # contentUrl is required, and validate below would map into Asset,
-    # due to schemaKey
     if Version(DANDI_SCHEMA_VERSION) >= Version("0.5.0"):
         data_as_dict["contentUrl"] = ["http://example.com"]
     validate(data_as_dict)
@@ -1135,7 +1146,6 @@ def test_nwb2asset(simple2_nwb: Path) -> None:
     # Classes with ANY_AWARE_DATETIME fields need to be constructed with
     # model_construct()
     assert nwb2asset(simple2_nwb, digest=DUMMY_DANDI_ETAG) == BareAsset.model_construct(
-        schemaKey="Asset",
         schemaVersion=DANDI_SCHEMA_VERSION,
         keywords=["keyword1", "keyword 2"],
         access=[
@@ -1218,7 +1228,6 @@ def test_nwb2asset_remote_asset(nwb_dandiset: SampleDandiset) -> None:
     # Classes with ANY_AWARE_DATETIME fields need to be constructed with
     # model_construct()
     assert nwb2asset(r, digest=digest) == BareAsset.model_construct(
-        schemaKey="Asset",
         schemaVersion=DANDI_SCHEMA_VERSION,
         keywords=["keyword1", "keyword 2"],
         access=[

dandi/tests/test_upload.py

@@ -50,6 +50,10 @@ def test_upload_download(
     (dspath / parent).mkdir()
     copyfile(nwb_file, dspath / parent / name)
     new_dandiset.upload()
+    # The uploaded asset's metadata is derived from a ``BareAsset``, but it must
+    # be uploaded and stored with a ``schemaKey`` of `"Asset"`
+    (asset,) = d.get_assets()
+    assert asset.get_raw_metadata()["schemaKey"] == "Asset"
     download(d.version_api_url, tmp_path)
     assert list_paths(tmp_path) == [
         tmp_path / d.identifier / dandiset_metadata_file,

pyproject.toml

@@ -48,7 +48,10 @@ dependencies = [
     "bids-validator-deno >= 2.0.5",
     "click >= 7.1",
     "click-didyoumean",
-    "dandischema ~= 0.12.0",
+    # Temporarily point at the dandi-schema #419 branch, where
+    # BareAsset.schemaKey is "BareAsset", so CI runs the client against the
+    # consolidated schema to check the schemaKey handling on this branch.
+    "dandischema @ git+https://github.com/dandi/dandi-schema.git@consolidate-models",
     "etelemetry >= 0.2.2",
     "fasteners",
     "fscacher >= 0.3.0",