From 445cc33292a599e3b66e21360630811d014ee106 Mon Sep 17 00:00:00 2001 From: Egor Yakovlev Date: Tue, 29 Sep 2026 22:01:48 +0200 Subject: [PATCH 1/3] ontology: add Dataset, DatasetQuery, FieldLayer (agriculture) Dataset = immutable snapshot of curated files (images/annotations) referencing a manifest file; DatasetQuery = the re-runnable selection criteria (stored separately so the same query can pull more data as the corpus grows); FieldLayer = a georeferenced raster analytics product per field (e.g. Soil Organic Matter, NDVI) an external service writes to the eVault as a GeoTIFF. Used by nlagromap (agri map + imagery library); share ids with annotation/training. Co-Authored-By: Claude Opus 4.8 --- services/ontology/schemas/dataset.json | 38 ++++++++++++ services/ontology/schemas/datasetQuery.json | 32 ++++++++++ services/ontology/schemas/fieldLayer.json | 65 +++++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 services/ontology/schemas/dataset.json create mode 100644 services/ontology/schemas/datasetQuery.json create mode 100644 services/ontology/schemas/fieldLayer.json diff --git a/services/ontology/schemas/dataset.json b/services/ontology/schemas/dataset.json new file mode 100644 index 000000000..4af68ab87 --- /dev/null +++ b/services/ontology/schemas/dataset.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ontology.w3ds.metastate.foundation/schemas/7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e5f", + "schemaId": "7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e5f", + "title": "Dataset", + "domain": "agriculture", + "description": "An immutable snapshot of a curated set of files (images or annotations) selected for downstream use (annotation, model training). It references a manifest file that lists the members and links to the DatasetQuery that produced it. The dataset references file URIs; it does not copy blobs.", + "type": "object", + "properties": { + "schemaVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "recordedAt": { "type": "string", "format": "date-time" }, + "supersedes": { "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" }, + "deleted": { "type": "boolean" }, + "name": { "type": "string", "minLength": 1 }, + "kind": { "type": "string", "enum": ["images", "annotations"] }, + "queryRef": { + "description": "Reference to the DatasetQuery envelope whose criteria produced this snapshot (kept separately so the same criteria can be re-run later to pull more data). May be a bare envelope id or a w3ds:// ref.", + "type": "string", "minLength": 1 + }, + "manifestRef": { + "description": "w3ds:// reference to the manifest file (a table of the member files). Content-addressed via manifestSha256.", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "manifestUrl": { "description": "Convenience direct URL to the manifest file, if published.", "type": "string", "format": "uri" }, + "manifestSha256": { "description": "SHA-256 hex of the manifest content — pins the snapshot for reproducibility.", "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "fileCount": { "type": "integer", "minimum": 0 }, + "observationCount": { "type": "integer", "minimum": 0 }, + "observationRefs": { + "description": "Optional inline list of the source observation ids (only for small sets; large sets live in the manifest).", + "type": "array", "items": { "type": "string" } + }, + "frozen": { "description": "True for an immutable snapshot (the normal case).", "type": "boolean" }, + "createdAt": { "type": "string", "format": "date-time" }, + "createdBy": { "description": "eName of the person who saved the dataset.", "type": "string", "pattern": "^@[^/\\s]+$" } + }, + "required": ["name", "kind", "manifestRef", "createdAt"], + "additionalProperties": true +} diff --git a/services/ontology/schemas/datasetQuery.json b/services/ontology/schemas/datasetQuery.json new file mode 100644 index 000000000..f51a3c9cf --- /dev/null +++ b/services/ontology/schemas/datasetQuery.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ontology.w3ds.metastate.foundation/schemas/7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e60", + "schemaId": "7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e60", + "title": "DatasetQuery", + "domain": "agriculture", + "description": "The re-runnable selection CRITERIA that produced a Dataset, stored separately from the snapshot so the same criteria can be applied to the growing corpus later to pull more data. Referenced by Dataset.queryRef.", + "type": "object", + "properties": { + "schemaVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "recordedAt": { "type": "string", "format": "date-time" }, + "supersedes": { "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" }, + "deleted": { "type": "boolean" }, + "name": { "type": "string", "minLength": 1 }, + "scope": { "description": "Which corpus the criteria run over.", "type": "string", "enum": ["public", "mine"] }, + "access": { "description": "Access filter within one's own vaults.", "type": "string", "enum": ["public", "private", "both"] }, + "query": { "description": "Free-text search term.", "type": "string" }, + "imgMode": { "type": "string", "enum": ["all", "annotated", "unannotated"] }, + "crops": { "type": "array", "items": { "type": "string" } }, + "weeds": { "type": "array", "items": { "type": "string" } }, + "rating": { "type": "array", "items": { "type": "string" } }, + "soil": { "type": "array", "items": { "type": "string" } }, + "robot": { "type": "array", "items": { "type": "string" } }, + "region": { "type": "array", "items": { "type": "string" } }, + "dateFrom": { "anyOf": [{ "type": "string", "format": "date" }, { "type": "string", "format": "date-time" }] }, + "dateTo": { "anyOf": [{ "type": "string", "format": "date" }, { "type": "string", "format": "date-time" }] }, + "createdAt": { "type": "string", "format": "date-time" }, + "createdBy": { "type": "string", "pattern": "^@[^/\\s]+$" } + }, + "required": ["name"], + "additionalProperties": true +} diff --git a/services/ontology/schemas/fieldLayer.json b/services/ontology/schemas/fieldLayer.json new file mode 100644 index 000000000..0ceee2a18 --- /dev/null +++ b/services/ontology/schemas/fieldLayer.json @@ -0,0 +1,65 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ontology.w3ds.metastate.foundation/schemas/7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", + "schemaId": "7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", + "title": "FieldLayer", + "domain": "agriculture", + "description": "A georeferenced raster (or vector) analytics product for one field — e.g. Soil Organic Matter, NDVI, biomass — produced by an external service and written to the eVault as a file. Carries the layer type, acquisition date, per-layer statistics, CRS and bounds, and references the raster file blob (fileRef) plus an optional rendered preview. Distinct from Observation (a point/footprint scan with photos): a FieldLayer is a whole-field gridded product.", + "type": "object", + "properties": { + "schemaVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "recordedAt": { "type": "string", "format": "date-time" }, + "supersedes": { "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" }, + "deleted": { "type": "boolean" }, + "logicalFieldRef": { + "description": "w3ds:// reference to the LogicalField this layer covers.", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "fieldExternalId": { "description": "The producing service's own field id, when there is no LogicalField ref.", "type": "string" }, + "layerType": { + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": "string", "minLength": 1 }, + "dataType": { "type": "string", "enum": ["raster", "vector"] } + }, + "required": ["name", "dataType"], + "additionalProperties": true + }, + "acquiredAt": { "type": "string", "format": "date-time" }, + "acquiredAtEnd": { "anyOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "fileRef": { + "description": "w3ds:// reference to the raster file blob (e.g. a GeoTIFF).", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "previewRef": { + "description": "Optional w3ds:// reference to a rendered preview (e.g. a coloured PNG).", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "format": { "description": "File format of fileRef.", "type": "string", "examples": ["geotiff", "png", "cog"] }, + "crs": { "description": "Coordinate reference system of the raster, e.g. EPSG:28992.", "type": "string" }, + "bbox": { + "description": "Bounds in the raster CRS: [minX, minY, maxX, maxY].", + "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 + }, + "bbox4326": { + "description": "Convenience WGS84 bounds [west, south, east, north] for map overlay.", + "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 + }, + "nodata": { "description": "The nodata sentinel value in the raster.", "type": "number" }, + "unit": { "anyOf": [{ "type": "string" }, { "type": "null" }] }, + "statistics": { + "type": "object", + "properties": { + "count": { "type": "number" }, "min": { "type": "number" }, "max": { "type": "number" }, + "mean": { "type": "number" }, "median": { "type": "number" }, "std": { "type": "number" }, + "variance": { "type": "number" }, "LCL_95": { "type": "number" }, "UCL_95": { "type": "number" } + }, + "additionalProperties": true + }, + "source": { "description": "The producing service (e.g. Bioscope).", "type": "string" }, + "integrity": { "anyOf": [{ "type": "string" }, { "type": "null" }] } + }, + "required": ["layerType", "acquiredAt", "fileRef"], + "additionalProperties": true +} From 867f04150f6e7a3fc658078b77648ae0b7a055fc Mon Sep 17 00:00:00 2001 From: Egor Yakovlev Date: Tue, 29 Sep 2026 22:28:12 +0200 Subject: [PATCH 2/3] =?UTF-8?q?ontology:=20generalise=20FieldLayer=20?= =?UTF-8?q?=E2=86=92=20MapLayer=20(universal=20raster+vector=20layer)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same schemaId (7d5a2c1e-…4e61). MapLayer covers raster (GeoTIFF/COG) AND vector (point/line/polygon/many features, inline GeoJSON or fileRef), with a dataType discriminator + conditional required fields (raster→fileRef+crs; vector→features|fileRef). Subject link generalised from field-specific logicalFieldRef to subjectRef/subjectExternalId. Domain geo (was agriculture) since it is not field-specific. Co-Authored-By: Claude Opus 4.8 --- services/ontology/schemas/fieldLayer.json | 65 ---------------- services/ontology/schemas/mapLayer.json | 93 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 65 deletions(-) delete mode 100644 services/ontology/schemas/fieldLayer.json create mode 100644 services/ontology/schemas/mapLayer.json diff --git a/services/ontology/schemas/fieldLayer.json b/services/ontology/schemas/fieldLayer.json deleted file mode 100644 index 0ceee2a18..000000000 --- a/services/ontology/schemas/fieldLayer.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://ontology.w3ds.metastate.foundation/schemas/7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", - "schemaId": "7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", - "title": "FieldLayer", - "domain": "agriculture", - "description": "A georeferenced raster (or vector) analytics product for one field — e.g. Soil Organic Matter, NDVI, biomass — produced by an external service and written to the eVault as a file. Carries the layer type, acquisition date, per-layer statistics, CRS and bounds, and references the raster file blob (fileRef) plus an optional rendered preview. Distinct from Observation (a point/footprint scan with photos): a FieldLayer is a whole-field gridded product.", - "type": "object", - "properties": { - "schemaVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, - "recordedAt": { "type": "string", "format": "date-time" }, - "supersedes": { "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" }, - "deleted": { "type": "boolean" }, - "logicalFieldRef": { - "description": "w3ds:// reference to the LogicalField this layer covers.", - "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" - }, - "fieldExternalId": { "description": "The producing service's own field id, when there is no LogicalField ref.", "type": "string" }, - "layerType": { - "type": "object", - "properties": { - "id": { "type": "string" }, - "name": { "type": "string", "minLength": 1 }, - "dataType": { "type": "string", "enum": ["raster", "vector"] } - }, - "required": ["name", "dataType"], - "additionalProperties": true - }, - "acquiredAt": { "type": "string", "format": "date-time" }, - "acquiredAtEnd": { "anyOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, - "fileRef": { - "description": "w3ds:// reference to the raster file blob (e.g. a GeoTIFF).", - "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" - }, - "previewRef": { - "description": "Optional w3ds:// reference to a rendered preview (e.g. a coloured PNG).", - "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" - }, - "format": { "description": "File format of fileRef.", "type": "string", "examples": ["geotiff", "png", "cog"] }, - "crs": { "description": "Coordinate reference system of the raster, e.g. EPSG:28992.", "type": "string" }, - "bbox": { - "description": "Bounds in the raster CRS: [minX, minY, maxX, maxY].", - "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 - }, - "bbox4326": { - "description": "Convenience WGS84 bounds [west, south, east, north] for map overlay.", - "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 - }, - "nodata": { "description": "The nodata sentinel value in the raster.", "type": "number" }, - "unit": { "anyOf": [{ "type": "string" }, { "type": "null" }] }, - "statistics": { - "type": "object", - "properties": { - "count": { "type": "number" }, "min": { "type": "number" }, "max": { "type": "number" }, - "mean": { "type": "number" }, "median": { "type": "number" }, "std": { "type": "number" }, - "variance": { "type": "number" }, "LCL_95": { "type": "number" }, "UCL_95": { "type": "number" } - }, - "additionalProperties": true - }, - "source": { "description": "The producing service (e.g. Bioscope).", "type": "string" }, - "integrity": { "anyOf": [{ "type": "string" }, { "type": "null" }] } - }, - "required": ["layerType", "acquiredAt", "fileRef"], - "additionalProperties": true -} diff --git a/services/ontology/schemas/mapLayer.json b/services/ontology/schemas/mapLayer.json new file mode 100644 index 000000000..8bcebe502 --- /dev/null +++ b/services/ontology/schemas/mapLayer.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://ontology.w3ds.metastate.foundation/schemas/7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", + "schemaId": "7d5a2c1e-6b3f-4c8a-9e21-0a1b2c3d4e61", + "title": "MapLayer", + "domain": "geo", + "description": "A universal georeferenced map layer that can be raster (e.g. a GeoTIFF: Soil Organic Matter, NDVI, elevation) OR vector (points, lines, polygons, or many features). Carries the layer type, acquisition time, CRS, bounds, per-layer statistics and optional rendering style, and references the underlying file blob (fileRef → a File/MediaAsset) — vector layers may instead inline small GeoJSON in `features`. Kept generic (not field-specific): the subject it describes is linked via `subjectRef`/`subjectExternalId`. The `dataType` discriminator drives which fields are required.", + "type": "object", + "properties": { + "schemaVersion": { "type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+$" }, + "recordedAt": { "type": "string", "format": "date-time" }, + "supersedes": { "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" }, + "deleted": { "type": "boolean" }, + "name": { "type": "string", "minLength": 1 }, + "dataType": { "description": "Raster grid or vector features.", "type": "string", "enum": ["raster", "vector"] }, + "geometryType": { + "description": "For vector layers: the geometry kind (or 'collection' for mixed / many features).", + "type": "string", + "enum": ["point", "line", "polygon", "multipoint", "multiline", "multipolygon", "collection"] + }, + "layerType": { + "description": "What the layer represents, e.g. { name: 'Soil Organic Matter', dataType: 'raster' }.", + "type": "object", + "properties": { + "id": { "type": "string" }, + "name": { "type": "string", "minLength": 1 }, + "dataType": { "type": "string", "enum": ["raster", "vector"] } + }, + "required": ["name"], + "additionalProperties": true + }, + "subjectRef": { + "description": "w3ds:// reference to what this layer describes (a LogicalField, parcel, region, asset, …). Generic on purpose.", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "subjectExternalId": { "description": "The producing service's own subject id, when there is no w3ds subject ref.", "type": "string" }, + "acquiredAt": { "type": "string", "format": "date-time" }, + "acquiredAtEnd": { "anyOf": [{ "type": "string", "format": "date-time" }, { "type": "null" }] }, + "fileRef": { + "description": "w3ds:// reference to the layer's file blob — a GeoTIFF/COG for raster, or GeoJSON/GeoPackage/FlatGeobuf for large vector.", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "features": { + "description": "Inline GeoJSON for small vector layers (a Geometry, Feature, or FeatureCollection). Large vector data should use fileRef instead.", + "type": "object" + }, + "featureCount": { "type": "integer", "minimum": 0 }, + "previewRef": { + "description": "Optional w3ds:// reference to a rendered preview (e.g. a coloured PNG).", + "type": "string", "pattern": "^w3ds:\\/\\/(@[^/\\s]+)\\/([^/\\s]+)$" + }, + "format": { "description": "File format of fileRef.", "type": "string", "examples": ["geotiff", "cog", "png", "geojson", "geopackage", "flatgeobuf"] }, + "crs": { "description": "Coordinate reference system, e.g. EPSG:28992.", "type": "string" }, + "bbox": { + "description": "Bounds in the layer CRS: [minX, minY, maxX, maxY].", + "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 + }, + "bbox4326": { + "description": "Convenience WGS84 bounds [west, south, east, north] for map overlay.", + "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 + }, + "nodata": { "description": "Raster nodata sentinel value.", "type": "number" }, + "unit": { "anyOf": [{ "type": "string" }, { "type": "null" }] }, + "statistics": { + "description": "Per-layer summary (raster pixel stats or vector attribute stats).", + "type": "object", + "properties": { + "count": { "type": "number" }, "min": { "type": "number" }, "max": { "type": "number" }, + "mean": { "type": "number" }, "median": { "type": "number" }, "std": { "type": "number" }, + "variance": { "type": "number" }, "LCL_95": { "type": "number" }, "UCL_95": { "type": "number" } + }, + "additionalProperties": true + }, + "style": { + "description": "Optional rendering hints — raster: { ramp, min, max }; vector: { stroke, fill, weight, radius }.", + "type": "object", "additionalProperties": true + }, + "source": { "description": "The producing service (e.g. Bioscope).", "type": "string" }, + "integrity": { "anyOf": [{ "type": "string" }, { "type": "null" }] } + }, + "required": ["name", "dataType", "layerType", "acquiredAt"], + "allOf": [ + { + "if": { "properties": { "dataType": { "const": "raster" } } }, + "then": { "required": ["fileRef", "crs"] } + }, + { + "if": { "properties": { "dataType": { "const": "vector" } } }, + "then": { "anyOf": [{ "required": ["features"] }, { "required": ["fileRef"] }] } + } + ], + "additionalProperties": true +} From f8908e8e18e3c35ab81e074147145a107ece4ec9 Mon Sep 17 00:00:00 2001 From: Egor Yakovlev Date: Thu, 1 Oct 2026 08:27:48 +0200 Subject: [PATCH 3/3] Dataset ontology: drop redundant frozen and observationRefs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit frozen is implied — a Dataset is an immutable snapshot by definition, and growing the corpus is DatasetQuery's job (re-run criteria -> new snapshot). observationRefs duplicated the manifest (the single source of truth) and risked drift; the manifest already lists all members, content-addressed. Co-Authored-By: Claude Opus 4.8 --- services/ontology/schemas/dataset.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/ontology/schemas/dataset.json b/services/ontology/schemas/dataset.json index 4af68ab87..8e5633fff 100644 --- a/services/ontology/schemas/dataset.json +++ b/services/ontology/schemas/dataset.json @@ -25,11 +25,6 @@ "manifestSha256": { "description": "SHA-256 hex of the manifest content — pins the snapshot for reproducibility.", "type": "string", "pattern": "^[0-9a-f]{64}$" }, "fileCount": { "type": "integer", "minimum": 0 }, "observationCount": { "type": "integer", "minimum": 0 }, - "observationRefs": { - "description": "Optional inline list of the source observation ids (only for small sets; large sets live in the manifest).", - "type": "array", "items": { "type": "string" } - }, - "frozen": { "description": "True for an immutable snapshot (the normal case).", "type": "boolean" }, "createdAt": { "type": "string", "format": "date-time" }, "createdBy": { "description": "eName of the person who saved the dataset.", "type": "string", "pattern": "^@[^/\\s]+$" } },