From be7f62a31aedea5ae11f68a6365b724e4c477e77 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:58:26 -0700 Subject: [PATCH] Establish Core 0.4 capability contract and compatibility release gates --- Cargo.lock | 8 +- README.md | 7 +- bindings/node/Cargo.toml | 2 +- bindings/node/package-lock.json | 4 +- bindings/node/package.json | 2 +- bindings/python/Cargo.toml | 2 +- bindings/python/pyproject.toml | 2 +- .../python/python/datafog_core/__init__.py | 2 + .../python/python/datafog_core/__init__.pyi | 22 ++ bindings/python/src/lib.rs | 9 + .../python/tests/capabilities_conformance.py | 86 +++++++ bindings/python/tests/test_installed.py | 2 + bindings/python/tests/test_typing.py | 11 +- bindings/wasm/Cargo.toml | 2 +- bindings/wasm/package-lock.json | 4 +- bindings/wasm/package.json | 2 +- crates/core/Cargo.toml | 2 +- crates/core/src/capabilities.rs | 213 ++++++++++++++++++ crates/core/src/german.rs | 73 +++--- crates/core/src/lib.rs | 32 ++- crates/core/src/structured.rs | 10 +- crates/core/tests/capabilities.rs | 211 +++++++++++++++++ docs/development.mdx | 12 +- docs/docs.json | 23 +- docs/get-started/installation.mdx | 2 + docs/guides/configuration.mdx | 8 +- docs/index.mdx | 2 + docs/reference/browser-wasm.mdx | 2 + docs/reference/capabilities.mdx | 105 +++++++++ docs/reference/compatibility.mdx | 48 ++++ docs/reference/german-entities.mdx | 6 +- docs/reference/jwt.mdx | 2 +- docs/reference/node.mdx | 2 + docs/reference/npi.mdx | 2 +- docs/reference/private-keys.mdx | 2 +- docs/reference/python.mdx | 6 +- docs/reference/rust.mdx | 4 + docs/reference/us-routing-number.mdx | 2 +- docs/reference/uuid.mdx | 2 +- docs/releases/0-4-0.mdx | 77 +++++++ 40 files changed, 913 insertions(+), 102 deletions(-) create mode 100644 bindings/python/tests/capabilities_conformance.py create mode 100644 crates/core/src/capabilities.rs create mode 100644 crates/core/tests/capabilities.rs create mode 100644 docs/reference/capabilities.mdx create mode 100644 docs/reference/compatibility.mdx create mode 100644 docs/releases/0-4-0.mdx diff --git a/Cargo.lock b/Cargo.lock index 07971d5d..b095a198 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" [[package]] name = "datafog-core" -version = "0.3.1" +version = "0.4.0" dependencies = [ "base64", "futures", @@ -94,7 +94,7 @@ dependencies = [ [[package]] name = "datafog-core-python" -version = "0.3.1" +version = "0.4.0" dependencies = [ "datafog-core", "pyo3", @@ -104,7 +104,7 @@ dependencies = [ [[package]] name = "datafog-node" -version = "0.3.1" +version = "0.4.0" dependencies = [ "datafog-core", "napi", @@ -115,7 +115,7 @@ dependencies = [ [[package]] name = "datafog-wasm" -version = "0.3.1" +version = "0.4.0" dependencies = [ "datafog-core", "serde", diff --git a/README.md b/README.md index 2776bca2..50a356fa 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ full-match regex values: } ``` -`scan_and_transform` uses `{ scan?: { locale?: string }, transform: ... }` so +`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so detection settings remain separate from transformation policy. ## Packages @@ -278,7 +278,8 @@ fixtures/ Shared conformance fixtures Pass `{"locale":"de"}` to text or structured scans. Trimmed, ASCII case-insensitive `de`, `de-DE`, and `de_DE` activate all seven German detectors; -omitted and other nonempty locales keep base detection only. +omitted locale and recognized `en-US`/`fr` aliases keep base detection only. +The 0.4.0 candidate rejects unsupported explicit locales. ```python from datafog_core import scan_and_transform @@ -307,3 +308,5 @@ structured scans, independently of locale. It emits `UUID` findings for versions 1–8 with the IETF variant and original casing/ranges. UUID syntax does not imply sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries, excluded sentinel forms and transformation examples. + +The source candidate targets **0.4.0**; publication and downstream Python integration are separate release gates. See the [candidate release checklist](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx). diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 76cbc805..4e88754f 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-node" -version = "0.3.1" +version = "0.4.0" edition = "2024" rust-version = "1.88" description = "Node.js bindings for datafog-core" diff --git a/bindings/node/package-lock.json b/bindings/node/package-lock.json index f5ffd6de..b097d5a3 100644 --- a/bindings/node/package-lock.json +++ b/bindings/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/node", - "version": "0.3.1", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/node", - "version": "0.3.1", + "version": "0.4.0", "license": "MIT", "devDependencies": { "@napi-rs/cli": "3.8.6", diff --git a/bindings/node/package.json b/bindings/node/package.json index 718c0d43..3b6083a1 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/node", - "version": "0.3.1", + "version": "0.4.0", "description": "Fast structured PII detection for Node.js, powered by Rust", "license": "MIT", "repository": { diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 5ff13df6..ac31952b 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core-python" -version = "0.3.1" +version = "0.4.0" edition = "2024" rust-version = "1.88" description = "Python bindings for datafog-core" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 9d030276..199cca05 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "datafog-core" -version = "0.3.1" +version = "0.4.0" description = "Fast structured PII detection implemented in Rust" readme = "../../README.md" license = "MIT" diff --git a/bindings/python/python/datafog_core/__init__.py b/bindings/python/python/datafog_core/__init__.py index c26a7b3f..25afa7e2 100644 --- a/bindings/python/python/datafog_core/__init__.py +++ b/bindings/python/python/datafog_core/__init__.py @@ -19,6 +19,7 @@ TextRange as TextRange, Transformation as Transformation, TransformResult as TransformResult, + capabilities as capabilities, discover_fields as discover_fields, scan as scan, scan_and_transform as scan_and_transform, @@ -47,6 +48,7 @@ "StructuredRestoration", "StructuredRestoreResult", "PrivacyManager", + "capabilities", "scan", "transform", "scan_and_transform", diff --git a/bindings/python/python/datafog_core/__init__.pyi b/bindings/python/python/datafog_core/__init__.pyi index ac058444..a3753c6d 100644 --- a/bindings/python/python/datafog_core/__init__.pyi +++ b/bindings/python/python/datafog_core/__init__.pyi @@ -12,6 +12,26 @@ class _ScanConfig(TypedDict, total=False): locale: str detect_uuid: bool +class _CapabilityActivationRequired(TypedDict): + kind: Literal["default", "locale", "config", "structured"] + +class _CapabilityActivation(_CapabilityActivationRequired, total=False): + scan_config: _ScanConfig + +class _EntityCapability(TypedDict): + scopes: list[Literal["structured", "text"]] + activation: _CapabilityActivation + +class _LocaleCapability(TypedDict): + enabled_entities: list[str] + +class _Capabilities(TypedDict): + contract_version: Literal[1] + supported_entities: list[str] + default_entities: list[str] + locales: dict[str, _LocaleCapability] + entities: dict[str, _EntityCapability] + class _StructuredScanConfig(_ScanConfig, total=False): discover_person: bool mappings: dict[str, Literal["PERSON"]] @@ -359,6 +379,8 @@ class PrivacyManager: self, data: _JsonDocument, context: _PrivacyContext ) -> Awaitable[StructuredRestoreResult]: ... +def capabilities() -> _Capabilities: ... + def scan(text: str, config: _ScanConfig | None = None) -> list[Finding]: ... def transform( text: str, findings: Sequence[Finding], config: _TransformationConfig diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index cfe0a7e0..ba1434be 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -956,6 +956,14 @@ fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr { exception } +/// Return an independent, JSON-compatible snapshot of the Core capability contract. +#[pyfunction] +fn capabilities(py: Python<'_>) -> PyResult> { + let json = serde_json::to_string(&core::capabilities()) + .map_err(|_| internal_error(py, "failed to serialize Core capabilities"))?; + Ok(py.import("json")?.call_method1("loads", (json,))?.unbind()) +} + /// Scan text for supported PII findings. #[pyfunction] #[pyo3(signature = (text, config=None))] @@ -1300,6 +1308,7 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; + module.add_function(wrap_pyfunction!(capabilities, module)?)?; module.add_function(wrap_pyfunction!(scan, module)?)?; module.add_function(wrap_pyfunction!(transform, module)?)?; module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?; diff --git a/bindings/python/tests/capabilities_conformance.py b/bindings/python/tests/capabilities_conformance.py new file mode 100644 index 00000000..a8111b7d --- /dev/null +++ b/bindings/python/tests/capabilities_conformance.py @@ -0,0 +1,86 @@ +"""Capability discovery contract against the installed, compiled Core wheel.""" +import json +from pathlib import Path + +import datafog_core as api + +SUPPORTED = sorted([ + "CREDIT_CARD", "DATE", "DE_IBAN", "DE_PASSPORT_NUMBER", "DE_POSTAL_CODE", + "DE_RESIDENCE_PERMIT_NUMBER", "DE_SOCIAL_SECURITY_NUMBER", "DE_TAX_ID", + "DE_VAT_ID", "EMAIL", "IP_ADDRESS", "JWT", "NPI", "PERSON", "PHONE", + "PRIVATE_KEY", "SSN", "US_ROUTING_NUMBER", "UUID", "ZIP_CODE", +]) +GERMAN = [entity for entity in SUPPORTED if entity.startswith("DE_")] +DEFAULT = [entity for entity in SUPPORTED if entity not in GERMAN + ["PERSON", "UUID"]] + + +def verify(): + assert callable(api.capabilities) + try: + api.capabilities({}) + except TypeError: + pass + else: + raise AssertionError("capabilities accepted arguments") + snapshot = api.capabilities() + assert type(snapshot) is dict + assert snapshot["contract_version"] == 1 + assert json.loads(json.dumps(snapshot)) == snapshot + assert snapshot["supported_entities"] == SUPPORTED + assert snapshot["default_entities"] == DEFAULT + assert sorted(snapshot["entities"]) == SUPPORTED + for key in ("supported_entities", "default_entities"): + assert snapshot[key] == sorted(set(snapshot[key])) + assert set(snapshot["locales"]) == {"de", "de-DE", "de_DE", "en-US", "fr"} + for locale, metadata in snapshot["locales"].items(): + assert metadata["enabled_entities"] == (GERMAN if locale.startswith("de") else []) + for label, entity in snapshot["entities"].items(): + assert entity["scopes"] == (["structured"] if label == "PERSON" else ["structured", "text"]) + kind = "locale" if label in GERMAN else "config" if label == "UUID" else "structured" if label == "PERSON" else "default" + assert entity["activation"]["kind"] == kind + uuid_config = snapshot["entities"]["UUID"]["activation"]["scan_config"] + assert uuid_config == {"detect_uuid": True} + text = "👋 café é 550e8400-e29b-41d4-a716-446655440000" + assert not any(f.entity_type == "UUID" for f in api.scan(text)) + uuid_findings = [f for f in api.scan(text, uuid_config) if f.entity_type == "UUID"] + assert len(uuid_findings) == 1 + finding = uuid_findings[0] + assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text + assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text + samples = {} + for line in (Path(__file__).resolve().parents[3] / "fixtures/german.jsonl").read_text().splitlines(): + row = json.loads(line) + for expected in row["entities"]: + samples.setdefault(expected["label"], row["text"]) + for locale in ("de", "de-DE", "de_DE", "DE-de", " De "): + for label, text in samples.items(): + assert any(f.entity_type == label for f in api.scan(text, {"locale": locale})), (locale, label) + for label, text in samples.items(): + activation = snapshot["entities"][label]["activation"]["scan_config"] + assert any(f.entity_type == label for f in api.scan(text, activation)) + for locale in ("en-US", "fr"): + assert api.scan("jane@example.com", {"locale": locale}) == api.scan("jane@example.com") + for call, path in [ + (lambda: api.scan("", {"locale": "zz-ZZ"}), "/locale"), + (lambda: api.scan_structured({}, {"locale": "zz-ZZ"}), "/locale"), + (lambda: api.scan_and_transform("", {"scan": {"locale": "zz-ZZ"}, "transform": {"default": {"strategy": "redact"}}}), "/scan/locale"), + (lambda: api.scan_and_transform_structured({}, {"scan": {"locale": "zz-ZZ"}, "transform": {"default": {"strategy": "redact"}}}), "/scan/locale"), + ]: + try: + call() + except api.DataFogConfigurationError as error: + assert error.code == "invalid_configuration" + assert error.reason == "invalid_value" + assert error.path == path + else: + raise AssertionError("unsupported locale accepted") + assert not any(f.entity_type == "PERSON" for f in api.scan("Jane Doe")) + assert any(f.finding.entity_type == "PERSON" for f in api.scan_structured({"full_name": "Jane Doe"}).findings) + mutated = api.capabilities() + assert mutated is not snapshot + mutated["supported_entities"].clear() + mutated["default_entities"].append("FAKE") + mutated["locales"]["de"]["enabled_entities"].clear() + mutated["entities"]["EMAIL"]["scopes"].clear() + mutated["entities"]["UUID"]["activation"]["scan_config"]["detect_uuid"] = False + assert api.capabilities() == snapshot diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index 66ab7661..1d6b70ec 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -5,6 +5,7 @@ import json import asyncio from pathlib import Path +import capabilities_conformance import german_conformance import jwt_conformance import private_key_conformance @@ -121,6 +122,7 @@ def verify_structured() -> None: def main() -> None: + capabilities_conformance.verify() german_conformance.verify() jwt_conformance.verify() private_key_conformance.verify() diff --git a/bindings/python/tests/test_typing.py b/bindings/python/tests/test_typing.py index c98a476a..a37febb1 100644 --- a/bindings/python/tests/test_typing.py +++ b/bindings/python/tests/test_typing.py @@ -20,15 +20,19 @@ Restoration, RestoreResult, FieldMapping, StructuredFinding, StructuredScanResult, StructuredTransformation, StructuredTransformResult, StructuredRestoration, StructuredRestoreResult, PrivacyManager, - scan, transform, scan_and_transform, discover_fields, scan_structured, + capabilities, scan, transform, scan_and_transform, discover_fields, scan_structured, transform_structured, scan_and_transform_structured, ) if TYPE_CHECKING: from datafog_core import ( _TransformationConfig, _JsonDocument, _TokenizeItem, _TokenizeResult, - _RestoreItem, _RestoreResult, _ResolvedKey, + _RestoreItem, _RestoreResult, _ResolvedKey, _Capabilities, ) +assert_type(capabilities(), _Capabilities) +assert_type(capabilities()["supported_entities"], list[str]) +assert_type(capabilities()["locales"]["de"]["enabled_entities"], list[str]) + config: _TransformationConfig = { "default": {"strategy": "mask", "character": "*", "reveal": {"direction": "last", "count": 4}}, "entities": ["EMAIL"], @@ -120,7 +124,8 @@ async def use_manager() -> None: # Each line must produce a diagnostic; a missing stub or Any cannot satisfy this. INVALID = """ -from datafog_core import scan, transform, scan_structured, PrivacyManager, TransformResult +from datafog_core import capabilities, scan, transform, scan_structured, PrivacyManager, TransformResult +capabilities(1) # error scan(123) # error scan("text", {"locale": 123}) # error scan("text", {"detect_uuid": "true"}) # error diff --git a/bindings/wasm/Cargo.toml b/bindings/wasm/Cargo.toml index 37fda8da..29ba8459 100644 --- a/bindings/wasm/Cargo.toml +++ b/bindings/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-wasm" -version = "0.3.1" +version = "0.4.0" edition = "2024" rust-version = "1.88" description = "WebAssembly bindings for datafog-core" diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json index 2fc86b92..63a27d9d 100644 --- a/bindings/wasm/package-lock.json +++ b/bindings/wasm/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/wasm", - "version": "0.3.1", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/wasm", - "version": "0.3.1", + "version": "0.4.0", "license": "MIT", "devDependencies": { "playwright": "1.62.1", diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json index fa0c98b7..c103faae 100644 --- a/bindings/wasm/package.json +++ b/bindings/wasm/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/wasm", - "version": "0.3.1", + "version": "0.4.0", "description": "Fast structured PII detection for browsers, powered by Rust and WebAssembly", "license": "MIT", "repository": { diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 14af6d3c..0272e3a0 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core" -version = "0.3.1" +version = "0.4.0" edition = "2024" rust-version = "1.88" description = "Fast structured PII detection library" diff --git a/crates/core/src/capabilities.rs b/crates/core/src/capabilities.rs new file mode 100644 index 00000000..6bd8f8dd --- /dev/null +++ b/crates/core/src/capabilities.rs @@ -0,0 +1,213 @@ +//! Versioned detector inventory derived from the same registry used by scans. +use crate::{Candidate, Label, ScanConfig, german, structured}; +use serde::Serialize; +use std::{collections::BTreeMap, sync::LazyLock}; + +/// Core capability contract. Version 1 is stable throughout the 0.4 series. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Capabilities { + pub contract_version: u32, + pub supported_entities: Vec, + /// Entities enabled by a default text scan; structured PERSON is separate. + pub default_entities: Vec, + pub locales: BTreeMap, + pub entities: BTreeMap, +} + +/// Additional entities enabled by an accepted locale identifier. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LocaleCapabilities { + pub enabled_entities: Vec, +} + +/// Scan scopes and activation recipe for an entity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct EntityCapabilities { + pub scopes: Vec, + pub activation: EntityActivation, +} + +/// Configuration required to activate detection; transformation selection is separate. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum EntityActivation { + Default, + Locale { scan_config: ActivationScanConfig }, + Config { scan_config: ActivationScanConfig }, + Structured, +} + +/// A minimal scan configuration fragment that enables the entity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ActivationScanConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub locale: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub detect_uuid: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Activation { + Default, + German, + Uuid, +} + +impl Activation { + fn enabled(self, config: &ScanConfig) -> bool { + match self { + Self::Default => true, + Self::German => config.locale().and_then(locale_activation) == Some(Self::German), + Self::Uuid => config.uuid_detection_enabled(), + } + } + + fn metadata(self) -> EntityActivation { + match self { + Self::Default => EntityActivation::Default, + Self::German => EntityActivation::Locale { + scan_config: ActivationScanConfig { + locale: Some("de".into()), + detect_uuid: None, + }, + }, + Self::Uuid => EntityActivation::Config { + scan_config: ActivationScanConfig { + locale: None, + detect_uuid: Some(true), + }, + }, + } + } +} + +struct Detector { + labels: Vec