Establish Core 0.4 capabilities and compatibility release gates

README.md

@@ -56,7 +56,7 @@ full-match regex values:
 }
 ```
 
-`scan_and_transform` uses `{ scan?: { locale?: string }, transform: ... }` so
+`scan_and_transform` uses `{ scan?: { locale?: string, detect_uuid?: boolean }, transform: ... }` so
 detection settings remain separate from transformation policy.
 
 ## Packages
@@ -278,7 +278,8 @@ fixtures/           Shared conformance fixtures
 
 Pass `{"locale":"de"}` to text or structured scans. Trimmed, ASCII
 case-insensitive `de`, `de-DE`, and `de_DE` activate all seven German detectors;
-omitted and other nonempty locales keep base detection only.
+omitted locale and recognized `en-US`/`fr` aliases keep base detection only.
+The 0.4.0 candidate rejects unsupported explicit locales.
 
 ```python
 from datafog_core import scan_and_transform
@@ -307,3 +308,5 @@ structured scans, independently of locale. It emits `UUID` findings for versions
 1–8 with the IETF variant and original casing/ranges. UUID syntax does not imply
 sensitivity. See the [UUID reference](docs/reference/uuid.mdx) for boundaries,
 excluded sentinel forms and transformation examples.
+
+The source candidate targets **0.4.0**; publication and downstream Python integration are separate release gates. See the [candidate release checklist](docs/releases/0-4-0.mdx), [runtime capabilities](docs/reference/capabilities.mdx), and [0.4.x compatibility policy](docs/reference/compatibility.mdx).

bindings/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-node"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2024"
 rust-version = "1.88"
 description = "Node.js bindings for datafog-core"

bindings/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datafog/node",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Fast structured PII detection for Node.js, powered by Rust",
   "license": "MIT",
   "repository": {

bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core-python"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2024"
 rust-version = "1.88"
 description = "Python bindings for datafog-core"

bindings/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "datafog-core"
-version = "0.3.1"
+version = "0.4.0"
 description = "Fast structured PII detection implemented in Rust"
 readme = "../../README.md"
 license = "MIT"

bindings/python/python/datafog_core/__init__.py

@@ -19,6 +19,7 @@
     TextRange as TextRange,
     Transformation as Transformation,
     TransformResult as TransformResult,
+    capabilities as capabilities,
     discover_fields as discover_fields,
     scan as scan,
     scan_and_transform as scan_and_transform,
@@ -47,6 +48,7 @@
     "StructuredRestoration",
     "StructuredRestoreResult",
     "PrivacyManager",
+    "capabilities",
     "scan",
     "transform",
     "scan_and_transform",

bindings/python/python/datafog_core/__init__.pyi

@@ -12,6 +12,26 @@ class _ScanConfig(TypedDict, total=False):
     locale: str
     detect_uuid: bool
 
+class _CapabilityActivationRequired(TypedDict):
+    kind: Literal["default", "locale", "config", "structured"]
+
+class _CapabilityActivation(_CapabilityActivationRequired, total=False):
+    scan_config: _ScanConfig
+
+class _EntityCapability(TypedDict):
+    scopes: list[Literal["structured", "text"]]
+    activation: _CapabilityActivation
+
+class _LocaleCapability(TypedDict):
+    enabled_entities: list[str]
+
+class _Capabilities(TypedDict):
+    contract_version: Literal[1]
+    supported_entities: list[str]
+    default_entities: list[str]
+    locales: dict[str, _LocaleCapability]
+    entities: dict[str, _EntityCapability]
+
 class _StructuredScanConfig(_ScanConfig, total=False):
     discover_person: bool
     mappings: dict[str, Literal["PERSON"]]
@@ -359,6 +379,8 @@ class PrivacyManager:
         self, data: _JsonDocument, context: _PrivacyContext
     ) -> Awaitable[StructuredRestoreResult]: ...
 
+def capabilities() -> _Capabilities: ...
+
 def scan(text: str, config: _ScanConfig | None = None) -> list[Finding]: ...
 def transform(
     text: str, findings: Sequence[Finding], config: _TransformationConfig

bindings/python/src/lib.rs

@@ -956,6 +956,14 @@ fn privacy_error(py: Python<'_>, error: core::PrivacyError) -> PyErr {
     exception
 }
 
+/// Return an independent, JSON-compatible snapshot of the Core capability contract.
+#[pyfunction]
+fn capabilities(py: Python<'_>) -> PyResult<Py<PyAny>> {
+    let json = serde_json::to_string(&core::capabilities())
+        .map_err(|_| internal_error(py, "failed to serialize Core capabilities"))?;
+    Ok(py.import("json")?.call_method1("loads", (json,))?.unbind())
+}
+
 /// Scan text for supported PII findings.
 #[pyfunction]
 #[pyo3(signature = (text, config=None))]
@@ -1300,6 +1308,7 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_class::<Restoration>()?;
     module.add_class::<RestoreResult>()?;
     module.add_class::<PrivacyManager>()?;
+    module.add_function(wrap_pyfunction!(capabilities, module)?)?;
     module.add_function(wrap_pyfunction!(scan, module)?)?;
     module.add_function(wrap_pyfunction!(transform, module)?)?;
     module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?;

bindings/python/tests/capabilities_conformance.py

@@ -0,0 +1,86 @@
+"""Capability discovery contract against the installed, compiled Core wheel."""
+import json
+from pathlib import Path
+
+import datafog_core as api
+
+SUPPORTED = sorted([
+    "CREDIT_CARD", "DATE", "DE_IBAN", "DE_PASSPORT_NUMBER", "DE_POSTAL_CODE",
+    "DE_RESIDENCE_PERMIT_NUMBER", "DE_SOCIAL_SECURITY_NUMBER", "DE_TAX_ID",
+    "DE_VAT_ID", "EMAIL", "IP_ADDRESS", "JWT", "NPI", "PERSON", "PHONE",
+    "PRIVATE_KEY", "SSN", "US_ROUTING_NUMBER", "UUID", "ZIP_CODE",
+])
+GERMAN = [entity for entity in SUPPORTED if entity.startswith("DE_")]
+DEFAULT = [entity for entity in SUPPORTED if entity not in GERMAN + ["PERSON", "UUID"]]
+
+
+def verify():
+    assert callable(api.capabilities)
+    try:
+        api.capabilities({})
+    except TypeError:
+        pass
+    else:
+        raise AssertionError("capabilities accepted arguments")
+    snapshot = api.capabilities()
+    assert type(snapshot) is dict
+    assert snapshot["contract_version"] == 1
+    assert json.loads(json.dumps(snapshot)) == snapshot
+    assert snapshot["supported_entities"] == SUPPORTED
+    assert snapshot["default_entities"] == DEFAULT
+    assert sorted(snapshot["entities"]) == SUPPORTED
+    for key in ("supported_entities", "default_entities"):
+        assert snapshot[key] == sorted(set(snapshot[key]))
+    assert set(snapshot["locales"]) == {"de", "de-DE", "de_DE", "en-US", "fr"}
+    for locale, metadata in snapshot["locales"].items():
+        assert metadata["enabled_entities"] == (GERMAN if locale.startswith("de") else [])
+    for label, entity in snapshot["entities"].items():
+        assert entity["scopes"] == (["structured"] if label == "PERSON" else ["structured", "text"])
+        kind = "locale" if label in GERMAN else "config" if label == "UUID" else "structured" if label == "PERSON" else "default"
+        assert entity["activation"]["kind"] == kind
+    uuid_config = snapshot["entities"]["UUID"]["activation"]["scan_config"]
+    assert uuid_config == {"detect_uuid": True}
+    text = "👋 café é 550e8400-e29b-41d4-a716-446655440000"
+    assert not any(f.entity_type == "UUID" for f in api.scan(text))
+    uuid_findings = [f for f in api.scan(text, uuid_config) if f.entity_type == "UUID"]
+    assert len(uuid_findings) == 1
+    finding = uuid_findings[0]
+    assert text[finding.codepoint_range.start:finding.codepoint_range.end] == finding.matched_text
+    assert text.encode()[finding.byte_range.start:finding.byte_range.end].decode() == finding.matched_text
+    samples = {}
+    for line in (Path(__file__).resolve().parents[3] / "fixtures/german.jsonl").read_text().splitlines():
+        row = json.loads(line)
+        for expected in row["entities"]:
+            samples.setdefault(expected["label"], row["text"])
+    for locale in ("de", "de-DE", "de_DE", "DE-de", " De "):
+        for label, text in samples.items():
+            assert any(f.entity_type == label for f in api.scan(text, {"locale": locale})), (locale, label)
+    for label, text in samples.items():
+        activation = snapshot["entities"][label]["activation"]["scan_config"]
+        assert any(f.entity_type == label for f in api.scan(text, activation))
+    for locale in ("en-US", "fr"):
+        assert api.scan("jane@example.com", {"locale": locale}) == api.scan("jane@example.com")
+    for call, path in [
+        (lambda: api.scan("", {"locale": "zz-ZZ"}), "/locale"),
+        (lambda: api.scan_structured({}, {"locale": "zz-ZZ"}), "/locale"),
+        (lambda: api.scan_and_transform("", {"scan": {"locale": "zz-ZZ"}, "transform": {"default": {"strategy": "redact"}}}), "/scan/locale"),
+        (lambda: api.scan_and_transform_structured({}, {"scan": {"locale": "zz-ZZ"}, "transform": {"default": {"strategy": "redact"}}}), "/scan/locale"),
+    ]:
+        try:
+            call()
+        except api.DataFogConfigurationError as error:
+            assert error.code == "invalid_configuration"
+            assert error.reason == "invalid_value"
+            assert error.path == path
+        else:
+            raise AssertionError("unsupported locale accepted")
+    assert not any(f.entity_type == "PERSON" for f in api.scan("Jane Doe"))
+    assert any(f.finding.entity_type == "PERSON" for f in api.scan_structured({"full_name": "Jane Doe"}).findings)
+    mutated = api.capabilities()
+    assert mutated is not snapshot
+    mutated["supported_entities"].clear()
+    mutated["default_entities"].append("FAKE")
+    mutated["locales"]["de"]["enabled_entities"].clear()
+    mutated["entities"]["EMAIL"]["scopes"].clear()
+    mutated["entities"]["UUID"]["activation"]["scan_config"]["detect_uuid"] = False
+    assert api.capabilities() == snapshot

bindings/python/tests/test_installed.py

@@ -5,6 +5,7 @@
 import json
 import asyncio
 from pathlib import Path
+import capabilities_conformance
 import german_conformance
 import jwt_conformance
 import private_key_conformance
@@ -121,6 +122,7 @@ def verify_structured() -> None:
 
 
 def main() -> None:
+    capabilities_conformance.verify()
     german_conformance.verify()
     jwt_conformance.verify()
     private_key_conformance.verify()

bindings/python/tests/test_typing.py

@@ -20,15 +20,19 @@
     Restoration, RestoreResult, FieldMapping, StructuredFinding,
     StructuredScanResult, StructuredTransformation, StructuredTransformResult,
     StructuredRestoration, StructuredRestoreResult, PrivacyManager,
-    scan, transform, scan_and_transform, discover_fields, scan_structured,
+    capabilities, scan, transform, scan_and_transform, discover_fields, scan_structured,
     transform_structured, scan_and_transform_structured,
 )
 if TYPE_CHECKING:
     from datafog_core import (
         _TransformationConfig, _JsonDocument, _TokenizeItem, _TokenizeResult,
-        _RestoreItem, _RestoreResult, _ResolvedKey,
+        _RestoreItem, _RestoreResult, _ResolvedKey, _Capabilities,
     )
 
+assert_type(capabilities(), _Capabilities)
+assert_type(capabilities()["supported_entities"], list[str])
+assert_type(capabilities()["locales"]["de"]["enabled_entities"], list[str])
+
 config: _TransformationConfig = {
     "default": {"strategy": "mask", "character": "*", "reveal": {"direction": "last", "count": 4}},
     "entities": ["EMAIL"],
@@ -120,7 +124,8 @@ async def use_manager() -> None:
 
 # Each line must produce a diagnostic; a missing stub or Any cannot satisfy this.
 INVALID = """
-from datafog_core import scan, transform, scan_structured, PrivacyManager, TransformResult
+from datafog_core import capabilities, scan, transform, scan_structured, PrivacyManager, TransformResult
+capabilities(1)  # error
 scan(123)  # error
 scan("text", {"locale": 123})  # error
 scan("text", {"detect_uuid": "true"})  # error

bindings/wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-wasm"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2024"
 rust-version = "1.88"
 description = "WebAssembly bindings for datafog-core"

bindings/wasm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datafog/wasm",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Fast structured PII detection for browsers, powered by Rust and WebAssembly",
   "license": "MIT",
   "repository": {

crates/core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "datafog-core"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2024"
 rust-version = "1.88"
 description = "Fast structured PII detection library"