Draft: Python optional Blob and SecureDocument modules (F2 lifecycle in progress)

.gitattributes

@@ -1 +1,3 @@
 test-vectors/*.tsv whitespace=-blank-at-eol
+tests/fixtures/reference-modules/*.doc binary
+tests/fixtures/reference-modules/*.xls binary

.github/workflows/example-gate.yml

@@ -21,6 +21,7 @@ jobs:
           python-version: '3.10'
           cache: pip
       - run: pip install -e ".[test,telemetry,object-storage-s3]"
+      - run: pip install --no-deps -e ./packages/teaql-blob-reference
       - name: Start dedicated S3 example fixture
         run: |
           docker run -d --name teaql-python-example-s3 \

.github/workflows/optional-blob-wheel.yml

@@ -0,0 +1,58 @@
+name: Optional reference wheels
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+jobs:
+  optional-wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: pip
+      - run: python -m pip install --upgrade build
+      - run: python -m pip install -e ".[test,object-storage-s3]"
+      - run: python -m pip install -e "./packages/teaql-blob-reference[image,pdf,office]"
+      - run: python -m pip install --no-deps -e ./packages/teaql-secure-document-reference
+      - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py tests/provider/sqlite/test_blob_module.py
+      - run: python -m pytest -q packages/teaql-secure-document-reference/tests
+      - run: python -m build --wheel --no-isolation --outdir .artifacts/base .
+      - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference
+      - run: python -m build --wheel --no-isolation --outdir .artifacts/secure ./packages/teaql-secure-document-reference
+      - name: Check wheel isolation
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+          from zipfile import ZipFile
+
+          base = list(Path('.artifacts/base').glob('*.whl'))
+          blob = list(Path('.artifacts/blob').glob('*.whl'))
+          secure = list(Path('.artifacts/secure').glob('*.whl'))
+          assert len(base) == len(blob) == len(secure) == 1, (base, blob, secure)
+          with ZipFile(base[0]) as wheel:
+              entries = wheel.namelist()
+              assert any(name.startswith('teaql/') for name in entries)
+              assert not any(name.startswith('teaql_blob_reference/') for name in entries)
+              assert not any(name.startswith('teaql_secure_document_reference/') for name in entries)
+          with ZipFile(blob[0]) as wheel:
+              entries = wheel.namelist()
+              assert any(name.startswith('teaql_blob_reference/') for name in entries)
+              assert not any(name.startswith('teaql/') for name in entries)
+              assert not any(name.startswith('teaql_secure_document_reference/') for name in entries)
+              for module in ('__init__.py', 'module.py', 'mutation_participant.py',
+                             'file_storage.py', 'sqlite_catalog.py', 'upload_service.py'):
+                  name = f'teaql_blob_reference/{module}'
+                  assert name in entries
+                  assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes()
+          with ZipFile(secure[0]) as wheel:
+              entries = wheel.namelist()
+              assert any(name.startswith('teaql_secure_document_reference/') for name in entries)
+              assert not any(name.startswith('teaql/') or name.startswith('teaql_blob_reference/') for name in entries)
+              for module in ('__init__.py', 'round_trip_reference.py', 'document_round_trip.py'):
+                  name = f'teaql_secure_document_reference/{module}'
+                  assert name in entries
+                  assert wheel.read(name) == (Path('packages/teaql-secure-document-reference/src') / name).read_bytes()
+          PY

.github/workflows/release.yml

@@ -8,17 +8,38 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
       - run: python -m pip install --upgrade build
       - run: python -m pip install --editable ".[test,telemetry]"
+      - run: python -m pip install --editable "./packages/teaql-blob-reference[image,pdf,office]"
+      - run: python -m pip install --no-deps --editable ./packages/teaql-secure-document-reference
       - name: Verify tag matches package version
         run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')"
+      - name: Require a verified signed release tag
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python scripts/verify_signed_release_tag.py
+      - name: Test signed release tag verifier
+        run: python scripts/test_verify_signed_release_tag.py
+      - name: Test PyPI version-absence verifier
+        run: python scripts/test_verify_public_release_absence.py
       - run: python -m pytest
-      - run: python -m build
+      - run: python -m pytest packages/teaql-secure-document-reference/tests
+      - run: python -m build --outdir dist .
+      - run: python -m build --outdir dist ./packages/teaql-blob-reference
+      - run: python -m build --outdir dist ./packages/teaql-secure-document-reference
+      - name: Verify exact release wheels and source distributions
+        run: python scripts/verify-reference-release-artifacts.py dist
+      - name: Run four isolated consumers from exact wheels
+        run: python scripts/verify-reference-release-consumers.py dist
+      - name: Resolve and install exact wheels with declared dependencies in clean environments
+        run: python scripts/verify-reference-release-dependencies.py dist
       - uses: actions/upload-artifact@v4
         with:
           name: python-distributions
@@ -31,10 +52,21 @@ jobs:
       name: pypi
       url: https://pypi.org/project/teaql/
     permissions:
+      contents: read
       id-token: write
     steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
       - uses: actions/download-artifact@v4
         with:
           name: python-distributions
           path: dist/
+      - name: Reverify exact downloaded distributions
+        run: python scripts/verify-reference-release-artifacts.py dist
+      - name: Refuse any occupied PyPI version
+        run: python scripts/verify_public_release_absence.py dist
       - uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Verify exact public wheel and source distribution bytes
+        run: python scripts/verify_public_release_absence.py --published dist

README.md

@@ -424,7 +424,7 @@ distinguish implemented source, executed examples, artifact verification and rel
 
 ```bash
 python -m pip install -e '.[test]'
-PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src python -m pytest -q
+PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src:packages/teaql-blob-reference/src python -m pytest -q
 PYTHONDONTWRITEBYTECODE=1 bash scripts/verify-examples.sh
 ```
 

packages/teaql-blob-reference/README.md

@@ -0,0 +1,99 @@
+# teaql-blob-reference
+
+Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import.
+
+`BlobReferenceModule.ready(context)` checks that the current native SQLite
+business executor and catalog refer to the same database before activating the
+content verifier. The executor's `uses_sql_database` SPI asks the transport for
+identity; the native SQLite transport compares SQLite's actual `main` filename
+from `PRAGMA database_list`, so ordinary paths and native file URIs can match.
+An empty filename cannot establish a durable database for its independent
+per-operation connections and is rejected. Missing/non-native owners, a different
+dialect or file, and drift between the installed services' catalogs fail with
+`BLOB_CAPABILITY_MISSING`. Separate native transports to the same file remain
+supported. Installation stays passive; explicit readiness performs metadata
+reads, schema validation and verifier readiness without migrating or uploading.
+
+`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open.
+
+For permissions backed by mutable business rows, the host may implement `current_on_lease(actor, lease, target, action, input)` and return `BlobUploadAuthorizationState(scope, field, allowed)`. Begin, Write, and Confirm then recheck that state on the same physical SQLite transaction as the upload journal; a missing row, revoked permission, malformed state, or scope/field mismatch denies the operation. The ordinary `scope`, `field`, and `authorize` callbacks remain the preflight path and the compatibility path for hosts without this optional hook.
+
+To use several optional parsers in one upload service, pass a
+`CompositeBlobContentVerifier` with explicit extension-to-parser routes. It
+copies and validates the route table at construction, calls each distinct
+parser's readiness check once, and rejects unregistered formats before reading
+stored bytes. It does not import optional decoder packages or add formats by
+itself; install only the image/PDF/Office/media extras the host selects.
+
+For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests.
+
+`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence and cross-machine S3 retention remain outside this service.
+
+`SqliteBlobFileGenerationSweepService` supplies a separate, host-scheduled
+catalog-driven File sweep. After every process sharing the private File root
+has upgraded to the writer-lock protocol, the host constructs it with
+`all_writers_upgraded=True` and calls `run_batch(actor, provider, cursor)`
+outside graph SQL. It pages only the authorized scope's `server_stream` rows,
+including `deleted` rows that may retain inaccessible old data generations.
+The provider must be the exact `FileObjectStorage` installed in the current
+context. Each outcome reports removed generations or a sanitized failure;
+restart the scan from the beginning to retry a failed row. Keep terminal
+catalog rows until the File sweep has completed. The host assertion cannot
+detect an older external writer on its own; no worker or timer starts at import.
+
+For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject
+`teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra
+pins Pillow 12.3.0, the newest release supporting Python 3.10 at this checkpoint.
+The adapter validates complete JPEG/PNG containers and PNG CRCs, checks the
+decoder's real format and loads all pixels with explicit input, pixel and
+concurrent-work bounds. It has no hard CPU/process isolation. Construction is
+passive; `BlobReferenceModule.ready` invokes its optional decoder readiness check.
+It accepts `jpg`, `jpeg` and `png` only. The other formats need separate
+production adapters; the earlier PDF fixture test does not establish them.
+
+For actual PDF parsing, install `teaql-blob-reference[pdf]` and inject
+`teaql_blob_reference.content_pypdf.PypdfBlobContentVerifier`. The PDF extra
+pins pypdf 6.21.3. This adapter checks a complete PDF container with strict
+cross-reference parsing, a real page tree and content operators, and bounded
+input, pages, decoded page content and concurrent work. It rejects encrypted
+PDFs. This is parser validation, not malware scanning or hard CPU/process
+isolation. The PDF fixture verifier used in older S3 tests remains test-only.
+
+For actual DOCX/XLSX parsing, install `teaql-blob-reference[office]` and inject
+`teaql_blob_reference.content_office.OfficeBlobContentVerifier`. The office extra
+pins python-docx 1.2.0 and openpyxl 3.1.5. The adapter checks every ZIP member
+and its CRC under input, decoded-byte and part-count limits, rejects XML entity
+declarations, checks the expected
+Office main content type and package relationship, then parses the Word body or
+every Excel worksheet under an element/cell count limit. It hashes original
+bytes and returns the precise Office MIME type. Construction is passive, and
+`validate_ready` requires both optional parsers. The in-process parsers do not
+provide hard CPU or process-memory isolation.
+
+For binary DOC/XLS on Linux, inject
+`teaql_blob_reference.content_legacy_office.LegacyOfficeBlobContentVerifier`
+with absolute paths to patched LibreOffice (`soffice`), `bwrap`, `prlimit` and
+an owner-only (`0700`) canonical scratch directory. Install the `office` extra
+for its converted OOXML verifier. The adapter checks original CFB header and
+sector layout, converts each file in a network/PID-isolated bubblewrap
+namespace with read-only system mounts and only one private writable scratch,
+then fully validates the DOCX/XLSX output. CPU, address-space, file-size,
+wall-time, input and concurrency limits apply. It returns original DOC/XLS
+SHA, size and MIME, so storage keeps the original bytes. This is bounded
+decodability verification, not malware or macro detection.
+
+For actual MP3/MP4 validation on POSIX hosts, inject
+`teaql_blob_reference.content_ffmpeg.FfmpegBlobContentVerifier` with absolute
+paths to trusted `ffprobe` and `ffmpeg` executables. The host installs and
+patches those binaries separately; this wheel does not bundle or fetch them.
+`validate_ready` checks availability when the module is activated. The adapter
+walks complete MP3 frames or top-level MP4 boxes before forcing the matching
+FFmpeg demuxer, probing decoded frame counts and stream properties, and fully
+decoding media from a pipe. Client filenames and URLs never reach FFmpeg. It
+allows only the `pipe` protocol and bounds input bytes, frame count, video
+pixels, concurrent checks, child CPU time and address space, and wall time.
+The accepted MP3 profile is MPEG Layer III with fixed, non-free bitrate and a
+constant sample rate; MP4 must contain a supported `ftyp` brand, `moov`,
+`mdat`, and at least one decodable non-cover-art video stream. The adapter returns the SHA-256
+of the original bytes and `audio/mpeg` or `video/mp4`. It is content validation,
+not malware scanning; ffprobe/ffmpeg themselves must be kept current.

packages/teaql-blob-reference/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "teaql-blob-reference"
+version = "0.3.1"
+description = "Optional TeaQL Blob reference module"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "TeaQL" }]
+dependencies = ["teaql==0.3.1"]
+
+[project.optional-dependencies]
+image = ["Pillow==12.3.0"]
+pdf = ["pypdf==6.21.3"]
+office = ["python-docx==1.2.0", "openpyxl==3.1.5"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["teaql_blob_reference*"]

packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py

@@ -0,0 +1,28 @@
+"""Optional Blob reference services; the base teaql runtime does not import them."""
+
+from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog
+from .read_service import SqliteBlobResponseContributor
+from .module import BlobReferenceModule
+from .maintenance_service import (BlobMaintenanceAuthorization, BlobMaintenanceBatch,
+                                  BlobMaintenanceCursor, BlobMaintenanceOutcome,
+                                  SqliteBlobMaintenanceService)
+from .mutation_participant import SqliteBlobMutationParticipant
+from .file_storage import FileObjectStorage
+from .file_generation_maintenance import (BlobFileGenerationSweepBatch,
+                                          BlobFileGenerationSweepOutcome,
+                                          SqliteBlobFileGenerationSweepService)
+from .retention import BlobRetentionPolicy
+from .content_composite import CompositeBlobContentVerifier
+from .upload_service import (BlobUploadAuthorizationState, BlobUploadInput,
+                             BlobUploadSession, BlobUploadTarget,
+                             BlobVerifiedContent, SqliteBlobUploadService)
+
+__all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead",
+           "BlobMaintenanceAuthorization", "BlobMaintenanceBatch", "BlobMaintenanceCursor",
+           "BlobMaintenanceOutcome", "SqliteBlobMaintenanceService",
+           "BlobFileGenerationSweepBatch", "BlobFileGenerationSweepOutcome",
+           "SqliteBlobFileGenerationSweepService",
+           "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage",
+           "BlobUploadAuthorizationState", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget",
+           "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy",
+           "CompositeBlobContentVerifier"]