From a36c5cf2eed6792984d0de81ab9a0ca4be12dcc2 Mon Sep 17 00:00:00 2001 From: DevFoundry-labs Date: Mon, 17 Aug 2026 02:25:52 +0400 Subject: [PATCH 1/3] Add V2 planning and discovery package --- .dev/V2_PLAN.md | 264 +++++++++++++++++++++++ .dev/v2/DECISIONS.md | 45 ++++ .dev/v2/DISCOVERY_GUIDE.md | 107 +++++++++ .dev/v2/DISCOVERY_TRACKER.md | 49 +++++ .dev/v2/EVIDENCE_V2_SPEC.md | 223 +++++++++++++++++++ .dev/v2/INTERVIEW_RECORD_TEMPLATE.md | 90 ++++++++ .dev/v2/ISSUE_BACKLOG.md | 212 ++++++++++++++++++ .dev/v2/MILESTONES.md | 145 +++++++++++++ .dev/v2/RECRUITMENT_SCRIPT.md | 42 ++++ .dev/v2/RETENTION_AND_OPERATIONS_SPEC.md | 132 ++++++++++++ .dev/v2/SCOPE_DECISION_TEMPLATE.md | 98 +++++++++ .dev/v2/WEBHOOK_SPEC.md | 215 ++++++++++++++++++ 12 files changed, 1622 insertions(+) create mode 100644 .dev/V2_PLAN.md create mode 100644 .dev/v2/DECISIONS.md create mode 100644 .dev/v2/DISCOVERY_GUIDE.md create mode 100644 .dev/v2/DISCOVERY_TRACKER.md create mode 100644 .dev/v2/EVIDENCE_V2_SPEC.md create mode 100644 .dev/v2/INTERVIEW_RECORD_TEMPLATE.md create mode 100644 .dev/v2/ISSUE_BACKLOG.md create mode 100644 .dev/v2/MILESTONES.md create mode 100644 .dev/v2/RECRUITMENT_SCRIPT.md create mode 100644 .dev/v2/RETENTION_AND_OPERATIONS_SPEC.md create mode 100644 .dev/v2/SCOPE_DECISION_TEMPLATE.md create mode 100644 .dev/v2/WEBHOOK_SPEC.md diff --git a/.dev/V2_PLAN.md b/.dev/V2_PLAN.md new file mode 100644 index 0000000..5cbdae9 --- /dev/null +++ b/.dev/V2_PLAN.md @@ -0,0 +1,264 @@ +# OffboardProof V2 Plan + +Status: DRAFT CONTRACTS READY FOR PRODUCT REVIEW + +Created: 2026-08-17 + +Assumption: "V2" means the next meaningful release after v0.1.0, provisionally v0.2.0—not a multi-year 2.0 rewrite. + +## 1. Product thesis + +V2 should make OffboardProof credible for a real single-organization pilot. It should not chase a broad connector catalog, multi-tenancy, or a dashboard before the core workflow is used with real offboarding cases. + +The release thesis is: + +> Accept an authenticated departure from an external system, produce cryptographically verifiable evidence with an explicit lifecycle, and measure whether operators complete offboarding faster without weakening approval or verification guarantees. + +## 2. Target user outcome + +An IT/security operator can connect an HR or ITSM system to OffboardProof, process a real departure through the existing approval and verification workflow, give an auditor a portable evidence bundle plus public verification material, and report completion/exception metrics without sending employee data to a hosted analytics service. + +## 3. North-star and guardrail metrics + +### North-star + +- Percentage of pilot cases whose required controls reach verified, acknowledged, or validly waived state by the policy deadline and whose evidence verifies offline. + +### Supporting metrics + +- Median operator active minutes per case. +- Median time from authorized trigger to containment. +- Percentage of controls independently observed versus acknowledged, human-attested, or waived. +- Exceptions per case, retry recovery rate, and median exception age. +- Duplicate triggers suppressed and duplicate external writes prevented. +- Evidence verification success rate after export and transfer. + +### Guardrails + +- Zero execution without current digest-bound approvals. +- Zero webhook replay resulting in a second case or action. +- Zero evidence bundles labeled independently verified when only acknowledged. +- Zero private signing keys, webhook secrets, or bearer tokens stored in evidence or logs. +- No automatic deletion in V2; retention expiry is reported and gated for explicit follow-up. + +The current 14-touch baseline and 60% active-time reduction target remain hypotheses until measured with design partners. + +## 4. Proposed committed scope + +### Epic A — Design-partner validation and measurement + +Deliverables: + +- Interview guide and workflow inventory for 5–8 operators, approvers, or auditors across at least 3 organizations. +- Sanitized baseline worksheet covering trigger source, manual touches, elapsed time, active time, exceptions, proof requested, and systems involved. +- Pilot runbook using synthetic identities first and real identities only under the partner's data-handling policy. +- Local-only metric export for the north-star and supporting metrics. +- Decision record confirming or revising the V2 scope after discovery. + +Exit criteria: + +- At least 3 participants confirm the verification/evidence problem is recurring and material. +- At least 2 organizations can name a viable signed trigger source and will evaluate the evidence bundle. +- At least 1 pilot can run Google Workspace or the mock provider end to end. + +If these criteria are not met, stop connector expansion and revisit the target user/problem. + +### Epic B — Authenticated webhook intake + +Capabilities: + +- `POST /v1/intake/webhooks/{source}` accepts a versioned canonical event contract. +- Verify an HMAC-SHA256 signature over the raw body plus timestamp before parsing. +- Enforce timestamp skew, body-size limit, content type, source/key identifier, and a unique delivery identifier. +- Store only a secret digest/identifier or load secrets from environment/file-backed configuration; never persist plaintext webhook secrets. +- Support active and previous keys during a bounded rotation window. +- Map a valid event into the existing `CaseCreate` path so case and action idempotency invariants remain unchanged. +- Return the same case on a repeated valid delivery; reject stale, malformed, or signature-invalid deliveries without creating state. +- Add structured, non-sensitive audit events for accepted, replayed, and rejected deliveries. + +Acceptance tests: + +- Valid signature creates one case. +- Same delivery and same semantic case replay safely. +- Same delivery ID with a different body is rejected as a conflict. +- Invalid signature, stale timestamp, oversized body, wrong content type, and unknown key ID create no case. +- Rotation accepts the previous key only within its configured window. +- Logs and API errors do not disclose the secret, signature, or sensitive body. + +### Epic C — Evidence schema v2 and signatures + +Capabilities: + +- Version the evidence envelope independently from the application release. +- Keep deterministic canonical JSON as the signed payload. +- Generate an Ed25519 signature over the final evidence digest. +- Include algorithm, key ID, public-key fingerprint, evidence digest, completion cutoff, and schema version in a detached manifest. +- Add `offboardproof evidence-verify` and a read-only API verification endpoint. +- Export the public key in a portable format; load the private key from an operator-controlled path or signing-provider adapter. +- Fail closed when the configured signing key is unreadable or has unsafe permissions where the operating system exposes that check. +- Preserve an explicit unsigned mode for local evaluation, labeled `unsigned` rather than silently appearing sealed. +- Define signing-key rotation without invalidating historical bundles. + +Acceptance tests: + +- A transferred evidence bundle verifies without database access. +- Any changed evidence byte, manifest field, signature, or public key fails verification. +- Historical evidence verifies after signing-key rotation. +- Re-export at the same completion cutoff produces the same canonical evidence digest. +- The private key never appears in SQLite, logs, API responses, evidence, fixtures, or source control. + +Reference: https://cryptography.io/en/stable/hazmat/primitives/asymmetric/serialization/ + +### Epic D — Evidence lifecycle without unsafe deletion + +Capabilities: + +- Add organization-level default retention days and case-level `retention_until`. +- Add security-authorized legal hold with reason, actor, timestamp, and release event. +- Expose expiring/expired evidence in API, CLI, metrics, and audit events. +- Add a deterministic retention dry-run report listing records and files that would be affected. +- Document backup, restore, legal hold, and filesystem/database coordination. + +Explicit V2 limit: + +- V2 does not physically purge evidence or promise secure erasure from SQLite, WAL files, backups, or storage media. A later release may add a separately approved purge workflow after recovery and legal requirements are validated. + +Acceptance tests: + +- Retention date is deterministic and visible in the signed evidence metadata. +- Legal hold prevents an item from appearing as purge-eligible. +- Only security/admin roles can create or release a hold. +- Retention configuration changes are audit logged and do not rewrite historical completion evidence. + +### Epic E — Pilot hardening + +Capabilities: + +- Correlation/request IDs propagated through API, worker, provider, and audit events. +- Backup/restore commands and a tested restore drill for database plus evidence files. +- Documented upgrade from schema v1 to v2 with rollback boundaries. +- Operational endpoints for queue depth, oldest queued job, open/overdue exceptions, and evidence awaiting signature. +- Load and fault tests for 100 concurrent intake deliveries, 10,000 audit events, lease expiry, crash-before-receipt, crash-after-receipt, and database restart. +- Redaction tests for every structured log/error field. + +## 5. Conditional scope + +### Microsoft Entra ID adapter + +Implement only if at least two credible pilot organizations require Entra ID or one committed pilot cannot proceed without it. + +A bounded adapter appears technically feasible using Microsoft Graph v1.0 for direct membership discovery, group-member removal, session revocation, and user updates. The implementation must preserve the existing observed-versus-acknowledged distinction and must append `/$ref` when removing a group member; omitting it can delete the directory object when broader permissions are present. + +Required discovery before commitment: + +- Exact least-privilege application permissions and admin roles. +- Behavior for dynamic, role-assignable, nested, and hidden-membership groups. +- Observable postcondition and propagation delay for `accountEnabled=false` and `revokeSignInSessions`. +- National-cloud base URLs and tenant configuration boundaries. +- Contract fixtures for throttling, eventual consistency, partial permissions, and external users. + +References: + +- https://learn.microsoft.com/en-us/graph/api/user-revokesigninsessions?view=graph-rest-1.0 +- https://learn.microsoft.com/en-us/graph/api/user-list-memberof?view=graph-rest-1.0 +- https://learn.microsoft.com/en-us/graph/api/group-delete-members?view=graph-rest-1.0 + +### Google audit correlation spike + +Time-box a spike to determine whether Reports API events can add useful corroboration after session sign-out. Do not promote a sign-out control from `acknowledged` to `verified` unless an event can be causally and reliably tied to the administrative action. Google's login report includes `logout`, but that alone does not prove that the Admin SDK `users.signOut` operation invalidated every session. + +References: + +- https://developers.google.com/workspace/admin/reports/v1/guides/manage-audit-login +- https://developers.google.com/workspace/admin/reports/v1/appendix/activity/login + +## 6. Explicit non-goals + +- Browser dashboard. +- Multi-tenant SaaS, organization isolation, billing, or hosted control plane. +- PostgreSQL/multi-worker migration. +- OIDC/SSO replacing local actor tokens. +- Automatic evidence deletion or claimed secure erasure. +- Account deletion, device wipe, mailbox/Drive transfer, or joiner/mover workflows. +- Broad HRIS/ITSM connector catalog. +- AI-generated policy or remediation decisions. +- Claimed compliance certification. + +These remain candidates for later releases. They should not displace pilot learning or core assurance work. + +## 7. Delivery sequence + +### Phase 0 — Validate (decision gate) + +1. Conduct interviews and capture baselines. +2. Select pilot trigger source and evidence consumer. +3. Confirm data handling, retention, and signing-key ownership. +4. Decide whether Entra is committed, conditional, or deferred. + +### Phase 1 — Freeze contracts + +1. Write webhook event schema and threat model. +2. Write evidence schema v2 and signature manifest specification. +3. Write retention/hold state model and migration plan. +4. Add acceptance tests before production code. + +### Phase 2 — Build vertical slices + +1. Signed webhook to idempotent case creation. +2. Completed case to signed portable evidence and offline verification. +3. Retention metadata to dry-run/hold operations. +4. Metrics and operator visibility across the complete path. + +### Phase 3 — Pilot hardening + +1. Fault, replay, migration, backup/restore, and redaction tests. +2. Synthetic tenant rehearsal with external writes disabled, then enabled under a reviewed plan. +3. Pilot run, measurement, and issue review. +4. Resolve release blockers and publish v0.2.0. + +## 8. Release gates + +- All v0.1.0 safety and quality gates remain mandatory. +- At least 90% branch-aware coverage for new security-critical modules. +- Independent signature verification from the built wheel in a clean environment. +- Webhook replay/fuzz/property tests and timing-safe signature comparison. +- Migration test from a real v0.1.0 fixture plus clean install. +- Backup/restore drill reproduces audit-chain and evidence verification. +- Threat-model review has no unresolved critical/high finding. +- No externally mutable action is enabled by default. +- Pilot report distinguishes synthetic measurements from observed partner results. + +## 9. Proposed issue breakdown + +1. V2 discovery: interview guide, baseline worksheet, and scope decision record. +2. Specify canonical webhook event and signature protocol. +3. Implement webhook source/key configuration and rotation. +4. Implement replay-safe webhook endpoint and rejection audit. +5. Specify evidence schema v2 and detached manifest. +6. Implement Ed25519 signing and public-key export. +7. Implement offline evidence verification CLI/API. +8. Add retention metadata and migration. +9. Add legal hold and retention dry-run. +10. Add queue/evidence/exception operational metrics. +11. Add backup/restore command and drill. +12. Run Google audit-correlation spike. +13. Decide and, if gated in, specify Entra provider contract. +14. Execute synthetic rehearsal and design-partner pilot. +15. Publish v0.2.0 with measured results and explicit assurance limits. + +No GitHub issues should be opened until the product thesis, pilot source, and committed/conditional scope are approved. + +## 10. Working assumptions accepted for continued planning + +1. The next release is provisionally v0.2.0 rather than a 2.0 reset. +2. Generic HMAC webhook intake anchors the trigger contract. +3. The deployment operator owns and rotates evidence-signing keys. +4. Entra remains conditional on the pilot-demand gate. + +These are recorded in `.dev/v2/DECISIONS.md` and remain reversible before implementation. + +## 11. Remaining decisions + +1. Is a Google Workspace pilot available, and is Entra required by a committed pilot? +2. What retention and legal-hold obligations apply to the first pilot's evidence? +3. What minimum pilot evidence would justify work on a UI, SSO, PostgreSQL, or additional connectors? diff --git a/.dev/v2/DECISIONS.md b/.dev/v2/DECISIONS.md new file mode 100644 index 0000000..967a484 --- /dev/null +++ b/.dev/v2/DECISIONS.md @@ -0,0 +1,45 @@ +# V2 Working Decisions + +These are planning assumptions adopted on 2026-08-17 so contract work can continue. They are reversible before GitHub issues or production code are created. + +## V2-001: Treat V2 as v0.2.0 + +Decision: evolve v0.1.0 without a rewrite or stability reset. + +Reason: the current workflow invariants, API, SQLite model, and provider boundary are useful foundations. No evidence justifies a 2.0 reset. + +## V2-002: Anchor intake on a generic HMAC protocol + +Decision: specify one vendor-neutral signed event and build vendor adapters outside or in front of the core endpoint. + +Reason: it validates the trust/replay boundary once and avoids committing to an HRIS before pilot demand is known. + +## V2-003: Operator owns signing keys + +Decision: V2 loads an operator-controlled private key from outside the repository/database and exports portable public verification material. + +Reason: a local-first product should not introduce a hosted signing dependency. A signing-provider interface may later support HSM/KMS systems. + +## V2-004: Entra is conditional + +Decision: commit Entra only when the discovery gate shows at least two credible pilots need it or one committed pilot is blocked without it. + +Reason: provider breadth is expensive and can distract from validating the evidence workflow. + +## V2-005: No physical purge + +Decision: implement retention truth, legal holds, and dry-run visibility, but no automated deletion. + +Reason: SQLite pages, WAL, backups, snapshots, and storage media make secure-erasure claims unsafe without a separate architecture and recovery policy. + +## V2-006: Trusted origin requires an external trust anchor + +Decision: offline verification requires a supplied trusted public key or pinned fingerprint by default. + +Reason: accepting only a public key shipped beside a signature proves integrity against accidental change but does not establish who signed it. + +## V2-007: Planning changes remain local + +Decision: do not open milestones/issues, commit, push, or modify the release until the user approves the planning package. + +Reason: issue topology and public commitments should follow scope review, not precede it. diff --git a/.dev/v2/DISCOVERY_GUIDE.md b/.dev/v2/DISCOVERY_GUIDE.md new file mode 100644 index 0000000..6765874 --- /dev/null +++ b/.dev/v2/DISCOVERY_GUIDE.md @@ -0,0 +1,107 @@ +# V2 Design-Partner Discovery Guide + +Status: READY FOR INTERVIEWS + +Execution files: + +- `RECRUITMENT_SCRIPT.md` provides the invitation, eligibility screen, and consent opening. +- `DISCOVERY_TRACKER.md` tracks recruitment, coverage, and the M0 gate without names or contact details. +- `INTERVIEW_RECORD_TEMPLATE.md` is copied once per completed interview and stores only sanitized findings. +- `SCOPE_DECISION_TEMPLATE.md` is completed after the discovery set; it is the decision artifact for V2-01. + +Do not count an interview toward the gate until its sanitized record is complete. Keep scheduling details and participant contact information outside this repository. + +## Goal + +Determine whether organizations will use and trust a verification-first offboarding workflow, which trigger/provider must anchor the pilot, and which evidence/retention requirements are real rather than assumed. + +This is problem and workflow discovery, not a product pitch. Do not ask participants to share employee names, live credentials, confidential screenshots, or tenant configuration. + +## Participant mix + +Target 5–8 conversations across at least 3 organizations: + +- 2–3 IT operators or MSP technicians who execute departures. +- 1–2 HR or line-manager approvers. +- 1–2 security, compliance, or audit evidence consumers. +- At least one organization with Google Workspace; record Entra demand without forcing it. + +## Interview questions + +1. Walk through the last completed departure from the first authorized signal to the point you considered it done. +2. Which system originated the request, and how did you know it was authorized? +3. Which steps were automated, manual, checked later, or assumed successful? +4. What access or ownership transfer is most often forgotten? +5. What exceptions create the most delay or risk? +6. Who can approve timing, ownership transfer, high-risk handling, and exceptions? +7. What proves that suspension, session invalidation, and group removal actually happened? +8. When an API times out, how do you decide whether to retry? +9. What evidence has an auditor, insurer, customer, or incident reviewer requested? +10. How is that evidence retained, held, exported, or deleted today? +11. Which identities/providers are mandatory for a pilot: Google Workspace, Entra ID, another IdP, or manual systems? +12. Could the trigger system send a signed webhook? If not, what export or middleware is realistic? +13. Who would own webhook secrets and evidence-signing keys? +14. What data is unacceptable in logs, metrics, evidence, or support channels? +15. What result after 5–10 cases would make you continue using this? + +## Baseline worksheet + +Capture ranges when exact measurements are unavailable: + +| Field | Value | +|---|---| +| Organization alias | | +| Participant role | | +| Approximate employee count | | +| Departures/role changes per month | | +| Trigger source | | +| Primary identity provider | | +| Number of systems/controls touched | | +| Human handoffs | | +| Operator active minutes | | +| End-to-end elapsed time | | +| Time to primary containment | | +| Rework/duplicate actions | | +| Exceptions and typical age | | +| Approval roles | | +| Evidence requested/produced | | +| Retention/legal-hold requirement | | +| Pilot blocker | | +| Success threshold | | + +## Pilot acceptance + +A partner is pilot-ready when: + +- A named operator and evidence consumer will evaluate results. +- The trigger source and provider path are known. +- Synthetic rehearsal is allowed before real identities. +- Approval roles and success states are documented. +- Credential/key ownership and data handling are agreed. +- The partner accepts the product's observed/acknowledged distinction. +- Baseline and outcome metrics can be captured without sending PII externally. + +## Scope decision rubric + +Score each candidate capability 0–3 for frequency, pain/risk, pilot necessity, observable postcondition, setup feasibility, and reuse across partners. + +- 14–18: commit to V2. +- 9–13: conditional spike or follow-up release. +- 0–8: defer. + +Security invariants and release gates are mandatory regardless of score. + +## Interview synthesis + +After each interview, record: + +- confirmed facts versus participant estimates +- repeated workflow pattern +- evidence gap +- requested capability +- observable success condition +- integration/setup constraint +- disconfirming evidence +- scope score + +After the discovery set, publish a short decision record: keep/revise/stop thesis, committed pilot path, conditional items, explicit deferrals, baseline ranges, and measurable release success criteria. diff --git a/.dev/v2/DISCOVERY_TRACKER.md b/.dev/v2/DISCOVERY_TRACKER.md new file mode 100644 index 0000000..fdb30ec --- /dev/null +++ b/.dev/v2/DISCOVERY_TRACKER.md @@ -0,0 +1,49 @@ +# V2 Discovery Tracker + +Status: RECRUITING — 0 OF 5–8 INTERVIEWS COMPLETE + +This tracker contains aliases and aggregate findings only. Do not record participant names, employer names, email addresses, employee identities, tenant identifiers, credentials, or confidential configuration. + +## Coverage target + +| Requirement | Target | Current | Gate state | +|---|---:|---:|---| +| Completed conversations | 5–8 | 0 | Not met | +| Distinct organizations | At least 3 | 0 | Not met | +| IT operators/MSP technicians | 2–3 | 0 | Not met | +| HR/line-manager approvers | 1–2 | 0 | Not met | +| Security/compliance/evidence consumers | 1–2 | 0 | Not met | +| Organizations identifying a viable signed trigger and willing evidence consumer | At least 2 | 0 | Not met | +| Google Workspace or mock-provider pilot path | At least 1 | 0 | Not met | + +## Participant pipeline + +Use organization aliases `O1`, `O2`, and so on, and participant aliases such as `O1-P1`. Keep recruitment contact information outside the repository. + +| Participant alias | Organization alias | Role category | Provider | Trigger candidate | Stage | Record | +|---|---|---|---|---|---|---| +| TBD | TBD | Operator / approver / evidence consumer | Unknown | Unknown | Candidate | — | + +Allowed stages: `Candidate`, `Invited`, `Scheduled`, `Completed`, `Declined`, `Excluded`. + +## Evidence ledger + +Add one row after each completed and sanitized interview. + +| Record | Facts measured | Participant estimates | Recurring/material problem? | Viable signed trigger? | Will evaluate evidence? | Pilot path | Entra required? | +|---|---|---|---|---|---|---|---| +| — | — | — | — | — | — | — | — | + +## M0 exit gate + +- [ ] At least three participants confirm the evidence/verification problem is recurring and material. +- [ ] At least two organizations identify a viable signed trigger source and will evaluate the evidence bundle. +- [ ] At least one pilot can run Google Workspace or the mock provider end to end. +- [ ] Baseline statements are classified as measured, participant-estimated, or unknown. +- [ ] Pilot operator, approval roles, and evidence consumer are identified by aliases. +- [ ] Webhook-secret and signing-key ownership are agreed. +- [ ] Retention, legal-hold, and prohibited-data boundaries are documented. +- [ ] Entra is explicitly committed, conditional, or deferred. +- [ ] `SCOPE_DECISION_TEMPLATE.md` has been completed as a keep/revise/stop record. + +Gate state: **NOT PASSED**. Implementation issues V2-05 onward must remain unopened. diff --git a/.dev/v2/EVIDENCE_V2_SPEC.md b/.dev/v2/EVIDENCE_V2_SPEC.md new file mode 100644 index 0000000..135aaf4 --- /dev/null +++ b/.dev/v2/EVIDENCE_V2_SPEC.md @@ -0,0 +1,223 @@ +# Evidence Schema V2 and Signature Contract + +Status: DRAFT + +## Purpose + +Make a completed-case export independently portable and cryptographically verifiable without overstating who signed it or what provider state was observable. + +V2 separates three properties: + +1. **Integrity:** the evidence bytes have not changed. +2. **Origin:** the signing key matches a public key the verifier already trusts. +3. **Assurance:** each control remains labeled observed, acknowledged, human-attested, or waived. + +A bundled public key alone proves self-consistency, not trusted origin. The default verifier therefore requires an operator-supplied trusted public key or pinned fingerprint. + +## Export set + +For case `{case_id}`: + +```text +{case_id}.evidence.v2.json +{case_id}.evidence.v2.manifest.json +``` + +The evidence file is canonical UTF-8 JSON with no BOM and exactly one trailing LF. It does not contain its own digest or signature. + +The manifest is deterministic canonical JSON with exactly one trailing LF. It contains the signing statement, signature, and optional transport copy of the public key. + +## Evidence envelope + +Top-level keys: + +```json +{ + "schema_version": 2, + "application": { + "name": "offboardproof", + "version": "0.2.0" + }, + "case": {}, + "controls": [], + "approvals": [], + "actions": [], + "exceptions": [], + "observations": [], + "audit_events": [], + "completion": { + "sequence_no": 42, + "event_id": "uuid", + "event_hash": "sha256-hex", + "completed_at": "RFC3339 UTC" + }, + "retention": { + "retention_until": "RFC3339 UTC or null", + "policy_snapshot": "default-365-days or explicit identifier" + }, + "assurance": { + "observed_controls": 3, + "acknowledged_controls": 1, + "human_attested_controls": 1, + "waived_controls": 0, + "limit": "Evidence records configured provider observations and authorized human statements; it is not an independent certification." + } +} +``` + +Rules: + +- Only events at or before the first `case.completed` sequence are included. +- Rows use explicit allowlists; internal-only or future columns never appear automatically. +- JSON-valued database columns are embedded as JSON values, not serialized JSON strings. +- Collections have stable order based on domain sequence then stable identifiers. +- Timestamps are normalized to UTC with a `Z` suffix and fixed microsecond policy. +- Approval and actor identifiers remain because evidence must establish authorization; bearer token metadata never appears. +- Post-completion lifecycle events do not rewrite the sealed evidence. They belong to later lifecycle reports. +- Evidence schema v1 remains readable and verifiable by its legacy digest behavior. + +## Signing statement + +The signer computes the exact evidence file bytes and constructs: + +```json +{ + "domain": "offboardproof.evidence.signature.v1", + "case_id": "uuid", + "evidence_schema_version": 2, + "evidence_sha256": "64 lowercase hex characters", + "completion_event_hash": "64 lowercase hex characters", + "key_id": "operator-defined-stable-id" +} +``` + +The Ed25519 signature is calculated over the canonical UTF-8 bytes of this statement with no trailing LF. The domain string prevents the key/signature from being confused with another protocol. + +## Manifest envelope + +```json +{ + "manifest_schema_version": 1, + "statement": { + "domain": "offboardproof.evidence.signature.v1", + "case_id": "uuid", + "evidence_schema_version": 2, + "evidence_sha256": "hex", + "completion_event_hash": "hex", + "key_id": "pilot-2026-q3" + }, + "signature": { + "algorithm": "Ed25519", + "encoding": "base64url-no-padding", + "value": "..." + }, + "public_key": { + "encoding": "raw-base64url-no-padding", + "fingerprint_sha256": "hex", + "value": "optional transport copy" + }, + "signing_status": "signed" +} +``` + +Unsigned evaluation exports use `signing_status: "unsigned"`, omit `signature`, and may contain a safe reason code. They are never described as sealed or origin-verified. + +## Key configuration + +Settings: + +- `OFFBOARDPROOF_SIGNING_KEY_FILE`: absolute path to an encrypted PKCS8 or explicitly supported raw private key file. +- `OFFBOARDPROOF_SIGNING_KEY_PASSWORD_FILE`: optional absolute path, never an inline environment password. +- `OFFBOARDPROOF_SIGNING_KEY_ID`: required stable identifier when signing is enabled. +- `OFFBOARDPROOF_REQUIRE_SIGNED_EVIDENCE`: defaults false for evaluation; pilot configuration sets true. + +Rules: + +- Private key files remain outside the database, repository, evidence directory, and ordinary evidence backup. +- Public keys are exported separately and distributed through an authenticated channel. +- Key ID is not a trust anchor. +- When signed evidence is required, key load/sign failure prevents case evidence completion and opens an operational exception; it never silently emits unsigned evidence. +- When signing is optional, an unsigned artifact and audit event state that fact explicitly. +- Historical evidence keeps its original key ID. Rotation changes only new signatures. + +## Verification interface + +CLI: + +```text +offboardproof evidence-verify \ + --evidence CASE.evidence.v2.json \ + --manifest CASE.evidence.v2.manifest.json \ + --trusted-public-key pilot-2026-q3.pub +``` + +Machine-readable output: + +```json +{ + "valid": true, + "integrity": "verified", + "origin": "trusted_key_match", + "assurance": "mixed_observed_and_acknowledged", + "case_id": "uuid", + "key_id": "pilot-2026-q3", + "evidence_sha256": "hex" +} +``` + +The CLI exits: + +- `0`: valid signature and trusted-key match. +- `2`: malformed inputs or unsupported schema/algorithm. +- `3`: digest mismatch. +- `4`: signature failure. +- `5`: supplied trust anchor does not match manifest fingerprint. +- `6`: unsigned evidence. + +API: + +- `POST /v1/evidence/verify` is read-only and accepts bounded multipart files only if upload handling is deliberately added. +- Preferred first slice is CLI/library verification to avoid expanding the HTTP attack surface. +- A later API endpoint should return verification facts, never persist uploaded third-party evidence by default. + +## Persistence changes + +Migration `0003_evidence_v2.sql` extends `evidence_artifacts` with: + +```text +- schema_version integer not null default 1 +- manifest_path text nullable +- evidence_sha256 text nullable +- signing_status text not null default 'legacy' +- signing_algorithm text nullable +- signing_key_id text nullable +- public_key_fingerprint text nullable +- completion_event_hash text nullable +``` + +Existing `sha256` remains the exact artifact-file digest for backward compatibility. V2 sets it equal to `evidence_sha256` because the evidence file is the signed byte sequence. + +## Required tests + +- Golden v2 canonical serialization across Windows and Linux. +- Byte change, whitespace change, reordered key, changed manifest statement, changed signature, and changed trusted key all fail as specified. +- Embedded public-key substitution fails when a trusted key is supplied. +- Verification refuses implicit trust of only an embedded key. +- Historical key verifies after active-key rotation. +- Encrypted key load success/failure and absent password behavior. +- Private material absent from database, audit events, logs, exception messages, evidence, built distributions, and test snapshots. +- v1 evidence remains readable and reports `legacy_digest_only` rather than signed origin. +- Concurrent export converges on one artifact record and identical bytes. +- Signing-required failure cannot mark evidence ready. + +## Implementation boundary + +Suggested modules: + +- `offboardproof/evidence/schema_v2.py` +- `offboardproof/evidence/canonical.py` +- `offboardproof/evidence/signing.py` +- `offboardproof/evidence/verification.py` +- `offboardproof/evidence/export.py` + +The existing single `evidence.py` module should be split only as part of this vertical slice, retaining a compatibility import for v1 callers. diff --git a/.dev/v2/INTERVIEW_RECORD_TEMPLATE.md b/.dev/v2/INTERVIEW_RECORD_TEMPLATE.md new file mode 100644 index 0000000..b1c6a07 --- /dev/null +++ b/.dev/v2/INTERVIEW_RECORD_TEMPLATE.md @@ -0,0 +1,90 @@ +# V2 Sanitized Interview Record + +File naming: copy this file to `interviews/O-P.md`. Create the directory only when the first interview is complete. + +Do not include names, employer names, employee identities, tenant IDs, credentials, live URLs, confidential screenshots, or verbatim text that could identify the participant. Use ranges where exact values could be identifying. + +## Record metadata + +| Field | Value | +|---|---| +| Participant alias | | +| Organization alias | | +| Role category | Operator / approver / evidence consumer | +| Interview date | YYYY-MM-DD | +| Evidence basis | Direct observation / measured record / participant estimate / unknown | +| Sanitization reviewed by | | + +## Baseline + +| Field | Value | Classification | +|---|---|---| +| Approximate employee-count range | | Estimate / unknown | +| Departures or role changes per month | | Measured / estimate / unknown | +| Trigger source | | Fact / estimate / unknown | +| Primary identity provider | | Fact / estimate / unknown | +| Systems or controls touched | | Measured / estimate / unknown | +| Human handoffs | | Measured / estimate / unknown | +| Operator active minutes | | Measured / estimate / unknown | +| End-to-end elapsed time | | Measured / estimate / unknown | +| Time to primary containment | | Measured / estimate / unknown | +| Rework or duplicate actions | | Measured / estimate / unknown | +| Exceptions and typical age | | Measured / estimate / unknown | + +## Workflow and authority + +- Authorized starting signal: +- How authorization is established: +- Required approval roles: +- Most frequently missed access or ownership transfer: +- Highest-risk or slowest exception: +- Current retry and duplicate-action handling: + +## Evidence and lifecycle + +- Evidence currently requested: +- Evidence producer and consumer: +- What counts as proof of success: +- Retention requirement: +- Legal-hold requirement: +- Data prohibited from logs, metrics, evidence, or support: + +## Pilot feasibility + +- Trigger can send a signed webhook: Yes / No / Unknown +- Realistic alternative if not: +- Webhook-secret owner: +- Evidence-signing-key owner: +- Provider path: Google Workspace / Entra / mock / other / unknown +- Synthetic rehearsal permitted: Yes / No / Unknown +- Named operator alias: +- Evidence-consumer alias: +- Success threshold after 5–10 cases: +- Pilot blocker: + +## Synthesis + +- Confirmed facts: +- Participant estimates: +- Unknowns requiring follow-up: +- Repeated workflow pattern: +- Evidence gap: +- Requested capability: +- Observable success condition: +- Integration or setup constraint: +- Disconfirming evidence: + +## Capability score + +Score one candidate per table from 0–3 on each dimension. + +| Candidate capability | Frequency | Pain/risk | Pilot necessity | Observable postcondition | Setup feasibility | Reuse | Total | Decision | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| | | | | | | | | Commit 14–18 / Conditional 9–13 / Defer 0–8 | + +## Sanitization check + +- [ ] No names, contact details, employer names, tenant identifiers, or employee identities. +- [ ] No credentials, tokens, secrets, private keys, live URLs, or screenshots. +- [ ] Verbatim quotations are omitted or safely paraphrased. +- [ ] Every quantitative claim is labeled measured, participant-estimated, or unknown. diff --git a/.dev/v2/ISSUE_BACKLOG.md b/.dev/v2/ISSUE_BACKLOG.md new file mode 100644 index 0000000..7429369 --- /dev/null +++ b/.dev/v2/ISSUE_BACKLOG.md @@ -0,0 +1,212 @@ +# V2 Issue Backlog Draft + +Status: FIRST APPROVED BATCH PUBLISHED — V2-01 THROUGH V2-04 + +Published issues: + +- V2-01: https://github.com/DevFoundry-labs/offboardproof/issues/8 +- V2-02: https://github.com/DevFoundry-labs/offboardproof/issues/9 +- V2-03: https://github.com/DevFoundry-labs/offboardproof/issues/10 +- V2-04: https://github.com/DevFoundry-labs/offboardproof/issues/11 + +V2-05 onward remain local backlog items until the M0 gate passes. + +Suggested labels: `v2`, `discovery`, `security`, `api`, `evidence`, `operations`, `provider`, `documentation`, `release`, plus `size:S/M/L`. + +## M0 — Pilot validation gate + +### V2-01: Run design-partner discovery and publish scope decision + +Labels: `v2`, `discovery`, `size:M` + +Deliverables: + +- Complete the interview/baseline guide with 5–8 participants across at least 3 organizations. +- Score requested capabilities using the documented rubric. +- Identify a pilot trigger/provider/evidence consumer. +- Publish a PII-free keep/revise/stop decision. + +Acceptance: all M0 exit criteria are answered with evidence or the build is explicitly stopped. + +Dependencies: none. + +## M1 — Contracts and migrations + +### V2-02: Freeze webhook event and HMAC protocol + +Labels: `v2`, `security`, `api`, `size:M` + +Acceptance: + +- JSON Schema and golden signing vectors exist. +- Header parsing, skew, size, rotation, replay, conflict, and error contracts are unambiguous. +- Threat cases map to tests. + +Dependencies: V2-01. + +### V2-03: Freeze evidence v2 and detached signature manifest + +Labels: `v2`, `security`, `evidence`, `size:M` + +Acceptance: + +- Allowlists, canonical bytes, signing statement, trust-anchor behavior, exit codes, and v1 compatibility are specified. +- Cross-platform golden evidence/signature vectors exist. + +Dependencies: V2-01. + +### V2-04: Add forward-only v0.1→v0.2 migrations and fixtures + +Labels: `v2`, `operations`, `size:M` + +Acceptance: + +- `0002_webhook_intake`, `0003_evidence_v2`, and `0004_retention_operations` migrate a real v0.1 fixture. +- Clean install and repeated migration are tested. +- Released migration 0001 is unchanged. + +Dependencies: V2-02, V2-03. + +## M2 — Secure intake + +### V2-05: Load and validate webhook sources with rotating file-backed secrets + +Labels: `v2`, `security`, `api`, `size:M` + +Acceptance: config validation, service actor mapping, current/previous keys, expiry, readiness, secret length, absolute paths, and redaction tests pass. + +Dependencies: V2-02. + +### V2-06: Implement raw-body HMAC authentication + +Labels: `v2`, `security`, `api`, `size:M` + +Acceptance: official golden vectors, constant-time comparison, duplicate-header rejection, timestamp/skew checks, and body limit tests pass before JSON parsing. + +Dependencies: V2-05. + +### V2-07: Implement replay-safe webhook delivery service + +Labels: `v2`, `api`, `security`, `size:L` + +Acceptance: first delivery, exact replay, event replay, conflicting reuse, concurrent delivery, database failures, and existing case idempotency tests pass. + +Dependencies: V2-04, V2-06. + +### V2-08: Add webhook endpoint, safe audit events, and synthetic sender + +Labels: `v2`, `api`, `documentation`, `size:M` + +Acceptance: endpoint contract/OpenAPI, error behavior, safe audit payloads, example sender, and end-to-end test pass. + +Dependencies: V2-07. + +## M3 — Signed evidence + +### V2-09: Implement evidence schema v2 canonical exporter + +Labels: `v2`, `evidence`, `size:L` + +Acceptance: allowlisted deterministic bytes match golden vectors on Windows/Linux; completion cutoff and assurance counts are correct; v1 behavior is preserved. + +Dependencies: V2-03, V2-04. + +### V2-10: Implement Ed25519 key loading and signing + +Labels: `v2`, `security`, `evidence`, `size:L` + +Acceptance: raw/encrypted supported format, public fingerprint, signing statement, required/optional mode, rotation, permissions where available, and secret-absence tests pass. + +Dependencies: V2-03. + +### V2-11: Implement offline evidence verification CLI/library + +Labels: `v2`, `security`, `evidence`, `size:L` + +Acceptance: trusted-key default, explicit result classes/exit codes, all tamper cases, unsigned and v1 classification, and clean-wheel test pass. + +Dependencies: V2-09, V2-10. + +## M4 — Lifecycle and operations + +### V2-12: Add retention snapshot and legal holds + +Labels: `v2`, `security`, `operations`, `size:L` + +Acceptance: default/explicit snapshot, role checks, active/released hold history, concurrency, audit events, and evidence snapshot tests pass. + +Dependencies: V2-04. + +### V2-13: Add mutation-free retention report and lifecycle metrics + +Labels: `v2`, `operations`, `size:M` + +Acceptance: every eligibility state is covered, output is deterministic/digested, aggregate metrics contain no PII, and indexed query plan is reviewed. + +Dependencies: V2-12. + +### V2-14: Add safe backup and scratch restore verification + +Labels: `v2`, `security`, `operations`, `size:L` + +Acceptance: SQLite online backup, bounded safe paths, inventory digests, excluded secrets, audit/evidence verification, corruption cases, and Windows/Linux tests pass. + +Dependencies: V2-11, V2-12. + +### V2-15: Propagate correlation IDs and test structured-log redaction + +Labels: `v2`, `security`, `operations`, `size:M` + +Acceptance: request/delivery/case/job/provider identifiers correlate without PII; tokens, secrets, signatures, bodies, private keys, and free text are absent from protected fields. + +Dependencies: V2-08. + +## M5 — Conditional provider evidence + +### V2-16: Spike Google Reports correlation for administrative sign-out + +Labels: `v2`, `provider`, `discovery`, `size:S` + +Acceptance: fixture-backed result states whether a causal signal exists; sign-out remains `acknowledged` unless the evidence supports promotion; scope/latency/retention limits are documented. + +Dependencies: V2-01. + +### V2-17: Decide whether Entra is in v0.2.0 + +Labels: `v2`, `provider`, `discovery`, `size:S` + +Acceptance: demand gate, permissions, group edge cases, observable postconditions, cloud boundaries, and schedule impact produce a build/defer decision. + +Dependencies: V2-01. + +If and only if V2-17 says build, create separate discovery, adapter, contract-test, least-privilege documentation, and synthetic-tenant rehearsal issues. Do not hide them inside V2-17. + +## M6 — Pilot and release + +### V2-18: Run synthetic pilot rehearsal and recovery drill + +Labels: `v2`, `security`, `release`, `size:L` + +Acceptance: signed trigger through signed evidence, duplicate/retry/exception paths, external-write safety review, backup/restore, key rotation, and retention report pass with artifacts. + +Dependencies: V2-08, V2-11, V2-14, V2-15, and any committed provider work. + +### V2-19: Run design-partner pilot and publish sanitized findings + +Labels: `v2`, `discovery`, `release`, `size:L` + +Acceptance: baseline and outcomes for 5–10 cases where permitted; no PII; observed results separated from estimates; defects and scope changes triaged. + +Dependencies: V2-18. + +### V2-20: Release v0.2.0 + +Labels: `v2`, `documentation`, `release`, `size:M` + +Acceptance: migration/security/operator docs, changelog, quality/security gates, built-wheel smoke, signed release artifacts, protected-main CI, release notes, and roadmap update are verified. + +Dependencies: V2-19 and all committed release blockers. + +## Recommended first GitHub batch after approval + +Open only V2-01 through V2-04 plus milestone shells. Create implementation issues V2-05 onward after M0 confirms the thesis and M1 freezes contracts. This keeps GitHub commitments aligned with evidence. diff --git a/.dev/v2/MILESTONES.md b/.dev/v2/MILESTONES.md new file mode 100644 index 0000000..6fa67bf --- /dev/null +++ b/.dev/v2/MILESTONES.md @@ -0,0 +1,145 @@ +# V2 Milestones and Critical Path + +Status: DRAFT + +The milestones are sequenced by dependency and decision risk, not calendar dates. Estimates are relative engineering effort for one focused contributor and exclude waiting for design-partner access. + +## M0 — Pilot validation gate + +Outcome: decide whether to proceed and freeze the pilot path. + +Deliverables: + +- Interview guide and sanitized baseline worksheet. +- 5–8 interviews across at least 3 organizations. +- Named pilot trigger, provider, operator, approvers, and evidence consumer. +- Signing-key and retention ownership decisions. +- Entra commit/defer decision. + +Exit gate: the discovery criteria in `V2_PLAN.md` pass and a product decision record is approved. + +Estimate: 3–5 engineering/product days plus interview scheduling. + +## M1 — Contracts and migration skeleton + +Outcome: freeze security-critical interfaces before implementation. + +Deliverables: + +- Webhook protocol and event JSON Schema. +- Evidence v2 schema, signing statement, and golden vectors. +- Retention/legal-hold model. +- Migrations `0002`–`0004` with upgrade fixtures from v0.1.0. +- Threat-model update and acceptance-test matrix. + +Exit gate: schemas have compatibility/versioning rules and tests fail for unimplemented behavior. + +Depends on: M0. + +Estimate: 4–6 days. + +## M2 — Secure intake vertical slice + +Outcome: a signed external event creates exactly one existing-style case. + +Deliverables: + +- File-backed source/key configuration and readiness checks. +- Raw-body HMAC verification and rotation. +- Delivery persistence, replay/conflict handling, and audit events. +- Thin FastAPI route and synthetic sender example. +- Fuzz/property/concurrency tests. + +Exit gate: the complete webhook acceptance matrix passes on Windows and Linux; no existing case/approval behavior changes. + +Depends on: M1 webhook schema/migration. + +Estimate: 5–7 days. + +## M3 — Signed evidence vertical slice + +Outcome: a completed case produces portable evidence that verifies against an external trust anchor. + +Deliverables: + +- Evidence v2 allowlisted serialization. +- Ed25519 key loading, signing statement, and detached manifest. +- Offline verification library and CLI. +- v1 compatibility classification. +- Key rotation and golden cross-platform tests. + +Exit gate: clean-wheel verification succeeds; every tamper case fails with the specified code; private material absence scan passes. + +Depends on: M1 evidence schema/migration. Can run alongside M2 after M1. + +Estimate: 6–8 days. + +## M4 — Lifecycle and operations + +Outcome: pilots can understand retention, holds, queue health, and recoverability without unsafe deletion. + +Deliverables: + +- Retention snapshot and legal holds. +- Retention dry-run report. +- Aggregate operational metrics. +- SQLite online backup, backup verification, and scratch restore drill. +- Correlation IDs and redaction tests. + +Exit gate: restore drill reproduces audit/evidence verification and retention reports are mutation-free. + +Depends on: M1 retention migration; M3 for signed-artifact restore verification. + +Estimate: 6–9 days. + +## M5 — Conditional provider decision + +Outcome: close provider uncertainty without bloating committed scope. + +Deliverables: + +- Time-boxed Google Reports correlation result. +- Entra provider contract only if M0 demand gate passes. +- Explicit assurance classification for every considered operation. + +Exit gate: decision record says build/defer and why. Any provider code requires contract tests and least-privilege documentation. + +Depends on: M0. Spike can run alongside M2/M3. + +Estimate: 1–3 days for spikes; 6–10 additional days if Entra is committed. + +## M6 — Pilot and v0.2.0 release + +Outcome: measured pilot-ready release. + +Deliverables: + +- Synthetic rehearsal with external writes disabled, then controlled enablement. +- Backup/restore rehearsal. +- 5–10 pilot cases if partner policy permits. +- Baseline/outcome report with PII removed. +- Documentation, changelog, migration guide, security review, and release artifacts. + +Exit gate: all V2 release gates pass; pilot results and assurance limitations are explicit. + +Depends on: M2, M3, M4, and any committed part of M5. + +Estimate: 4–6 engineering days plus pilot observation period. + +## Critical path + +```text +M0 validation + → M1 contracts/migrations + → M2 secure intake ─┐ + → M3 evidence ──────┼→ M4 operations/restore → M6 pilot/release + → M5 provider decision ┘ (only committed work blocks release) +``` + +## Scope controls + +- M0 failure stops implementation. +- M2 and M3 are the minimum coherent V2 product slice. +- M4 may be split only by deferring backup convenience—not retention truth or restore verification. +- Entra adds schedule only after the demand gate; it cannot silently become critical path. +- UI, SSO, PostgreSQL, multi-worker, and physical purge requests go to a later-candidate list. diff --git a/.dev/v2/RECRUITMENT_SCRIPT.md b/.dev/v2/RECRUITMENT_SCRIPT.md new file mode 100644 index 0000000..bd2896f --- /dev/null +++ b/.dev/v2/RECRUITMENT_SCRIPT.md @@ -0,0 +1,42 @@ +# V2 Design-Partner Recruitment Script + +## Short invitation + +> I am researching how organizations authorize, execute, and prove employee offboarding. I am looking for a 30-minute problem-discovery conversation with people who perform departures, approve them, or consume the resulting security/audit evidence. This is not a sales call. Please do not share employee names, credentials, tenant configuration, confidential screenshots, or other sensitive data. Findings will be stored only as anonymized patterns and ranges. Would you or an appropriate colleague be willing to participate? + +Keep names, email addresses, and scheduling details outside the repository. + +## Eligibility screen + +Ask only enough to establish coverage: + +1. Which role best describes the participant: IT operator/MSP technician, HR/line-manager approver, or security/compliance/evidence consumer? +2. Have they participated in an employee departure or role-change workflow during the last 12 months? +3. Can they discuss the workflow without disclosing employee identities, credentials, client names, or confidential configuration? +4. Is their primary identity path Google Workspace, Microsoft Entra ID, another provider, or manual/mock only? +5. Could their organization consider a synthetic rehearsal if the discovery criteria are met? + +Exclude anyone who would need to disclose protected or confidential information to provide useful answers. + +## Consent opening + +Read or paraphrase this before the interview: + +> I will capture sanitized workflow facts, estimates, gaps, and requirements under aliases. I will not record your name, employer, employee identities, tenant identifiers, credentials, live URLs, or screenshots in the project repository. You can skip any question or stop at any time. Please label estimates as estimates and avoid confidential details. Is it okay to continue on that basis? + +Record only `Consent: yes` or `Consent: no` outside the repository. Create a sanitized interview record only after consent is yes. + +## Scheduling mix + +Recruit 5–8 participants across at least 3 organizations: + +- 2–3 IT operators or MSP technicians; +- 1–2 HR or line-manager approvers; +- 1–2 security, compliance, or audit-evidence consumers; +- at least one organization with Google Workspace, while recording Entra demand without steering answers. + +Avoid filling the set with multiple participants who all represent the same organization and role perspective. + +## Follow-up message + +> Thank you. I recorded only anonymized workflow findings. If a pilot path appears viable, I may ask separately whether your organization would evaluate a synthetic rehearsal and an anonymized evidence bundle. Participation in this interview does not commit you or your organization to a pilot. diff --git a/.dev/v2/RETENTION_AND_OPERATIONS_SPEC.md b/.dev/v2/RETENTION_AND_OPERATIONS_SPEC.md new file mode 100644 index 0000000..b1f9685 --- /dev/null +++ b/.dev/v2/RETENTION_AND_OPERATIONS_SPEC.md @@ -0,0 +1,132 @@ +# V2 Retention and Operations Contract + +Status: DRAFT + +## Retention model + +Retention in V2 is a visibility and governance feature, not an automated deletion feature. + +At case creation, OffboardProof snapshots one of: + +- an explicit `retention_until` supplied by an authorized HR/operator request and bounded by policy; or +- the configured default retention period added to the case creation time. + +The snapshot is stored on the case and included in evidence schema v2. Later default-policy changes do not rewrite it. + +Migration `0004_retention_operations.sql` adds: + +```text +cases.retention_until text nullable +cases.retention_policy_id text nullable + +legal_holds +- id UUID primary key +- case_id text not null references cases(id) +- status text: active | released +- reason text not null +- created_by text not null references actors(id) +- created_at text not null +- released_by text nullable references actors(id) +- released_at text nullable +- release_reason text nullable +``` + +Only one active hold may exist per case, enforced by service logic and a partial unique index where supported by SQLite. + +## Authorization + +- HR/operator may set an allowed retention date only at case creation. +- Security/admin may create or release a legal hold. +- Auditor/operator/security/admin may read retention and hold state. +- No V2 role can physically purge evidence through the product. + +Audit events: + +- `retention.snapshot_created` +- `legal_hold.created` +- `legal_hold.released` +- `retention.expiring_reported` +- `retention.expired_reported` + +## Retention dry run + +CLI: + +```text +offboardproof retention-report --as-of 2027-08-17T00:00:00Z --format json +``` + +Each result includes case ID, completion state, retention date, active-hold flag, evidence artifact paths/digests, database row counts by table, and eligibility reason. It never deletes or moves data. + +Eligibility categories: + +- `not_expired` +- `active_legal_hold` +- `incomplete_case` +- `expired_review_required` +- `missing_retention_policy` + +The report itself includes generation time, database schema version, case count, and SHA-256 digest. + +## Operational metrics + +Extend the existing summary with bounded aggregate data only: + +- cases by state +- controls by evidence quality/state +- queued/leased/dead jobs +- oldest available queued job age +- open and overdue exceptions +- evidence signed/unsigned/failed +- cases expiring within configurable horizon +- cases expired with/without legal hold +- duplicate webhook deliveries suppressed +- webhook conflicts and authenticated schema rejections + +No metric contains employee email, display name, group name, free-text reason, or provider receipt. + +## Backup and restore + +Commands: + +```text +offboardproof backup-create --destination +offboardproof backup-verify --backup +offboardproof restore-verify --backup --scratch +``` + +V2 does not overwrite a live database through a restore command. `restore-verify` restores into a new scratch location, migrates only when explicitly requested, validates SQLite integrity/foreign keys, verifies the audit chain, checks evidence file digests/signatures, and emits a machine-readable report. + +Backup contents: + +- SQLite online backup output, not a raw copy of a live WAL database. +- Evidence and manifest files referenced by artifact rows. +- Public signing keys and non-secret configuration snapshot. +- Inventory manifest containing relative paths, sizes, and SHA-256 digests. + +Excluded by default: + +- actor bearer tokens +- webhook secret files +- evidence private signing keys/passwords +- `.env` files +- unrelated filesystem content + +The backup command refuses a destination inside the database, evidence, repository, or source directory and refuses a non-empty destination. + +## Required tests + +- Default and explicit retention snapshot behavior at UTC/date boundaries. +- Policy change does not mutate existing case snapshot or signed evidence. +- Legal-hold role enforcement and concurrent duplicate hold creation. +- Dry run classifies every state without mutation. +- Metrics contain no PII/free text and use indexed aggregate queries. +- SQLite online backup during worker activity is internally consistent. +- Restore drill catches missing, extra, changed, or path-traversal artifact entries. +- Restore drill validates audit chain and v1/v2 evidence. +- Backup never includes configured secret/private-key paths. +- Windows and Linux path behavior. + +## Explicit limit + +Secure erasure cannot be guaranteed by deleting a row or file because SQLite pages, WAL files, backups, filesystem snapshots, and storage media may retain bytes. A future purge design requires a separate threat model, legal requirements, backup policy, encryption-key strategy, recovery model, and explicit destructive-action confirmation. diff --git a/.dev/v2/SCOPE_DECISION_TEMPLATE.md b/.dev/v2/SCOPE_DECISION_TEMPLATE.md new file mode 100644 index 0000000..d4dc29f --- /dev/null +++ b/.dev/v2/SCOPE_DECISION_TEMPLATE.md @@ -0,0 +1,98 @@ +# V2 Discovery Scope Decision + +Status: NOT READY — COMPLETE ONLY AFTER THE M0 INTERVIEW SET + +Decision date: TBD + +Decision: KEEP / REVISE / STOP + +## Evidence set + +- Completed interviews: 0 +- Distinct organizations: 0 +- Role coverage: TBD +- Provider coverage: TBD +- Evidence basis: measured / participant-estimated / unknown counts TBD + +## M0 criteria + +| Criterion | Result | Evidence references | +|---|---|---| +| Three participants confirm a recurring, material evidence/verification problem | Not evaluated | | +| Two organizations identify a viable signed trigger and willing evidence consumer | Not evaluated | | +| One Google Workspace or mock-provider pilot path is viable | Not evaluated | | + +## Decision rationale + +Summarize supporting and disconfirming evidence. Do not infer consensus from unmeasured assumptions. + +## Committed pilot path + +- Organization alias: +- Trigger source: +- Provider: +- Operator alias: +- Approval roles: +- Evidence-consumer alias: +- Webhook-secret owner: +- Signing-key owner: +- Synthetic rehearsal boundary: +- Prohibited data: +- Retention and legal-hold obligations: + +## Capability decisions + +| Capability | Score range across interviews | Decision | Evidence references | +|---|---:|---|---| +| Authenticated webhook intake | | Commit / conditional / defer | | +| Signed portable evidence | | Commit / conditional / defer | | +| Retention metadata and legal hold | | Commit / conditional / defer | | +| Google audit correlation | | Commit / conditional / defer | | +| Microsoft Entra adapter | | Commit / conditional / defer | | + +## Scope result + +### Committed for v0.2.0 + +- TBD + +### Conditional spikes or follow-up + +- TBD + +### Explicitly deferred + +- TBD + +## Baseline ranges + +| Metric | Measured range | Participant-estimated range | Unknowns | +|---|---|---|---| +| Operator active minutes | | | | +| Trigger-to-containment time | | | | +| End-to-end elapsed time | | | | +| Human handoffs | | | | +| Exceptions per case and age | | | | + +## Measurable pilot success criteria + +- Required controls reaching verified, acknowledged, or validly waived state by deadline: +- Offline evidence verification target: +- Operator active-time target: +- Trigger-to-containment target: +- Replay and duplicate-write guardrail: +- Assurance-labeling guardrail: + +## Consequences + +- Issues authorized to open: +- Issues that must remain closed/unopened: +- Contract changes required: +- Planning assumptions rejected: + +## Approval + +- Product decision owner: +- Security decision owner: +- Pilot operator acknowledgement: +- Evidence-consumer acknowledgement: diff --git a/.dev/v2/WEBHOOK_SPEC.md b/.dev/v2/WEBHOOK_SPEC.md new file mode 100644 index 0000000..81f5d5d --- /dev/null +++ b/.dev/v2/WEBHOOK_SPEC.md @@ -0,0 +1,215 @@ +# V2 Webhook Intake Contract + +Status: DRAFT + +## Purpose + +Accept an authorized departure event from an external HRIS, ITSM, or integration layer without creating a second authorization or workflow model. A valid webhook must enter the existing `WorkflowService.create_case` path under a dedicated service actor. + +## Endpoint + +`POST /v1/intake/webhooks/{source_id}` + +This endpoint does not accept bearer-token authentication. It authenticates the exact request bytes through the signature protocol below. Unknown or inactive sources return a generic authentication failure. + +## Required headers + +| Header | Meaning | +|---|---| +| `Content-Type` | Must be exactly `application/json`, with an optional UTF-8 charset. | +| `OffboardProof-Delivery` | Source-unique delivery ID, 8–128 URL-safe characters. | +| `OffboardProof-Timestamp` | Unix epoch seconds as an integer. | +| `OffboardProof-Key-Id` | Identifier for the active or temporarily accepted previous key. | +| `OffboardProof-Signature` | Lowercase `sha256=<64 hex characters>`. | + +Header names are case-insensitive under HTTP. Duplicate security headers are rejected. + +## Signature protocol + +The sender computes: + +```text +signed_bytes = b"offboardproof.webhook.v1\n" + + timestamp_ascii + b"\n" + + delivery_id_utf8 + b"\n" + + raw_request_body + +signature = HMAC-SHA256(secret, signed_bytes) +header = "sha256=" + lowercase_hex(signature) +``` + +The receiver: + +1. Enforces a maximum body size before buffering the complete request. +2. Parses and validates the timestamp without using body data. +3. Loads the source/key configuration without revealing whether it exists. +4. Calculates the expected signature over the unmodified raw bytes. +5. Compares signatures with `hmac.compare_digest`. +6. Rejects timestamps outside the configured skew window, default 300 seconds. +7. Parses JSON only after authentication succeeds. + +The server never normalizes JSON before signature verification. + +## Source and secret configuration + +Non-secret source configuration is loaded from a JSON file referenced by `OFFBOARDPROOF_WEBHOOK_CONFIG_FILE`. Each source contains: + +```json +{ + "schema_version": 1, + "sources": { + "hr-primary": { + "active": true, + "actor_id": "service-actor-uuid", + "current_key": { + "key_id": "2026-08", + "secret_file": "C:/secure/offboardproof/hr-primary-2026-08.secret" + }, + "previous_key": { + "key_id": "2026-05", + "secret_file": "C:/secure/offboardproof/hr-primary-2026-05.secret", + "accept_until": "2026-08-24T00:00:00Z" + } + } + } +} +``` + +Rules: + +- Secret files contain raw high-entropy secret bytes and remain outside the repository, database, evidence directory, and backups unless the operator explicitly includes them. +- The service rejects secrets shorter than 32 bytes. +- Relative secret paths are rejected. +- A configured actor must exist, be active, and have role `service`. +- Configuration is validated at readiness time. An invalid source is disabled; a missing global config does not prevent non-webhook use. +- Previous keys are accepted only until `accept_until`; there is never more than one previous key. + +## Canonical event schema + +Unknown fields are rejected. Version 1 is: + +```json +{ + "schema_version": 1, + "event_type": "employee.departure.authorized", + "event_id": "hr-event-0192", + "occurred_at": "2026-08-17T09:30:00Z", + "subject": { + "email": "alex@example.test", + "display_name": "Alex Morgan" + }, + "departure": { + "effective_at": "2026-08-20T17:00:00Z", + "risk_tier": "standard", + "transfer_owner_email": "owner@example.test" + }, + "workflow": { + "provider": "google" + } +} +``` + +Validation: + +- Only `employee.departure.authorized` is accepted in V2. +- All timestamps require an explicit offset and are normalized to UTC. +- `occurred_at` cannot be more than five minutes in the future. +- Email and provider normalization reuse existing domain functions. +- `event_id` is 8–128 URL-safe characters. +- Display name is 1–200 characters after trimming. +- Event size defaults to 64 KiB and is configurable only downward/upward within 8–256 KiB. +- The external event does not supply approvals, controls, waiver state, or an arbitrary organization identifier. + +## Idempotency and conflict behavior + +Migration `0002_webhook_intake.sql` introduces: + +```text +webhook_deliveries +- id UUID primary key +- source_id text not null +- delivery_id text not null +- event_id text not null +- payload_sha256 text not null +- case_id text nullable references cases(id) +- status text: accepted | rejected | conflict +- received_at text not null +- completed_at text nullable +- unique(source_id, delivery_id) +- unique(source_id, event_id) +``` + +The mapped case idempotency key is `webhook:{source_id}:{event_id}`. + +Behavior: + +- Same source, delivery ID, event ID, and payload digest returns the original result. +- Same event ID under a new delivery ID and the same payload returns the original case and records the replay. +- Reuse of a delivery ID or event ID with a different authenticated payload returns `409 Conflict` and creates no case. +- Concurrent identical requests converge on one delivery/event and one case through database uniqueness plus one transaction boundary. +- Authentication failures are not persisted in `webhook_deliveries`; they increment an in-memory/log metric with no sensitive value. +- Authenticated schema failures may be stored as a safe rejection event containing source, delivery ID, event ID when parseable, reason code, and payload digest—but never the body. + +## Response contract + +### First accepted delivery + +Status `202 Accepted`: + +```json +{ + "delivery_id": "delivery-0192", + "event_id": "hr-event-0192", + "case_id": "uuid", + "case_state": "received", + "replayed": false +} +``` + +### Valid replay + +Status `200 OK` with the same representation and `replayed: true`. + +### Errors + +- `400`: authenticated but invalid schema/header formatting. +- `401`: unknown source/key, inactive source, invalid signature, or expired previous key. The message is deliberately generic. +- `409`: authenticated delivery/event identifier reused with different content. +- `413`: body too large; processing stops before JSON parsing. +- `415`: unsupported content type. +- `503`: database unavailable; sender may retry the identical delivery. + +## Audit events + +- `webhook.accepted`: source ID, delivery ID, event ID, payload digest, case ID. +- `webhook.replayed`: same safe identifiers and original case ID. +- `webhook.rejected`: authenticated source, safe reason code, identifiers if valid, payload digest. +- `webhook.conflict`: safe identifiers and both digests; no payload. + +Signature values, secrets, raw bodies, and full request headers are never audit payloads. + +## Threat cases that require tests + +- Signature computed after JSON reformatting. +- Timestamp represented with whitespace, sign, decimal, overflow, or duplicate header. +- Header/body Unicode normalization differences. +- Source/key enumeration through timing or error text. +- Valid request replayed before and after the skew window. +- Two simultaneous deliveries for one event. +- Delivery ID collision across sources. +- Secret rotation boundary and clock skew. +- Slow or oversized body. +- Database failure before delivery insert, after insert, and after case creation. +- Attempt to inject approvals, roles, organization, or control definitions through unknown fields. + +## Implementation boundary + +Suggested modules: + +- `offboardproof/webhooks/config.py` +- `offboardproof/webhooks/signature.py` +- `offboardproof/webhooks/schemas.py` +- `offboardproof/webhooks/service.py` +- one thin route in `api.py` + +The webhook service may depend on `WorkflowService`; the domain service must not depend on FastAPI request objects. From 38124fa8afe789e8a170adbebb53472e39d67677 Mon Sep 17 00:00:00 2001 From: DevFoundry-labs Date: Thu, 20 Aug 2026 17:25:23 +0400 Subject: [PATCH 2/3] Implement V2 secure intake and evidence lifecycle --- .dev/V2_PLAN.md | 2 +- .dev/v2/DECISIONS.md | 8 +- .env.example | 11 + CHANGELOG.md | 18 + README.md | 21 +- docs/roadmap.md | 9 +- docs/security-model.md | 9 +- docs/v2-operations.md | 58 +++ migrations/0002_webhook_intake.sql | 18 + migrations/0003_evidence_v2.sql | 13 + migrations/0004_retention_operations.sql | 26 ++ pyproject.toml | 3 +- src/offboardproof/__init__.py | 2 +- src/offboardproof/api.py | 131 +++++- src/offboardproof/backup.py | 290 +++++++++++++ src/offboardproof/cli.py | 96 +++++ src/offboardproof/config.py | 11 + src/offboardproof/db.py | 10 +- src/offboardproof/errors.py | 16 + src/offboardproof/evidence_signing.py | 104 +++++ src/offboardproof/evidence_v2.py | 397 ++++++++++++++++++ src/offboardproof/evidence_verification.py | 87 ++++ .../migrations/0002_webhook_intake.sql | 18 + .../migrations/0003_evidence_v2.sql | 13 + .../migrations/0004_retention_operations.sql | 26 ++ src/offboardproof/retention.py | 176 ++++++++ src/offboardproof/schemas.py | 21 +- src/offboardproof/service.py | 364 +++++++++++----- src/offboardproof/webhooks/__init__.py | 1 + src/offboardproof/webhooks/config.py | 73 ++++ src/offboardproof/webhooks/schemas.py | 64 +++ src/offboardproof/webhooks/service.py | 194 +++++++++ src/offboardproof/webhooks/signature.py | 53 +++ src/offboardproof/worker.py | 12 +- tests/test_api.py | 33 +- tests/test_backup.py | 136 ++++++ tests/test_evidence_v2.py | 205 +++++++++ tests/test_migrations.py | 128 ++++++ tests/test_retention.py | 131 ++++++ tests/test_security_edges.py | 173 ++++++++ tests/test_units.py | 6 +- tests/test_webhooks.py | 215 ++++++++++ 42 files changed, 3258 insertions(+), 124 deletions(-) create mode 100644 docs/v2-operations.md create mode 100644 migrations/0002_webhook_intake.sql create mode 100644 migrations/0003_evidence_v2.sql create mode 100644 migrations/0004_retention_operations.sql create mode 100644 src/offboardproof/backup.py create mode 100644 src/offboardproof/evidence_signing.py create mode 100644 src/offboardproof/evidence_v2.py create mode 100644 src/offboardproof/evidence_verification.py create mode 100644 src/offboardproof/migrations/0002_webhook_intake.sql create mode 100644 src/offboardproof/migrations/0003_evidence_v2.sql create mode 100644 src/offboardproof/migrations/0004_retention_operations.sql create mode 100644 src/offboardproof/retention.py create mode 100644 src/offboardproof/webhooks/__init__.py create mode 100644 src/offboardproof/webhooks/config.py create mode 100644 src/offboardproof/webhooks/schemas.py create mode 100644 src/offboardproof/webhooks/service.py create mode 100644 src/offboardproof/webhooks/signature.py create mode 100644 tests/test_backup.py create mode 100644 tests/test_evidence_v2.py create mode 100644 tests/test_migrations.py create mode 100644 tests/test_retention.py create mode 100644 tests/test_security_edges.py create mode 100644 tests/test_webhooks.py diff --git a/.dev/V2_PLAN.md b/.dev/V2_PLAN.md index 5cbdae9..01deb3e 100644 --- a/.dev/V2_PLAN.md +++ b/.dev/V2_PLAN.md @@ -1,6 +1,6 @@ # OffboardProof V2 Plan -Status: DRAFT CONTRACTS READY FOR PRODUCT REVIEW +Status: ENGINEERING IMPLEMENTED AT RISK — DESIGN-PARTNER GATE STILL OPEN Created: 2026-08-17 diff --git a/.dev/v2/DECISIONS.md b/.dev/v2/DECISIONS.md index 967a484..1c12297 100644 --- a/.dev/v2/DECISIONS.md +++ b/.dev/v2/DECISIONS.md @@ -1,6 +1,12 @@ # V2 Working Decisions -These are planning assumptions adopted on 2026-08-17 so contract work can continue. They are reversible before GitHub issues or production code are created. +These decisions began as planning assumptions on 2026-08-17 and are updated as implementation and discovery progress. + +## V2-005: Engineering may proceed at risk without treating M0 as passed + +Decision: The product owner explicitly directed full V2 continuation on 2026-08-20. Engineering may implement and verify the generic committed slices before discovery completes, but the project must not fabricate interviews, claim pilot validation, expand conditional provider scope, or publish ROI/compliance claims. + +Status: accepted for engineering; M0 remains an open pilot and release-evidence gate. ## V2-001: Treat V2 as v0.2.0 diff --git a/.env.example b/.env.example index 0427c04..d4c4711 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,14 @@ OFFBOARDPROOF_WORKER_LEASE_SECONDS=30 OFFBOARDPROOF_RETRY_DELAYS_SECONDS=1,4,16 # OFFBOARDPROOF_GOOGLE_SERVICE_ACCOUNT_FILE=C:/secure/offboardproof.json # OFFBOARDPROOF_GOOGLE_DELEGATED_ADMIN=admin@example.com +# OFFBOARDPROOF_WEBHOOK_CONFIG_FILE=C:/secure/offboardproof-webhooks.json +OFFBOARDPROOF_WEBHOOK_MAX_BODY_BYTES=65536 +OFFBOARDPROOF_WEBHOOK_TIMESTAMP_SKEW_SECONDS=300 +# OFFBOARDPROOF_SIGNING_KEY_FILE=C:/secure/offboardproof-evidence-key.pem +# OFFBOARDPROOF_SIGNING_KEY_PASSWORD_FILE=C:/secure/offboardproof-evidence-key.password +# OFFBOARDPROOF_SIGNING_KEY_ID=pilot-2026-q3 +OFFBOARDPROOF_REQUIRE_SIGNED_EVIDENCE=false +OFFBOARDPROOF_DEFAULT_RETENTION_DAYS=365 +OFFBOARDPROOF_DEFAULT_RETENTION_POLICY_ID=default-365-days +OFFBOARDPROOF_MAX_RETENTION_DAYS=3650 +OFFBOARDPROOF_RETENTION_EXPIRING_HORIZON_DAYS=30 diff --git a/CHANGELOG.md b/CHANGELOG.md index f4c2e23..506dfc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and semantic versioning. +## [Unreleased] + +### Added + +- HMAC-SHA256 authenticated webhook intake with raw-body verification, rotation windows, replay convergence, conflict detection, and safe audit events. +- Evidence schema v2 with deterministic canonical exports, detached Ed25519 manifests, trusted-key verification, explicit unsigned mode, and key-rotation preservation. +- Forward-only schema migrations from v0.1, including real-fixture upgrade and packaged-migration checks. +- Retention snapshots, role-gated legal holds, mutation-free retention reports, and aggregate lifecycle/queue/evidence/webhook metrics. +- SQLite online backups with inventory digests, secret exclusions, evidence verification, and scratch-only restore drills. +- Request correlation IDs and new CLI commands for evidence verification, retention reporting, backup, and restore verification. + +### Security + +- Signing-required failures cannot mark a case completed and are routed to an explicit operational exception. +- Evidence verification requires an operator-supplied trust anchor and rejects substituted embedded keys. +- Webhook signatures, secrets, raw request bodies, private signing keys, and password files are excluded from persisted audit evidence and backups. + ## [0.1.0] - 2026-08-10 ### Added @@ -13,3 +30,4 @@ All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1. - REST API, CLI, synthetic demo, tests, security scans, and release automation. [0.1.0]: https://github.com/DevFoundry-labs/offboardproof/releases/tag/v0.1.0 +[Unreleased]: https://github.com/DevFoundry-labs/offboardproof/compare/v0.1.0...HEAD diff --git a/README.md b/README.md index 58f5b60..96b094a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ OffboardProof is a verification-first employee-offboarding service. It turns an authorized departure into a digest-bound plan, collects role-separated approvals, executes replay-safe identity actions, observes resulting state, routes failures to explicit exceptions, and exports tamper-evident evidence. -It is intentionally local-first: one Python process, SQLite, a REST API, a CLI, and a durable worker. External writes are disabled by default. +It is intentionally local-first: one Python process, SQLite, a REST API, a CLI, and a durable worker. External writes are disabled by default. V2 adds authenticated external intake, portable signed evidence, and explicit evidence lifecycle operations without introducing a hosted control plane. ## Why it exists @@ -49,6 +49,10 @@ Interactive OpenAPI documentation is then available at `http://127.0.0.1:8000/do - Every action and case has an idempotency key. - Jobs use durable leases, bounded retries, reconciliation before mutation, and explicit outcome-unknown handling. - Audit events form a hash chain. Evidence is generated at the `case.completed` boundary. +- Webhook intake authenticates the exact raw request bytes with HMAC-SHA256, bounded clock skew, rotating file-backed secrets, and replay-safe delivery/event identifiers. +- Evidence v2 uses deterministic canonical JSON and a detached Ed25519 manifest. Verification requires an operator-trusted public key; a bundled key is never trusted implicitly. +- Retention is snapshotted at case creation. Legal holds and dry-run reports are supported, but automatic physical deletion is intentionally absent. +- Backups use SQLite online backup, inventory digests, evidence verification, and scratch-only restore drills; webhook secrets and private signing keys are excluded. - Google Workspace writes require both configuration and `OFFBOARDPROOF_ENABLE_EXTERNAL_WRITES=true`. - Logs and errors avoid credentials; evidence contains operational identity data and must be protected. @@ -58,9 +62,22 @@ Read [docs/security-model.md](docs/security-model.md) before enabling a real pro The built-in `mock` provider is deterministic and fully observable. The bounded Google Workspace adapter supports account suspension, session sign-out, and group removal. Google exposes observable suspension and membership state; its session sign-out endpoint provides acknowledgement rather than a queryable signed-out final state, so OffboardProof labels that control `acknowledged` instead of claiming independent verification. See [docs/google-workspace.md](docs/google-workspace.md). +## V2 operations + +The V2 commands and configuration boundaries are documented in [docs/v2-operations.md](docs/v2-operations.md). Common examples: + +```powershell +offboardproof evidence-export-v2 +offboardproof evidence-verify --evidence --manifest --trusted-public-key +offboardproof retention-report --as-of 2027-08-17T00:00:00Z +offboardproof backup-create --destination D:\offboardproof-backup +offboardproof backup-verify --backup D:\offboardproof-backup +offboardproof restore-verify --backup D:\offboardproof-backup --scratch D:\offboardproof-restore-check +``` + ## Scope -Version 0.1.0 is a focused engine and API, not a universal HRIS, ITSM, or secrets vault. It does not delete accounts, transfer Google Drive content, send notifications, or include a browser dashboard. See [docs/roadmap.md](docs/roadmap.md). +Version 0.2.0 remains a focused single-organization engine and API, not a universal HRIS, ITSM, or secrets vault. It does not physically purge evidence, delete accounts, transfer Google Drive content, send notifications, or include a browser dashboard. Its design-partner validation gate remains open; synthetic measurements are not customer ROI. See [docs/roadmap.md](docs/roadmap.md). ## Development diff --git a/docs/roadmap.md b/docs/roadmap.md index 8f47e51..ca05a9f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -5,9 +5,14 @@ The roadmap is evidence-led; items are candidates, not commitments. ## Near term - Design-partner validation of control sets and evidence exports. -- Signed evidence manifests and configurable retention. - Microsoft Entra ID provider contract. -- Webhook intake with signature verification and replay controls. + +## Implemented for v0.2.0 + +- Signed evidence manifests with explicit external trust anchors. +- Configurable retention snapshots, legal holds, and dry-run reporting without physical deletion. +- Webhook intake with signature verification, rotation windows, replay controls, and safe rejection handling. +- Online backup, inventory verification, and scratch-only restore drills. ## Later diff --git a/docs/security-model.md b/docs/security-model.md index 40a5ec0..eb5d8af 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -13,16 +13,19 @@ Protected assets include employee identifiers, workflow timing, group membership - Provider requests have timeouts and bounded retry classification. - SQL values are parameterized. API request bodies reject unknown fields. - Audit events are chained with SHA-256 over canonical JSON and can be verified offline. -- Evidence records the exact control quality and explicit assurance limitations. +- Evidence v2 is signed with an operator-controlled Ed25519 key and verified against an independently supplied public-key trust anchor. +- Webhooks are authenticated over exact raw bytes with HMAC-SHA256, bounded skew, generic authentication failures, and replay-safe identifiers. +- Retention/legal-hold operations are role-gated; reporting is non-destructive and physical purge is not implemented. +- Backup inventories are digested, evidence is reverified, restores are limited to new scratch locations, and private keys/webhook secrets are excluded. - CI performs linting, strict type checks, tests, dependency audit, secret detection, packaging, and CodeQL analysis. ## Residual risks -- SHA-256 chaining detects later alteration but is not an external timestamp or digital signature. A privileged database operator could rewrite the entire chain. +- SHA-256 chaining detects later alteration but is not an external timestamp. Ed25519 evidence signatures add portable integrity and origin only when the verifier obtained the trusted public key through an authenticated channel. - SQLite encryption at rest is delegated to the host volume. Use full-disk encryption and restrictive ACLs. - Bearer tokens are not an enterprise identity protocol. Put the service behind an SSO-aware proxy for shared deployments. - Acknowledged provider actions are weaker evidence than observed final state. - Evidence includes personal and access data. Apply retention, legal hold, export, and deletion rules appropriate to your jurisdiction. - Domain-wide delegation is powerful. Use a dedicated service account, least-privilege scopes, admin approval, credential rotation, and audit monitoring. -This release is a workflow-control foundation, not a compliance certification. +This release is a workflow-control foundation, not a compliance certification. Design-partner validation remains required before pilot or ROI claims. diff --git a/docs/v2-operations.md b/docs/v2-operations.md new file mode 100644 index 0000000..c2d09ab --- /dev/null +++ b/docs/v2-operations.md @@ -0,0 +1,58 @@ +# V2 operations + +V2 adds secure webhook intake, signed evidence, retention governance, and recoverable backup verification. It remains local-first and single-organization. + +## Authenticated webhook intake + +Set `OFFBOARDPROOF_WEBHOOK_CONFIG_FILE` to an absolute JSON configuration path. Secret paths inside that file must also be absolute, point outside the repository/database/evidence directories, and contain at least 32 bytes. The format and signing bytes are defined in `.dev/v2/WEBHOOK_SPEC.md`. + +Send `POST /v1/intake/webhooks/{source_id}` with one instance of each security header: + +- `OffboardProof-Delivery` +- `OffboardProof-Timestamp` +- `OffboardProof-Key-Id` +- `OffboardProof-Signature` + +Sign the unmodified body bytes. Reformatting the JSON after signing invalidates the request. Replays return the original case; identifier reuse with different authenticated content returns a conflict. + +## Evidence signing and verification + +Configure an absolute Ed25519 private-key path, stable key ID, and optional password-file path. Pilot environments should set `OFFBOARDPROOF_REQUIRE_SIGNED_EVIDENCE=true`. If required signing fails, case completion is rolled back and an operational exception is opened. + +Distribute the exported public key through an authenticated channel. The manifest's embedded public key is a transport copy, not a trust anchor. + +```powershell +offboardproof evidence-public-key-export --output D:\trusted\pilot.pub +offboardproof evidence-export-v2 +offboardproof evidence-verify --evidence --manifest --trusted-public-key D:\trusted\pilot.pub +``` + +## Retention and legal holds + +Cases snapshot their retention date and policy identifier when created. Security or admin actors can create/release legal holds through the API. The retention report classifies cases but never deletes or moves records: + +```powershell +offboardproof retention-report --as-of 2027-08-17T00:00:00Z +``` + +`expired_review_required` means a human/legal review is required. It is not authorization to purge. + +## Backup and restore drill + +Backups must use a new or empty directory outside the repository and evidence directory. They include an SQLite online backup, referenced evidence/manifest files, public keys, a safe configuration snapshot, and a digested inventory. Actor token digests remain in SQLite; raw tokens, webhook secrets, private signing keys, password files, and `.env` files are never copied by the backup command. + +```powershell +offboardproof backup-create --destination D:\offboardproof-backup +offboardproof backup-verify --backup D:\offboardproof-backup +offboardproof restore-verify --backup D:\offboardproof-backup --scratch D:\offboardproof-restore-check +``` + +Restore verification writes only to a new scratch directory. V2 has no command that overwrites the live database. + +## Assurance limits + +- Signed evidence proves integrity and origin only relative to the verifier's trusted public key. +- Provider acknowledgements remain weaker than observed final state. +- Retention reporting does not provide secure erasure. +- Entra ID remains deferred pending design-partner demand. +- Design-partner discovery is still required before pilot, ROI, or compliance claims. diff --git a/migrations/0002_webhook_intake.sql b/migrations/0002_webhook_intake.sql new file mode 100644 index 0000000..843d2b0 --- /dev/null +++ b/migrations/0002_webhook_intake.sql @@ -0,0 +1,18 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE webhook_deliveries ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, + event_id TEXT NOT NULL, + payload_sha256 TEXT NOT NULL CHECK (length(payload_sha256) = 64), + case_id TEXT REFERENCES cases(id) ON DELETE SET NULL, + status TEXT NOT NULL CHECK (status IN ('accepted','rejected','conflict')), + received_at TEXT NOT NULL, + completed_at TEXT, + UNIQUE (source_id, delivery_id), + UNIQUE (source_id, event_id) +); + +CREATE INDEX idx_webhook_deliveries_case ON webhook_deliveries(case_id); +CREATE INDEX idx_webhook_deliveries_status_received ON webhook_deliveries(status, received_at); diff --git a/migrations/0003_evidence_v2.sql b/migrations/0003_evidence_v2.sql new file mode 100644 index 0000000..8baf151 --- /dev/null +++ b/migrations/0003_evidence_v2.sql @@ -0,0 +1,13 @@ +PRAGMA foreign_keys = ON; + +ALTER TABLE evidence_artifacts ADD COLUMN schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version IN (1,2)); +ALTER TABLE evidence_artifacts ADD COLUMN manifest_path TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN evidence_sha256 TEXT CHECK (evidence_sha256 IS NULL OR length(evidence_sha256) = 64); +ALTER TABLE evidence_artifacts ADD COLUMN signing_status TEXT NOT NULL DEFAULT 'legacy' CHECK (signing_status IN ('legacy','unsigned','signed','failed')); +ALTER TABLE evidence_artifacts ADD COLUMN signing_algorithm TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN signing_key_id TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN public_key_fingerprint TEXT CHECK (public_key_fingerprint IS NULL OR length(public_key_fingerprint) = 64); +ALTER TABLE evidence_artifacts ADD COLUMN completion_event_hash TEXT CHECK (completion_event_hash IS NULL OR length(completion_event_hash) = 64); + +CREATE INDEX idx_evidence_artifacts_case_schema ON evidence_artifacts(case_id, schema_version); +CREATE INDEX idx_evidence_artifacts_signing_status ON evidence_artifacts(signing_status, created_at); diff --git a/migrations/0004_retention_operations.sql b/migrations/0004_retention_operations.sql new file mode 100644 index 0000000..ca6709c --- /dev/null +++ b/migrations/0004_retention_operations.sql @@ -0,0 +1,26 @@ +PRAGMA foreign_keys = ON; + +ALTER TABLE cases ADD COLUMN retention_until TEXT; +ALTER TABLE cases ADD COLUMN retention_policy_id TEXT; + +CREATE INDEX idx_cases_retention_until ON cases(retention_until); + +CREATE TABLE legal_holds ( + id TEXT PRIMARY KEY, + case_id TEXT NOT NULL REFERENCES cases(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('active','released')), + reason TEXT NOT NULL CHECK (length(trim(reason)) > 0), + created_by TEXT NOT NULL REFERENCES actors(id), + created_at TEXT NOT NULL, + released_by TEXT REFERENCES actors(id), + released_at TEXT, + release_reason TEXT, + CHECK ( + (status = 'active' AND released_by IS NULL AND released_at IS NULL AND release_reason IS NULL) + OR + (status = 'released' AND released_by IS NOT NULL AND released_at IS NOT NULL AND length(trim(release_reason)) > 0) + ) +); + +CREATE UNIQUE INDEX idx_legal_holds_one_active_per_case ON legal_holds(case_id) WHERE status = 'active'; +CREATE INDEX idx_legal_holds_case_status ON legal_holds(case_id, status); diff --git a/pyproject.toml b/pyproject.toml index d1a7985..3e556bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "offboardproof" -version = "0.1.0" +version = "0.2.0" description = "Verification-first employee offboarding with approvals, recoverable actions, and audit evidence." readme = "README.md" requires-python = ">=3.12" @@ -19,6 +19,7 @@ classifiers = [ "Topic :: Security", ] dependencies = [ + "cryptography>=44,<51", "fastapi>=0.115,<1", "google-auth>=2.38,<3", "pydantic-settings>=2.7,<3", diff --git a/src/offboardproof/__init__.py b/src/offboardproof/__init__.py index 289b27f..5362955 100644 --- a/src/offboardproof/__init__.py +++ b/src/offboardproof/__init__.py @@ -1,3 +1,3 @@ """OffboardProof: verification-first employee offboarding.""" -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/offboardproof/api.py b/src/offboardproof/api.py index 245c16a..5cf31e8 100644 --- a/src/offboardproof/api.py +++ b/src/offboardproof/api.py @@ -1,9 +1,14 @@ +import hashlib +import re import sqlite3 +import uuid +from datetime import UTC, datetime from typing import Annotated, Any from fastapi import Depends, FastAPI, Header, Request, status from fastapi.responses import JSONResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import ValidationError from offboardproof import __version__ from offboardproof.auth import Actor, authenticate, require_role @@ -15,16 +20,27 @@ ConflictError, NotFoundError, OffboardProofError, + PayloadTooLargeError, + UnsupportedMediaTypeError, + WebhookAuthenticationError, + WebhookValidationError, ) from offboardproof.evidence import build_evidence +from offboardproof.retention import RetentionService from offboardproof.schemas import ( ApprovalCreate, CaseCreate, + LegalHoldCreate, + LegalHoldRelease, ManualComplete, RequeueCreate, WaiverCreate, ) from offboardproof.service import WorkflowService +from offboardproof.webhooks.config import load_credential +from offboardproof.webhooks.schemas import WebhookEvent +from offboardproof.webhooks.service import WebhookService +from offboardproof.webhooks.signature import parse_timestamp, verify_signature from offboardproof.worker import Worker bearer = HTTPBearer(auto_error=False) @@ -35,6 +51,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: migrate(app_settings) service = WorkflowService(app_settings) worker = Worker(app_settings) + webhook_service = WebhookService(app_settings, service) + retention_service = RetentionService(app_settings) app = FastAPI( title="OffboardProof API", @@ -46,6 +64,17 @@ def create_app(settings: Settings | None = None) -> FastAPI: ) app.state.settings = app_settings app.state.service = service + app.state.webhook_service = webhook_service + app.state.retention_service = retention_service + + @app.middleware("http") + async def request_correlation(request: Request, call_next: Any) -> Any: + supplied = request.headers.get("x-request-id") + request_id = supplied if supplied and re.fullmatch(r"[A-Za-z0-9._~-]{8,128}", supplied) else str(uuid.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-ID"] = request_id + return response def current_actor( credentials: Annotated[HTTPAuthorizationCredentials | None, Depends(bearer)], @@ -65,6 +94,12 @@ async def domain_error(_: Request, exc: OffboardProofError) -> JSONResponse: code = status.HTTP_400_BAD_REQUEST if isinstance(exc, AuthorizationError): code = status.HTTP_403_FORBIDDEN + elif isinstance(exc, WebhookAuthenticationError): + code = status.HTTP_401_UNAUTHORIZED + elif isinstance(exc, PayloadTooLargeError): + code = status.HTTP_413_CONTENT_TOO_LARGE + elif isinstance(exc, UnsupportedMediaTypeError): + code = status.HTTP_415_UNSUPPORTED_MEDIA_TYPE elif isinstance(exc, NotFoundError): code = status.HTTP_404_NOT_FOUND elif isinstance(exc, ConflictError): @@ -86,6 +121,13 @@ async def database_error(_: Request, exc: sqlite3.Error) -> JSONResponse: }, ) + @app.exception_handler(ValueError) + async def value_error(_: Request, exc: ValueError) -> JSONResponse: + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content={"error": {"code": "InvalidRequest", "message": str(exc)}}, + ) + @app.get("/health/live", tags=["health"]) def live() -> dict[str, str]: return {"status": "ok", "version": __version__} @@ -107,6 +149,81 @@ def create_case( ) -> dict[str, Any]: return service.create_case(actor, body, idempotency_key) + @app.post("/v1/intake/webhooks/{source_id}", tags=["intake"]) + async def intake_webhook(source_id: str, request: Request) -> JSONResponse: + if re.fullmatch(r"[a-z][a-z0-9_-]{1,63}", source_id) is None: + raise WebhookAuthenticationError("Webhook authentication failed") + content_type = request.headers.get("content-type", "").lower().replace(" ", "") + if content_type not in {"application/json", "application/json;charset=utf-8"}: + raise UnsupportedMediaTypeError("Content-Type must be application/json") + + security_headers = { + "offboardproof-delivery", + "offboardproof-timestamp", + "offboardproof-key-id", + "offboardproof-signature", + } + values: dict[str, str] = {} + for name in security_headers: + raw_values = [ + value.decode("latin-1") + for header, value in request.scope["headers"] + if header.decode("latin-1").lower() == name + ] + if len(raw_values) != 1: + raise WebhookAuthenticationError("Webhook authentication failed") + values[name] = raw_values[0] + + now = datetime.now(UTC) + timestamp_text = values["offboardproof-timestamp"] + parse_timestamp(timestamp_text, now=now, skew_seconds=app_settings.webhook_timestamp_skew_seconds) + + body = bytearray() + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > app_settings.webhook_max_body_bytes: + raise PayloadTooLargeError("Webhook body exceeds the configured limit") + raw_body = bytes(body) + credential = load_credential( + app_settings, + source_id=source_id, + key_id=values["offboardproof-key-id"], + now=now, + ) + delivery_id = values["offboardproof-delivery"] + verify_signature( + secret=credential.secret, + timestamp_text=timestamp_text, + delivery_id=delivery_id, + raw_body=raw_body, + signature_header=values["offboardproof-signature"], + ) + payload_sha256 = hashlib.sha256(raw_body).hexdigest() + try: + event = WebhookEvent.model_validate_json(raw_body) + event.validate_clock(now) + except (ValidationError, ValueError) as exc: + webhook_service.record_rejection( + source_id=source_id, + actor_id=credential.actor_id, + delivery_id=delivery_id, + payload_sha256=payload_sha256, + reason_code="invalid_event_schema", + ) + raise WebhookValidationError("Authenticated webhook event failed schema validation") from exc + + result = webhook_service.accept( + source_id=source_id, + actor_id=credential.actor_id, + delivery_id=delivery_id, + event=event, + payload_sha256=payload_sha256, + ) + return JSONResponse( + status_code=status.HTTP_200_OK if result.replayed else status.HTTP_202_ACCEPTED, + content=result.as_dict(), + ) + @app.get("/v1/cases", tags=["cases"]) def list_cases(actor: ActorDep, limit: int = 100) -> list[dict[str, Any]]: require_role(actor, Role.HR, Role.MANAGER, Role.OPERATOR, Role.SECURITY, Role.AUDITOR) @@ -163,10 +280,22 @@ def evidence(case_id: str, actor: ActorDep) -> dict[str, Any]: connection.close() @app.get("/v1/metrics/summary", tags=["operations"]) - def metrics(actor: ActorDep) -> dict[str, int]: + def metrics(actor: ActorDep) -> dict[str, int | float]: require_role(actor, Role.OPERATOR, Role.SECURITY, Role.AUDITOR) return service.metrics() + @app.post("/v1/cases/{case_id}/legal-holds", status_code=status.HTTP_201_CREATED, tags=["retention"]) + def create_legal_hold(case_id: str, body: LegalHoldCreate, actor: ActorDep) -> dict[str, Any]: + return retention_service.create_hold(actor, case_id, body.reason) + + @app.post("/v1/cases/{case_id}/legal-holds/release", tags=["retention"]) + def release_legal_hold(case_id: str, body: LegalHoldRelease, actor: ActorDep) -> dict[str, Any]: + return retention_service.release_hold(actor, case_id, body.reason) + + @app.get("/v1/retention/report", tags=["retention"]) + def retention_report(actor: ActorDep, as_of: datetime | None = None) -> dict[str, Any]: + return retention_service.report(actor, as_of or datetime.now(UTC)) + return app diff --git a/src/offboardproof/backup.py b/src/offboardproof/backup.py new file mode 100644 index 0000000..2cc3963 --- /dev/null +++ b/src/offboardproof/backup.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from offboardproof.audit import verify_chain +from offboardproof.auth import Actor, require_role +from offboardproof.config import Settings +from offboardproof.db import connection_for, migrate +from offboardproof.enums import Role +from offboardproof.errors import ConfigurationError +from offboardproof.evidence_signing import b64url_decode, load_signing_key, write_public_key +from offboardproof.evidence_verification import EvidenceVerificationError, verify_evidence_v2 +from offboardproof.util import canonical_json, iso_now, sha256_text + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _inside(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + return True + except ValueError: + return False + + +def _safe_relative(value: str) -> Path: + path = Path(value) + if path.is_absolute() or not path.parts or ".." in path.parts: + raise ConfigurationError("Backup inventory contains an unsafe path") + return path + + +@dataclass(frozen=True) +class BackupVerification: + valid: bool + file_count: int + audit_event_count: int + signed_evidence_verified: int + unsigned_evidence_count: int + + def as_dict(self) -> dict[str, Any]: + return { + "valid": self.valid, + "file_count": self.file_count, + "audit_event_count": self.audit_event_count, + "signed_evidence_verified": self.signed_evidence_verified, + "unsigned_evidence_count": self.unsigned_evidence_count, + } + + +class BackupService: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _prepare_empty_directory(self, destination: Path) -> Path: + destination = destination.resolve() + repository_root = Path(__file__).resolve().parents[2] + forbidden = [repository_root, self.settings.evidence_dir.resolve()] + if any(_inside(destination, path) for path in forbidden): + raise ConfigurationError("Backup destination must be outside the repository and evidence directory") + if destination.exists(): + if not destination.is_dir() or any(destination.iterdir()): + raise ConfigurationError("Backup destination must be a new or empty directory") + else: + destination.mkdir(parents=True) + return destination + + def create(self, actor: Actor, destination: Path) -> dict[str, Any]: + require_role(actor, Role.OPERATOR, Role.SECURITY) + destination = self._prepare_empty_directory(destination) + database_dir = destination / "database" + evidence_dir = destination / "evidence" + public_dir = destination / "public-keys" + config_dir = destination / "config" + for directory in (database_dir, evidence_dir, public_dir, config_dir): + directory.mkdir() + + backup_database = database_dir / "offboardproof.db" + source = connection_for(self.settings) + target = sqlite3.connect(backup_database) + try: + source.backup(target) + finally: + target.close() + + entries: list[dict[str, Any]] = [] + entries.append(self._entry(destination, backup_database, kind="database")) + historical_public_keys: dict[str, bytes] = {} + try: + artifacts = source.execute( + """ + SELECT id, path, manifest_path, schema_version, signing_status, + signing_key_id, public_key_fingerprint + FROM evidence_artifacts ORDER BY id + """ + ).fetchall() + for artifact in artifacts: + for field, kind in (("path", "evidence"), ("manifest_path", "manifest")): + value = artifact[field] + if not value: + continue + original = Path(value).resolve() + if not original.is_file() or not _inside(original, self.settings.evidence_dir.resolve()): + raise ConfigurationError("Evidence artifact path is missing or outside the evidence directory") + copied = evidence_dir / str(artifact["id"]) / original.name + copied.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(original, copied) + entries.append( + self._entry( + destination, + copied, + kind=kind, + artifact_id=str(artifact["id"]), + signing_key_id=artifact["signing_key_id"], + ) + ) + if kind == "manifest" and artifact["signing_status"] == "signed": + try: + manifest = json.loads(original.read_text(encoding="utf-8")) + key_id = str(artifact["signing_key_id"]) + raw_public = b64url_decode(manifest["public_key"]["value"]) + fingerprint = hashlib.sha256(raw_public).hexdigest() + if ( + len(raw_public) != 32 + or fingerprint != artifact["public_key_fingerprint"] + or fingerprint != manifest["public_key"]["fingerprint_sha256"] + or (key_id in historical_public_keys and historical_public_keys[key_id] != raw_public) + ): + raise ConfigurationError("Signed evidence public-key metadata is inconsistent") + historical_public_keys[key_id] = raw_public + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ConfigurationError("Signed evidence manifest is malformed") from exc + finally: + source.close() + + for key_id, raw_public in sorted(historical_public_keys.items()): + public_path = public_dir / f"{key_id}.pub" + public_path.write_bytes(raw_public) + entries.append(self._entry(destination, public_path, kind="public_key", key_id=key_id)) + + signing_key = load_signing_key(self.settings) + if signing_key is not None and signing_key.key_id not in historical_public_keys: + public_path = public_dir / f"{signing_key.key_id}.pub" + write_public_key(signing_key, public_path) + entries.append(self._entry(destination, public_path, kind="public_key", key_id=signing_key.key_id)) + + safe_config = { + "schema_version": 1, + "default_retention_days": self.settings.default_retention_days, + "default_retention_policy_id": self.settings.default_retention_policy_id, + "max_retention_days": self.settings.max_retention_days, + "require_signed_evidence": self.settings.require_signed_evidence, + "signing_key_id": self.settings.signing_key_id, + "enable_external_writes": self.settings.enable_external_writes, + } + config_path = config_dir / "settings.json" + config_path.write_text(json.dumps(safe_config, indent=2, sort_keys=True) + "\n", encoding="utf-8") + entries.append(self._entry(destination, config_path, kind="configuration")) + + stable_inventory = {"schema_version": 1, "entries": sorted(entries, key=lambda item: item["path"])} + inventory = { + **stable_inventory, + "created_at": iso_now(), + "inventory_sha256": sha256_text(canonical_json(stable_inventory)), + } + inventory_path = destination / "inventory.json" + inventory_path.write_text(json.dumps(inventory, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return { + "backup": str(destination), + "file_count": len(entries) + 1, + "inventory_sha256": inventory["inventory_sha256"], + } + + @staticmethod + def _entry(root: Path, path: Path, *, kind: str, **metadata: Any) -> dict[str, Any]: + return { + "path": path.relative_to(root).as_posix(), + "size": path.stat().st_size, + "sha256": _sha256_file(path), + "kind": kind, + **metadata, + } + + def verify(self, backup: Path) -> BackupVerification: + backup = backup.resolve() + inventory_path = backup / "inventory.json" + try: + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + entries = inventory["entries"] + stable = {"schema_version": inventory["schema_version"], "entries": entries} + if inventory["inventory_sha256"] != sha256_text(canonical_json(stable)): + raise ConfigurationError("Backup inventory digest is invalid") + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise ConfigurationError("Backup inventory is missing or malformed") from exc + + expected = {"inventory.json"} + by_artifact: dict[str, dict[str, Path]] = {} + public_keys: dict[str, Path] = {} + for entry in entries: + relative = _safe_relative(entry["path"]) + expected.add(relative.as_posix()) + path = (backup / relative).resolve() + if not _inside(path, backup) or not path.is_file(): + raise ConfigurationError("Backup inventory references a missing or unsafe file") + if path.stat().st_size != entry["size"] or _sha256_file(path) != entry["sha256"]: + raise ConfigurationError(f"Backup file failed digest verification: {relative.as_posix()}") + artifact_id = entry.get("artifact_id") + if artifact_id: + by_artifact.setdefault(str(artifact_id), {})[str(entry["kind"])] = path + if entry.get("kind") == "public_key" and entry.get("key_id"): + public_keys[str(entry["key_id"])] = path + + actual = {path.relative_to(backup).as_posix() for path in backup.rglob("*") if path.is_file()} + if actual != expected: + raise ConfigurationError("Backup contains missing or unlisted files") + + database = backup / "database" / "offboardproof.db" + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + try: + if connection.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise ConfigurationError("Backup database integrity check failed") + if connection.execute("PRAGMA foreign_key_check").fetchall(): + raise ConfigurationError("Backup database foreign-key check failed") + chain_valid, event_count, _ = verify_chain(connection) + if not chain_valid: + raise ConfigurationError("Backup audit chain is invalid") + signed_verified = 0 + unsigned_count = 0 + for artifact in connection.execute( + "SELECT id, signing_status, signing_key_id FROM evidence_artifacts WHERE schema_version=2" + ).fetchall(): + paths = by_artifact.get(str(artifact["id"]), {}) + if "evidence" not in paths or "manifest" not in paths: + raise ConfigurationError("Backup is missing a v2 evidence pair") + if artifact["signing_status"] == "signed": + public_key = public_keys.get(str(artifact["signing_key_id"])) + if public_key is None: + raise ConfigurationError("Backup is missing a trusted public key") + try: + verify_evidence_v2(paths["evidence"], paths["manifest"], public_key) + except EvidenceVerificationError as exc: + raise ConfigurationError("Backup signed evidence verification failed") from exc + signed_verified += 1 + elif artifact["signing_status"] == "unsigned": + unsigned_count += 1 + finally: + connection.close() + return BackupVerification(True, len(expected), event_count, signed_verified, unsigned_count) + + def restore_verify(self, backup: Path, scratch: Path, *, migrate_schema: bool = False) -> dict[str, Any]: + verification = self.verify(backup) + scratch = self._prepare_empty_directory(scratch) + restored_database = scratch / "offboardproof.db" + shutil.copy2(backup.resolve() / "database" / "offboardproof.db", restored_database) + restored_evidence = scratch / "evidence" + source_evidence = backup.resolve() / "evidence" + if source_evidence.exists(): + shutil.copytree(source_evidence, restored_evidence) + restored_settings = Settings(database_path=restored_database, evidence_dir=restored_evidence) + if migrate_schema: + migrate(restored_settings) + connection = connection_for(restored_settings) + try: + integrity = connection.execute("PRAGMA integrity_check").fetchone()[0] + foreign_keys = connection.execute("PRAGMA foreign_key_check").fetchall() + chain_valid, event_count, broken = verify_chain(connection) + finally: + connection.close() + return { + **verification.as_dict(), + "scratch": str(scratch), + "database_integrity": integrity, + "foreign_key_violations": len(foreign_keys), + "audit_chain_valid": chain_valid, + "audit_event_count": event_count, + "broken_audit_event": broken, + } diff --git a/src/offboardproof/cli.py b/src/offboardproof/cli.py index 2e0f882..6a17bc2 100644 --- a/src/offboardproof/cli.py +++ b/src/offboardproof/cli.py @@ -13,12 +13,17 @@ from offboardproof.api import create_app from offboardproof.audit import append_event, verify_chain from offboardproof.auth import Actor, authenticate, create_actor +from offboardproof.backup import BackupService from offboardproof.config import Settings from offboardproof.db import connection_for, migrate, transaction from offboardproof.enums import ApprovalDecision, RiskTier, Role from offboardproof.evidence import build_evidence, write_evidence +from offboardproof.evidence_signing import load_signing_key, write_public_key +from offboardproof.evidence_v2 import write_evidence_v2 +from offboardproof.evidence_verification import EvidenceVerificationError, verify_evidence_v2 from offboardproof.logging_config import configure_logging from offboardproof.providers.mock import MockProvider +from offboardproof.retention import RetentionService from offboardproof.schemas import CaseCreate from offboardproof.service import WorkflowService from offboardproof.worker import Worker @@ -235,6 +240,54 @@ def evidence_export( connection.close() +@app.command("evidence-export-v2") +def evidence_export_v2( + case_id: Annotated[str, typer.Argument()], + database: Annotated[Path | None, typer.Option()] = None, +) -> None: + settings = _settings(database) + migrate(settings) + connection = connection_for(settings) + try: + evidence_path, manifest_path, digest = write_evidence_v2(connection, settings, case_id) + _print( + { + "evidence": str(evidence_path), + "manifest": str(manifest_path), + "evidence_sha256": digest, + } + ) + finally: + connection.close() + + +@app.command("evidence-public-key-export") +def evidence_public_key_export( + output: Annotated[Path, typer.Option(help="New or replacement public-key PEM file.")], +) -> None: + signing_key = load_signing_key(Settings()) + if signing_key is None: + raise typer.BadParameter("Signing is not configured") + output = output.resolve() + if output.exists() and not output.is_file(): + raise typer.BadParameter("Output must be a file path") + write_public_key(signing_key, output) + _print({"output": str(output), "key_id": signing_key.key_id}) + + +@app.command("evidence-verify") +def evidence_verify( + evidence: Annotated[Path, typer.Option(help="Canonical evidence v2 JSON file.")], + manifest: Annotated[Path, typer.Option(help="Detached evidence manifest JSON file.")], + trusted_public_key: Annotated[Path, typer.Option(help="Operator-trusted Ed25519 public key.")], +) -> None: + try: + _print(verify_evidence_v2(evidence, manifest, trusted_public_key)) + except EvidenceVerificationError as exc: + _print(exc.result) + raise typer.Exit(code=exc.exit_code) from exc + + @app.command("audit-verify") def audit_verify( database: Annotated[Path | None, typer.Option()] = None, @@ -250,6 +303,49 @@ def audit_verify( connection.close() +@app.command("retention-report") +def retention_report( + as_of: Annotated[str | None, typer.Option(help="ISO-8601 timestamp; defaults to now.")] = None, + database: Annotated[Path | None, typer.Option()] = None, +) -> None: + settings = _settings(database) + migrate(settings) + report_time = datetime.fromisoformat(as_of.replace("Z", "+00:00")) if as_of else datetime.now(UTC) + if report_time.tzinfo is None: + raise typer.BadParameter("--as-of must include a UTC offset") + _print(RetentionService(settings).report(_actor(settings), report_time)) + + +@app.command("backup-create") +def backup_create( + destination: Annotated[Path, typer.Option(help="New or empty backup directory outside the repository.")], + database: Annotated[Path | None, typer.Option()] = None, +) -> None: + settings = _settings(database) + migrate(settings) + _print(BackupService(settings).create(_actor(settings), destination)) + + +@app.command("backup-verify") +def backup_verify( + backup: Annotated[Path, typer.Option(help="Backup directory to verify without mutation.")], + database: Annotated[Path | None, typer.Option()] = None, +) -> None: + settings = _settings(database) + _print(BackupService(settings).verify(backup).as_dict()) + + +@app.command("restore-verify") +def restore_verify( + backup: Annotated[Path, typer.Option(help="Verified backup directory.")], + scratch: Annotated[Path, typer.Option(help="New or empty scratch restore directory.")], + migrate_schema: Annotated[bool, typer.Option(help="Migrate the scratch database after restoration.")] = False, + database: Annotated[Path | None, typer.Option()] = None, +) -> None: + settings = _settings(database) + _print(BackupService(settings).restore_verify(backup, scratch, migrate_schema=migrate_schema)) + + @app.command("serve") def serve( host: Annotated[str, typer.Option()] = "127.0.0.1", diff --git a/src/offboardproof/config.py b/src/offboardproof/config.py index 34f06f0..eb74e64 100644 --- a/src/offboardproof/config.py +++ b/src/offboardproof/config.py @@ -17,6 +17,17 @@ class Settings(BaseSettings): retry_delays_seconds: str = "1,4,16" google_service_account_file: Path | None = None google_delegated_admin: str | None = None + webhook_config_file: Path | None = None + webhook_max_body_bytes: int = Field(default=65_536, ge=8_192, le=262_144) + webhook_timestamp_skew_seconds: int = Field(default=300, ge=30, le=3_600) + signing_key_file: Path | None = None + signing_key_password_file: Path | None = None + signing_key_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._~-]{1,128}$") + require_signed_evidence: bool = False + default_retention_days: int = Field(default=365, ge=1, le=3_650) + default_retention_policy_id: str = Field(default="default-365-days", pattern=r"^[A-Za-z0-9._~-]{1,128}$") + max_retention_days: int = Field(default=3_650, ge=1, le=36_500) + retention_expiring_horizon_days: int = Field(default=30, ge=1, le=365) @property def retry_delays(self) -> tuple[int, ...]: diff --git a/src/offboardproof/db.py b/src/offboardproof/db.py index 9a46bcf..6c0b1d0 100644 --- a/src/offboardproof/db.py +++ b/src/offboardproof/db.py @@ -8,7 +8,13 @@ from offboardproof.config import Settings -SCHEMA_VERSION = 1 +MIGRATIONS = { + 1: "0001_initial.sql", + 2: "0002_webhook_intake.sql", + 3: "0003_evidence_v2.sql", + 4: "0004_retention_operations.sql", +} +SCHEMA_VERSION = max(MIGRATIONS) def connect(path: Path) -> sqlite3.Connection: @@ -41,7 +47,7 @@ def migrate(settings: Settings) -> int: if current > SCHEMA_VERSION: raise RuntimeError(f"Database schema version {current} is newer than supported {SCHEMA_VERSION}") for version in range(current + 1, SCHEMA_VERSION + 1): - migration_path = files("offboardproof").joinpath("migrations", f"{version:04d}_initial.sql") + migration_path = files("offboardproof").joinpath("migrations", MIGRATIONS[version]) sql = migration_path.read_text(encoding="utf-8") connection.executescript(sql) connection.execute(f"PRAGMA user_version = {version}") diff --git a/src/offboardproof/errors.py b/src/offboardproof/errors.py index 687fcf3..e17c616 100644 --- a/src/offboardproof/errors.py +++ b/src/offboardproof/errors.py @@ -25,6 +25,22 @@ class ConfigurationError(OffboardProofError): pass +class WebhookAuthenticationError(OffboardProofError): + pass + + +class PayloadTooLargeError(OffboardProofError): + pass + + +class UnsupportedMediaTypeError(OffboardProofError): + pass + + +class WebhookValidationError(OffboardProofError): + pass + + class ProviderError(OffboardProofError): def __init__( self, diff --git a/src/offboardproof/evidence_signing.py b/src/offboardproof/evidence_signing.py new file mode 100644 index 0000000..cc55740 --- /dev/null +++ b/src/offboardproof/evidence_signing.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import base64 +import hashlib +import os +import stat +from dataclasses import dataclass +from pathlib import Path + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey + +from offboardproof.config import Settings +from offboardproof.errors import ConfigurationError + + +def b64url_encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=") + + +def b64url_decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def public_key_bytes(public_key: Ed25519PublicKey) -> bytes: + return public_key.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + + +def public_key_fingerprint(public_key: Ed25519PublicKey) -> str: + return hashlib.sha256(public_key_bytes(public_key)).hexdigest() + + +@dataclass(frozen=True) +class SigningKey: + key_id: str + private_key: Ed25519PrivateKey + + @property + def public_key(self) -> Ed25519PublicKey: + return self.private_key.public_key() + + +def _absolute_file(path: Path | None, label: str) -> Path: + if path is None or not path.is_absolute() or not path.is_file(): + raise ConfigurationError(f"{label} must be an absolute readable file") + return path + + +def load_signing_key(settings: Settings) -> SigningKey | None: + if settings.signing_key_file is None: + if settings.require_signed_evidence: + raise ConfigurationError("Signed evidence is required but no signing key file is configured") + return None + if not settings.signing_key_id: + raise ConfigurationError("OFFBOARDPROOF_SIGNING_KEY_ID is required when signing is enabled") + key_path = _absolute_file(settings.signing_key_file, "OFFBOARDPROOF_SIGNING_KEY_FILE") + if os.name != "nt" and stat.S_IMODE(key_path.stat().st_mode) & 0o077: + raise ConfigurationError("Signing key file permissions must not grant group or other access") + password: bytes | None = None + if settings.signing_key_password_file is not None: + password_path = _absolute_file( + settings.signing_key_password_file, + "OFFBOARDPROOF_SIGNING_KEY_PASSWORD_FILE", + ) + password = password_path.read_bytes().rstrip(b"\r\n") + if not password: + raise ConfigurationError("Signing key password file is empty") + key_bytes = key_path.read_bytes() + try: + if len(key_bytes) == 32 and password is None: + private_key = Ed25519PrivateKey.from_private_bytes(key_bytes) + else: + loaded = serialization.load_pem_private_key(key_bytes, password=password) + if not isinstance(loaded, Ed25519PrivateKey): + raise ConfigurationError("Signing key must be Ed25519") + private_key = loaded + except (TypeError, ValueError) as exc: + raise ConfigurationError("Signing key could not be loaded") from exc + return SigningKey(key_id=settings.signing_key_id, private_key=private_key) + + +def load_public_key(path: Path) -> Ed25519PublicKey: + if not path.is_file(): + raise ValueError("Trusted public key file does not exist") + value = path.read_bytes() + try: + if len(value) == 32: + return Ed25519PublicKey.from_public_bytes(value) + loaded = serialization.load_pem_public_key(value) + if not isinstance(loaded, Ed25519PublicKey): + raise ValueError("Trusted public key must be Ed25519") + return loaded + except (TypeError, ValueError) as exc: + raise ValueError("Trusted public key could not be loaded") from exc + + +def write_public_key(signing_key: SigningKey, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes( + signing_key.public_key.public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) diff --git a/src/offboardproof/evidence_v2.py b/src/offboardproof/evidence_v2.py new file mode 100644 index 0000000..9816ba1 --- /dev/null +++ b/src/offboardproof/evidence_v2.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +from offboardproof import __version__ +from offboardproof.audit import verify_chain +from offboardproof.config import Settings +from offboardproof.errors import ConflictError, NotFoundError +from offboardproof.evidence_signing import ( + b64url_decode, + b64url_encode, + load_signing_key, + public_key_bytes, + public_key_fingerprint, +) +from offboardproof.util import canonical_json, iso_now, new_id, parse_utc + +ASSURANCE_LIMIT = ( + "Evidence records configured provider observations and authorized human statements; " + "it is not an independent certification." +) + + +def _existing_signed_is_valid( + existing: sqlite3.Row, + evidence_path: Path, + manifest_path: Path, + digest: str, + case_id: str, + completion_hash: str, +) -> bool: + try: + if hashlib.sha256(evidence_path.read_bytes()).hexdigest() != digest: + return False + manifest = json.loads(manifest_path.read_bytes()) + statement = manifest["statement"] + public_bytes = b64url_decode(manifest["public_key"]["value"]) + fingerprint = hashlib.sha256(public_bytes).hexdigest() + if ( + manifest["signing_status"] != "signed" + or statement["case_id"] != case_id + or statement["evidence_sha256"] != digest + or statement["completion_event_hash"] != completion_hash + or statement["key_id"] != existing["signing_key_id"] + or fingerprint != existing["public_key_fingerprint"] + or fingerprint != manifest["public_key"]["fingerprint_sha256"] + ): + return False + Ed25519PublicKey.from_public_bytes(public_bytes).verify( + b64url_decode(manifest["signature"]["value"]), + canonical_json(statement).encode("utf-8"), + ) + return True + except (InvalidSignature, KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return False + + +def _timestamp(value: str | None) -> str | None: + if value is None: + return None + return parse_utc(value).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def _json_value(value: str | None) -> Any: + return None if value is None else json.loads(value) + + +def _select_rows( + connection: sqlite3.Connection, + query: str, + parameters: tuple[object, ...], + fields: tuple[str, ...], + json_fields: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + json_fields = json_fields or {} + result: list[dict[str, Any]] = [] + for row in connection.execute(query, parameters).fetchall(): + item = {field: row[field] for field in fields} + for source, destination in json_fields.items(): + item[destination] = _json_value(row[source]) + if destination != source: + item.pop(source, None) + result.append(item) + return result + + +def build_evidence_v2(connection: sqlite3.Connection, case_id: str) -> dict[str, Any]: + case = connection.execute("SELECT * FROM cases WHERE id=?", (case_id,)).fetchone() + if case is None: + raise NotFoundError(f"Case '{case_id}' was not found") + if case["state"] != "completed": + raise ConflictError("Evidence can be exported only after the case is completed") + completion = connection.execute( + """ + SELECT sequence_no, event_id, event_hash, created_at FROM audit_events + WHERE case_id=? AND event_type='case.completed' + ORDER BY sequence_no LIMIT 1 + """, + (case_id,), + ).fetchone() + if completion is None: + raise ConflictError("Completed case has no case.completed audit event") + chain_valid, _, broken_event = verify_chain(connection) + if not chain_valid: + raise ConflictError(f"Audit chain is invalid at event '{broken_event}'") + + case_fields = ( + "id", + "organization_id", + "idempotency_key", + "subject_email", + "subject_name", + "transfer_owner", + "effective_at", + "risk_tier", + "provider", + "state", + "plan_version", + "plan_digest", + "created_by", + "created_at", + "updated_at", + "completed_at", + "retention_until", + "retention_policy_id", + ) + case_payload = {field: case[field] for field in case_fields} + case_payload["required_roles"] = _json_value(case["required_roles_json"]) + for field in ("effective_at", "created_at", "updated_at", "completed_at", "retention_until"): + case_payload[field] = _timestamp(case_payload[field]) + + controls = _select_rows( + connection, + "SELECT * FROM controls WHERE case_id=? ORDER BY sequence_no, id", + (case_id,), + ( + "id", + "control_type", + "provider", + "target", + "desired_json", + "state", + "required", + "manual", + "sequence_no", + "evidence_quality", + "created_at", + "updated_at", + ), + {"desired_json": "desired"}, + ) + approvals = _select_rows( + connection, + "SELECT * FROM approvals WHERE case_id=? ORDER BY created_at, id", + (case_id,), + ("id", "actor_id", "role", "decision", "reason", "plan_digest", "created_at"), + ) + actions = _select_rows( + connection, + "SELECT * FROM actions WHERE case_id=? ORDER BY created_at, id", + (case_id,), + ( + "id", + "control_id", + "idempotency_key", + "operation", + "payload_json", + "status", + "attempt_count", + "external_reference", + "outcome_kind", + "created_at", + "updated_at", + ), + {"payload_json": "payload"}, + ) + exceptions = _select_rows( + connection, + "SELECT * FROM exceptions WHERE case_id=? ORDER BY created_at, id", + (case_id,), + ( + "id", + "control_id", + "action_id", + "category", + "retryable", + "status", + "owner_actor_id", + "due_at", + "summary", + "resolution_note", + "created_at", + "updated_at", + ), + ) + observations = _select_rows( + connection, + "SELECT * FROM observations WHERE case_id=? ORDER BY created_at, id", + (case_id,), + ("id", "control_id", "provider", "external_id", "observed_json", "satisfied", "quality", "created_at"), + {"observed_json": "observed"}, + ) + audit_events = _select_rows( + connection, + """ + SELECT * FROM audit_events WHERE case_id=? AND sequence_no<=? + ORDER BY sequence_no + """, + (case_id, completion["sequence_no"]), + ( + "sequence_no", + "event_id", + "case_id", + "actor_id", + "event_type", + "payload_json", + "previous_hash", + "event_hash", + "created_at", + ), + {"payload_json": "payload"}, + ) + for collection in (controls, approvals, actions, exceptions, observations, audit_events): + for item in collection: + for key in ("created_at", "updated_at", "due_at"): + if key in item: + item[key] = _timestamp(item[key]) + + assurance_counts = {"observed": 0, "acknowledged": 0, "human_attested": 0, "waived": 0} + for control in controls: + if control["state"] == "waived": + assurance_counts["waived"] += 1 + elif control["evidence_quality"] in assurance_counts: + assurance_counts[str(control["evidence_quality"])] += 1 + + return { + "schema_version": 2, + "application": {"name": "offboardproof", "version": __version__}, + "case": case_payload, + "controls": controls, + "approvals": approvals, + "actions": actions, + "exceptions": exceptions, + "observations": observations, + "audit_events": audit_events, + "completion": { + "sequence_no": completion["sequence_no"], + "event_id": completion["event_id"], + "event_hash": completion["event_hash"], + "completed_at": _timestamp(completion["created_at"]), + }, + "retention": { + "retention_until": _timestamp(case["retention_until"]), + "policy_snapshot": case["retention_policy_id"], + }, + "assurance": { + "observed_controls": assurance_counts["observed"], + "acknowledged_controls": assurance_counts["acknowledged"], + "human_attested_controls": assurance_counts["human_attested"], + "waived_controls": assurance_counts["waived"], + "limit": ASSURANCE_LIMIT, + }, + } + + +def evidence_bytes(payload: dict[str, Any]) -> bytes: + return (canonical_json(payload) + "\n").encode("utf-8") + + +def write_evidence_v2( + connection: sqlite3.Connection, + settings: Settings, + case_id: str, +) -> tuple[Path, Path, str]: + payload = build_evidence_v2(connection, case_id) + rendered = evidence_bytes(payload) + digest = hashlib.sha256(rendered).hexdigest() + completion_hash = str(payload["completion"]["event_hash"]) + settings.evidence_dir.mkdir(parents=True, exist_ok=True) + evidence_path = settings.evidence_dir / f"{case_id}.evidence.v2.json" + manifest_path = settings.evidence_dir / f"{case_id}.evidence.v2.manifest.json" + existing = connection.execute( + """ + SELECT * FROM evidence_artifacts + WHERE case_id=? AND schema_version=2 AND evidence_sha256=? + ORDER BY created_at LIMIT 1 + """, + (case_id, digest), + ).fetchone() + if ( + existing is not None + and existing["signing_status"] == "signed" + and evidence_path.is_file() + and manifest_path.is_file() + and _existing_signed_is_valid(existing, evidence_path, manifest_path, digest, case_id, completion_hash) + ): + return evidence_path, manifest_path, digest + signing_key = load_signing_key(settings) + statement = { + "domain": "offboardproof.evidence.signature.v1", + "case_id": case_id, + "evidence_schema_version": 2, + "evidence_sha256": digest, + "completion_event_hash": completion_hash, + "key_id": signing_key.key_id if signing_key else "unsigned", + } + if signing_key is None: + manifest: dict[str, Any] = { + "manifest_schema_version": 1, + "statement": statement, + "signing_status": "unsigned", + "reason_code": "signing_not_configured", + } + signing_status = "unsigned" + algorithm = key_id = fingerprint = None + else: + statement_bytes = canonical_json(statement).encode("utf-8") + signature = signing_key.private_key.sign(statement_bytes) + fingerprint = public_key_fingerprint(signing_key.public_key) + manifest = { + "manifest_schema_version": 1, + "statement": statement, + "signature": { + "algorithm": "Ed25519", + "encoding": "base64url-no-padding", + "value": b64url_encode(signature), + }, + "public_key": { + "encoding": "raw-base64url-no-padding", + "fingerprint_sha256": fingerprint, + "value": b64url_encode(public_key_bytes(signing_key.public_key)), + }, + "signing_status": "signed", + } + signing_status = "signed" + algorithm = "Ed25519" + key_id = signing_key.key_id + + evidence_tmp = evidence_path.with_suffix(evidence_path.suffix + ".tmp") + manifest_tmp = manifest_path.with_suffix(manifest_path.suffix + ".tmp") + evidence_tmp.write_bytes(rendered) + manifest_tmp.write_bytes(evidence_bytes(manifest)) + os.replace(evidence_tmp, evidence_path) + os.replace(manifest_tmp, manifest_path) + + if existing is None: + connection.execute( + """ + INSERT INTO evidence_artifacts ( + id, case_id, path, sha256, mime_type, created_at, schema_version, + manifest_path, evidence_sha256, signing_status, signing_algorithm, + signing_key_id, public_key_fingerprint, completion_event_hash + ) VALUES (?, ?, ?, ?, 'application/json', ?, 2, ?, ?, ?, ?, ?, ?, ?) + """, + ( + new_id(), + case_id, + str(evidence_path), + digest, + iso_now(), + str(manifest_path), + digest, + signing_status, + algorithm, + key_id, + fingerprint, + completion_hash, + ), + ) + else: + connection.execute( + """ + UPDATE evidence_artifacts + SET manifest_path=?, sha256=?, signing_status=?, signing_algorithm=?, + signing_key_id=?, public_key_fingerprint=?, completion_event_hash=? + WHERE id=? + """, + ( + str(manifest_path), + digest, + signing_status, + algorithm, + key_id, + fingerprint, + completion_hash, + existing["id"], + ), + ) + return evidence_path, manifest_path, digest diff --git a/src/offboardproof/evidence_verification.py b/src/offboardproof/evidence_verification.py new file mode 100644 index 0000000..86a46a2 --- /dev/null +++ b/src/offboardproof/evidence_verification.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature + +from offboardproof.evidence_signing import b64url_decode, load_public_key, public_key_fingerprint +from offboardproof.util import canonical_json + + +@dataclass(frozen=True) +class EvidenceVerificationError(Exception): + exit_code: int + result: dict[str, Any] + + +def _fail(code: int, reason: str) -> EvidenceVerificationError: + return EvidenceVerificationError(code, {"valid": False, "reason": reason}) + + +def verify_evidence_v2(evidence_path: Path, manifest_path: Path, trusted_public_key: Path) -> dict[str, Any]: + try: + evidence_bytes = evidence_path.read_bytes() + manifest_bytes = manifest_path.read_bytes() + if len(evidence_bytes) > 10 * 1024 * 1024 or len(manifest_bytes) > 1024 * 1024: + raise _fail(2, "input_too_large") + evidence = json.loads(evidence_bytes) + manifest = json.loads(manifest_bytes) + except EvidenceVerificationError: + raise + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise _fail(2, "malformed_input") from exc + if evidence.get("schema_version") != 2 or manifest.get("manifest_schema_version") != 1: + raise _fail(2, "unsupported_schema") + if manifest.get("signing_status") == "unsigned": + raise _fail(6, "unsigned_evidence") + try: + statement = manifest["statement"] + signature_info = manifest["signature"] + public_info = manifest["public_key"] + if statement["domain"] != "offboardproof.evidence.signature.v1": + raise _fail(2, "unsupported_signing_domain") + if signature_info["algorithm"] != "Ed25519": + raise _fail(2, "unsupported_algorithm") + digest = hashlib.sha256(evidence_bytes).hexdigest() + if digest != statement["evidence_sha256"]: + raise _fail(3, "evidence_digest_mismatch") + if statement["case_id"] != evidence["case"]["id"]: + raise _fail(3, "case_id_mismatch") + if statement["completion_event_hash"] != evidence["completion"]["event_hash"]: + raise _fail(3, "completion_hash_mismatch") + trusted = load_public_key(trusted_public_key) + trusted_fingerprint = public_key_fingerprint(trusted) + embedded_bytes = b64url_decode(public_info["value"]) + if len(embedded_bytes) != 32 or hashlib.sha256(embedded_bytes).hexdigest() != public_info["fingerprint_sha256"]: + raise _fail(5, "embedded_key_mismatch") + if trusted_fingerprint != public_info["fingerprint_sha256"]: + raise _fail(5, "trust_anchor_mismatch") + statement_bytes = canonical_json(statement).encode("utf-8") + trusted.verify(b64url_decode(signature_info["value"]), statement_bytes) + except EvidenceVerificationError: + raise + except InvalidSignature as exc: + raise _fail(4, "signature_invalid") from exc + except (KeyError, TypeError, ValueError) as exc: + raise _fail(2, "malformed_manifest") from exc + assurance = evidence.get("assurance", {}) + assurance_kind = ( + "observed_only" + if assurance.get("acknowledged_controls", 0) == 0 + and assurance.get("human_attested_controls", 0) == 0 + and assurance.get("waived_controls", 0) == 0 + else "mixed_observed_and_acknowledged" + ) + return { + "valid": True, + "integrity": "verified", + "origin": "trusted_key_match", + "assurance": assurance_kind, + "case_id": statement["case_id"], + "key_id": statement["key_id"], + "evidence_sha256": statement["evidence_sha256"], + } diff --git a/src/offboardproof/migrations/0002_webhook_intake.sql b/src/offboardproof/migrations/0002_webhook_intake.sql new file mode 100644 index 0000000..843d2b0 --- /dev/null +++ b/src/offboardproof/migrations/0002_webhook_intake.sql @@ -0,0 +1,18 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE webhook_deliveries ( + id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + delivery_id TEXT NOT NULL, + event_id TEXT NOT NULL, + payload_sha256 TEXT NOT NULL CHECK (length(payload_sha256) = 64), + case_id TEXT REFERENCES cases(id) ON DELETE SET NULL, + status TEXT NOT NULL CHECK (status IN ('accepted','rejected','conflict')), + received_at TEXT NOT NULL, + completed_at TEXT, + UNIQUE (source_id, delivery_id), + UNIQUE (source_id, event_id) +); + +CREATE INDEX idx_webhook_deliveries_case ON webhook_deliveries(case_id); +CREATE INDEX idx_webhook_deliveries_status_received ON webhook_deliveries(status, received_at); diff --git a/src/offboardproof/migrations/0003_evidence_v2.sql b/src/offboardproof/migrations/0003_evidence_v2.sql new file mode 100644 index 0000000..8baf151 --- /dev/null +++ b/src/offboardproof/migrations/0003_evidence_v2.sql @@ -0,0 +1,13 @@ +PRAGMA foreign_keys = ON; + +ALTER TABLE evidence_artifacts ADD COLUMN schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version IN (1,2)); +ALTER TABLE evidence_artifacts ADD COLUMN manifest_path TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN evidence_sha256 TEXT CHECK (evidence_sha256 IS NULL OR length(evidence_sha256) = 64); +ALTER TABLE evidence_artifacts ADD COLUMN signing_status TEXT NOT NULL DEFAULT 'legacy' CHECK (signing_status IN ('legacy','unsigned','signed','failed')); +ALTER TABLE evidence_artifacts ADD COLUMN signing_algorithm TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN signing_key_id TEXT; +ALTER TABLE evidence_artifacts ADD COLUMN public_key_fingerprint TEXT CHECK (public_key_fingerprint IS NULL OR length(public_key_fingerprint) = 64); +ALTER TABLE evidence_artifacts ADD COLUMN completion_event_hash TEXT CHECK (completion_event_hash IS NULL OR length(completion_event_hash) = 64); + +CREATE INDEX idx_evidence_artifacts_case_schema ON evidence_artifacts(case_id, schema_version); +CREATE INDEX idx_evidence_artifacts_signing_status ON evidence_artifacts(signing_status, created_at); diff --git a/src/offboardproof/migrations/0004_retention_operations.sql b/src/offboardproof/migrations/0004_retention_operations.sql new file mode 100644 index 0000000..ca6709c --- /dev/null +++ b/src/offboardproof/migrations/0004_retention_operations.sql @@ -0,0 +1,26 @@ +PRAGMA foreign_keys = ON; + +ALTER TABLE cases ADD COLUMN retention_until TEXT; +ALTER TABLE cases ADD COLUMN retention_policy_id TEXT; + +CREATE INDEX idx_cases_retention_until ON cases(retention_until); + +CREATE TABLE legal_holds ( + id TEXT PRIMARY KEY, + case_id TEXT NOT NULL REFERENCES cases(id) ON DELETE CASCADE, + status TEXT NOT NULL CHECK (status IN ('active','released')), + reason TEXT NOT NULL CHECK (length(trim(reason)) > 0), + created_by TEXT NOT NULL REFERENCES actors(id), + created_at TEXT NOT NULL, + released_by TEXT REFERENCES actors(id), + released_at TEXT, + release_reason TEXT, + CHECK ( + (status = 'active' AND released_by IS NULL AND released_at IS NULL AND release_reason IS NULL) + OR + (status = 'released' AND released_by IS NOT NULL AND released_at IS NOT NULL AND length(trim(release_reason)) > 0) + ) +); + +CREATE UNIQUE INDEX idx_legal_holds_one_active_per_case ON legal_holds(case_id) WHERE status = 'active'; +CREATE INDEX idx_legal_holds_case_status ON legal_holds(case_id, status); diff --git a/src/offboardproof/retention.py b/src/offboardproof/retention.py new file mode 100644 index 0000000..bde698d --- /dev/null +++ b/src/offboardproof/retention.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import sqlite3 +from datetime import UTC, datetime +from typing import Any + +from offboardproof.audit import append_event +from offboardproof.auth import Actor, require_role +from offboardproof.config import Settings +from offboardproof.db import connection_for, transaction +from offboardproof.enums import Role +from offboardproof.errors import ConflictError, NotFoundError +from offboardproof.util import canonical_json, iso_now, new_id, sha256_text + +CASE_COUNT_QUERIES = { + "controls": "SELECT COUNT(*) FROM controls WHERE case_id=?", + "approvals": "SELECT COUNT(*) FROM approvals WHERE case_id=?", + "actions": "SELECT COUNT(*) FROM actions WHERE case_id=?", + "jobs": "SELECT COUNT(*) FROM jobs WHERE case_id=?", + "exceptions": "SELECT COUNT(*) FROM exceptions WHERE case_id=?", + "observations": "SELECT COUNT(*) FROM observations WHERE case_id=?", + "evidence_artifacts": "SELECT COUNT(*) FROM evidence_artifacts WHERE case_id=?", + "audit_events": "SELECT COUNT(*) FROM audit_events WHERE case_id=?", + "legal_holds": "SELECT COUNT(*) FROM legal_holds WHERE case_id=?", +} + + +class RetentionService: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + @staticmethod + def _case_exists(connection: sqlite3.Connection, case_id: str) -> None: + if connection.execute("SELECT 1 FROM cases WHERE id=?", (case_id,)).fetchone() is None: + raise NotFoundError(f"Case '{case_id}' was not found") + + def create_hold(self, actor: Actor, case_id: str, reason: str) -> dict[str, Any]: + require_role(actor, Role.SECURITY) + reason = reason.strip() + if len(reason) < 10: + raise ValueError("Legal-hold reason must be at least 10 characters") + connection = connection_for(self.settings) + try: + with transaction(connection): + self._case_exists(connection, case_id) + hold_id = new_id() + now = iso_now() + try: + connection.execute( + """ + INSERT INTO legal_holds (id, case_id, status, reason, created_by, created_at) + VALUES (?, ?, 'active', ?, ?, ?) + """, + (hold_id, case_id, reason, actor.id, now), + ) + except sqlite3.IntegrityError as exc: + raise ConflictError("Case already has an active legal hold") from exc + append_event( + connection, + case_id=case_id, + actor_id=actor.id, + event_type="legal_hold.created", + payload={"legal_hold_id": hold_id, "reason": reason}, + ) + return self.hold_view(hold_id) + finally: + connection.close() + + def release_hold(self, actor: Actor, case_id: str, reason: str) -> dict[str, Any]: + require_role(actor, Role.SECURITY) + reason = reason.strip() + if len(reason) < 10: + raise ValueError("Legal-hold release reason must be at least 10 characters") + connection = connection_for(self.settings) + try: + with transaction(connection): + hold = connection.execute( + "SELECT * FROM legal_holds WHERE case_id=? AND status='active'", + (case_id,), + ).fetchone() + if hold is None: + raise NotFoundError("Case has no active legal hold") + now = iso_now() + connection.execute( + """ + UPDATE legal_holds SET status='released', released_by=?, released_at=?, release_reason=? + WHERE id=? + """, + (actor.id, now, reason, hold["id"]), + ) + append_event( + connection, + case_id=case_id, + actor_id=actor.id, + event_type="legal_hold.released", + payload={"legal_hold_id": hold["id"], "release_reason": reason}, + ) + return self.hold_view(str(hold["id"])) + finally: + connection.close() + + def hold_view(self, hold_id: str) -> dict[str, Any]: + connection = connection_for(self.settings) + try: + row = connection.execute("SELECT * FROM legal_holds WHERE id=?", (hold_id,)).fetchone() + if row is None: + raise NotFoundError("Legal hold was not found") + return {key: row[key] for key in row.keys()} + finally: + connection.close() + + def report(self, actor: Actor, as_of: datetime) -> dict[str, Any]: + require_role(actor, Role.OPERATOR, Role.SECURITY, Role.AUDITOR) + as_of = as_of.astimezone(UTC) + connection = connection_for(self.settings) + try: + cases: list[dict[str, Any]] = [] + for case in connection.execute("SELECT * FROM cases ORDER BY id").fetchall(): + active_hold = connection.execute( + "SELECT 1 FROM legal_holds WHERE case_id=? AND status='active'", + (case["id"],), + ).fetchone() + retention_until = ( + datetime.fromisoformat(case["retention_until"]).astimezone(UTC) if case["retention_until"] else None + ) + if retention_until is None: + eligibility = "missing_retention_policy" + elif case["state"] != "completed": + eligibility = "incomplete_case" + elif active_hold: + eligibility = "active_legal_hold" + elif retention_until > as_of: + eligibility = "not_expired" + else: + eligibility = "expired_review_required" + counts = { + table: int(connection.execute(query, (case["id"],)).fetchone()[0]) + for table, query in CASE_COUNT_QUERIES.items() + } + artifacts = [ + { + "path": row["path"], + "sha256": row["sha256"], + "manifest_path": row["manifest_path"], + "signing_status": row["signing_status"], + } + for row in connection.execute( + """ + SELECT path, sha256, manifest_path, signing_status + FROM evidence_artifacts WHERE case_id=? ORDER BY path + """, + (case["id"],), + ).fetchall() + ] + cases.append( + { + "case_id": case["id"], + "case_state": case["state"], + "retention_until": case["retention_until"], + "retention_policy_id": case["retention_policy_id"], + "active_legal_hold": bool(active_hold), + "eligibility": eligibility, + "row_counts": counts, + "evidence_artifacts": artifacts, + } + ) + stable = { + "schema_version": 1, + "database_schema_version": int(connection.execute("PRAGMA user_version").fetchone()[0]), + "as_of": as_of.isoformat(), + "case_count": len(cases), + "cases": cases, + } + return {**stable, "generated_at": iso_now(), "report_sha256": sha256_text(canonical_json(stable))} + finally: + connection.close() diff --git a/src/offboardproof/schemas.py b/src/offboardproof/schemas.py index bcf327d..d3dc317 100644 --- a/src/offboardproof/schemas.py +++ b/src/offboardproof/schemas.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import UTC, datetime from typing import Any from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -20,12 +20,23 @@ class CaseCreate(StrictModel): effective_at: datetime risk_tier: RiskTier = RiskTier.STANDARD provider: str = Field(default="mock", pattern=r"^[a-z][a-z0-9_-]{1,31}$") + retention_until: datetime | None = None + retention_policy_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._~-]{1,128}$") @field_validator("subject_email", "transfer_owner") @classmethod def validate_email(cls, value: str) -> str: return normalize_email(value) + @field_validator("retention_until") + @classmethod + def validate_retention_until(cls, value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("retention_until must include an explicit UTC offset") + return value.astimezone(UTC) + class ApprovalCreate(StrictModel): decision: ApprovalDecision @@ -46,6 +57,14 @@ class RequeueCreate(StrictModel): resolution_note: str = Field(min_length=5, max_length=1000) +class LegalHoldCreate(StrictModel): + reason: str = Field(min_length=10, max_length=1000) + + +class LegalHoldRelease(StrictModel): + reason: str = Field(min_length=10, max_length=1000) + + class ActorView(StrictModel): id: str name: str diff --git a/src/offboardproof/service.py b/src/offboardproof/service.py index d03b97f..1c19814 100644 --- a/src/offboardproof/service.py +++ b/src/offboardproof/service.py @@ -2,7 +2,7 @@ import json import sqlite3 -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from offboardproof.audit import append_event @@ -19,8 +19,9 @@ RiskTier, Role, ) -from offboardproof.errors import ConflictError, InvalidTransitionError, NotFoundError +from offboardproof.errors import ConfigurationError, ConflictError, InvalidTransitionError, NotFoundError from offboardproof.evidence import write_evidence +from offboardproof.evidence_v2 import write_evidence_v2 from offboardproof.providers.factory import get_provider from offboardproof.schemas import CaseCreate from offboardproof.util import canonical_json, iso_now, new_id, normalize_email, sha256_text @@ -51,53 +52,92 @@ def create_case(self, actor: Actor, request: CaseCreate, idempotency_key: str) - raise ValueError("Idempotency-Key must be between 8 and 128 characters") connection = connection_for(self.settings) try: - existing = connection.execute( - "SELECT id FROM cases WHERE organization_id='local' AND idempotency_key=?", - (key,), - ).fetchone() - if existing: - return self.case_view(existing["id"], connection=connection) - now = iso_now() - case_id = new_id() with transaction(connection): - connection.execute( - """ - INSERT INTO cases - (id, organization_id, idempotency_key, subject_email, subject_name, - transfer_owner, effective_at, risk_tier, provider, state, created_by, - created_at, updated_at) - VALUES (?, 'local', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - case_id, - key, - normalize_email(request.subject_email), - request.subject_name.strip(), - normalize_email(request.transfer_owner), - request.effective_at.astimezone(UTC).isoformat(), - request.risk_tier.value, - request.provider, - CaseState.RECEIVED.value, - actor.id, - now, - now, - ), - ) - append_event( - connection, - case_id=case_id, - actor_id=actor.id, - event_type="case.received", - payload={ - "idempotency_key": key, - "provider": request.provider, - "risk_tier": request.risk_tier.value, - }, - ) + case_id = self.create_case_in_transaction(connection, actor, request, key) return self.case_view(case_id, connection=connection) finally: connection.close() + def create_case_in_transaction( + self, + connection: sqlite3.Connection, + actor: Actor, + request: CaseCreate, + idempotency_key: str, + ) -> str: + """Create or find a case inside the caller's transaction boundary.""" + require_role(actor, Role.HR, Role.OPERATOR, Role.SERVICE) + key = idempotency_key.strip() + if len(key) < 8 or len(key) > 300: + raise ValueError("Internal idempotency key must be between 8 and 300 characters") + existing = connection.execute( + "SELECT id FROM cases WHERE organization_id='local' AND idempotency_key=?", + (key,), + ).fetchone() + if existing: + return str(existing["id"]) + + now = iso_now() + created_at = datetime.fromisoformat(now) + if request.retention_until is None: + retention_until = created_at + timedelta(days=self.settings.default_retention_days) + retention_policy_id = self.settings.default_retention_policy_id + else: + retention_until = request.retention_until.astimezone(UTC) + if retention_until <= created_at: + raise ValueError("retention_until must be in the future") + if retention_until > created_at + timedelta(days=self.settings.max_retention_days): + raise ValueError("retention_until exceeds the configured maximum") + retention_policy_id = request.retention_policy_id or "explicit" + case_id = new_id() + connection.execute( + """ + INSERT INTO cases + (id, organization_id, idempotency_key, subject_email, subject_name, + transfer_owner, effective_at, risk_tier, provider, state, created_by, + created_at, updated_at, retention_until, retention_policy_id) + VALUES (?, 'local', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + case_id, + key, + normalize_email(request.subject_email), + request.subject_name.strip(), + normalize_email(request.transfer_owner), + request.effective_at.astimezone(UTC).isoformat(), + request.risk_tier.value, + request.provider, + CaseState.RECEIVED.value, + actor.id, + now, + now, + retention_until.isoformat(), + retention_policy_id, + ), + ) + append_event( + connection, + case_id=case_id, + actor_id=actor.id, + event_type="case.received", + payload={ + "idempotency_key": key, + "provider": request.provider, + "risk_tier": request.risk_tier.value, + }, + ) + append_event( + connection, + case_id=case_id, + actor_id=actor.id, + event_type="retention.snapshot_created", + payload={ + "retention_until": retention_until.isoformat(), + "retention_policy_id": retention_policy_id, + }, + ) + return case_id + def plan_case(self, actor: Actor, case_id: str) -> dict[str, Any]: require_role(actor, Role.OPERATOR) connection = connection_for(self.settings) @@ -375,51 +415,47 @@ def complete_manual(self, actor: Actor, case_id: str, control_id: str, evidence_ if control is None or not bool(control["manual"]): raise NotFoundError("Manual control was not found") now = iso_now() - with transaction(connection): - connection.execute( - "UPDATE controls SET state=?, updated_at=? WHERE id=?", - (ControlState.VERIFIED.value, now, control_id), - ) - connection.execute( - """ - INSERT INTO observations - (id, case_id, control_id, provider, external_id, observed_json, - satisfied, quality, created_at) - VALUES (?, ?, ?, 'manual', NULL, ?, 1, 'human_attested', ?) - """, - ( - new_id(), - case_id, - control_id, - canonical_json({"evidence_note": evidence_note.strip()}), - now, - ), - ) - connection.execute( - """ - UPDATE exceptions SET status='resolved', resolution_note=?, updated_at=? - WHERE control_id=? AND status='open' - """, - (evidence_note.strip(), now, control_id), - ) - append_event( - connection, - case_id=case_id, - actor_id=actor.id, - event_type="control.manual_completed", - payload={"control_id": control_id, "evidence_note": evidence_note.strip()}, - ) - new_state = self._refresh_case_state(connection, case_id, actor.id) - if new_state is CaseState.COMPLETED: - _, digest = write_evidence(connection, self.settings, case_id) + try: with transaction(connection): + connection.execute( + "UPDATE controls SET state=?, updated_at=? WHERE id=?", + (ControlState.VERIFIED.value, now, control_id), + ) + connection.execute( + """ + INSERT INTO observations + (id, case_id, control_id, provider, external_id, observed_json, + satisfied, quality, created_at) + VALUES (?, ?, ?, 'manual', NULL, ?, 1, 'human_attested', ?) + """, + ( + new_id(), + case_id, + control_id, + canonical_json({"evidence_note": evidence_note.strip()}), + now, + ), + ) + connection.execute( + """ + UPDATE exceptions SET status='resolved', resolution_note=?, updated_at=? + WHERE control_id=? AND status='open' + """, + (evidence_note.strip(), now, control_id), + ) append_event( connection, case_id=case_id, actor_id=actor.id, - event_type="evidence.generated", - payload={"sha256": digest}, + event_type="control.manual_completed", + payload={"control_id": control_id, "evidence_note": evidence_note.strip()}, ) + new_state = self._refresh_case_state(connection, case_id, actor.id) + if new_state is CaseState.COMPLETED: + self._write_completed_evidence(connection, case_id, actor.id) + except ConfigurationError: + self._record_evidence_failure(connection, case_id, control_id, actor.id) + raise return self.case_view(case_id, connection=connection) finally: connection.close() @@ -472,16 +508,8 @@ def waive_control( }, ) new_state = self._refresh_case_state(connection, case_id, actor.id) - if new_state is CaseState.COMPLETED: - _, digest = write_evidence(connection, self.settings, case_id) - with transaction(connection): - append_event( - connection, - case_id=case_id, - actor_id=actor.id, - event_type="evidence.generated", - payload={"sha256": digest}, - ) + if new_state is CaseState.COMPLETED: + self._write_completed_evidence(connection, case_id, actor.id) return self.case_view(case_id, connection=connection) finally: connection.close() @@ -587,6 +615,87 @@ def _refresh_case_state(self, connection: sqlite3.Connection, case_id: str, acto ) return new_state + def _write_completed_evidence( + self, + connection: sqlite3.Connection, + case_id: str, + actor_id: str | None, + ) -> str: + evidence_path, manifest_path, digest = write_evidence_v2(connection, self.settings, case_id) + write_evidence(connection, self.settings, case_id) + artifact = connection.execute( + """ + SELECT signing_status, signing_key_id, public_key_fingerprint + FROM evidence_artifacts + WHERE case_id=? AND schema_version=2 AND evidence_sha256=? + """, + (case_id, digest), + ).fetchone() + append_event( + connection, + case_id=case_id, + actor_id=actor_id, + event_type="evidence.generated", + payload={ + "schema_version": 2, + "evidence_sha256": digest, + "evidence_path": str(evidence_path), + "manifest_path": str(manifest_path), + "signing_status": artifact["signing_status"], + "signing_key_id": artifact["signing_key_id"], + "public_key_fingerprint": artifact["public_key_fingerprint"], + }, + ) + return digest + + def _record_evidence_failure( + self, + connection: sqlite3.Connection, + case_id: str, + control_id: str, + actor_id: str | None, + ) -> None: + now = iso_now() + with transaction(connection): + existing = connection.execute( + """ + SELECT id FROM exceptions + WHERE case_id=? AND control_id=? AND category='evidence_signing' AND status='open' + """, + (case_id, control_id), + ).fetchone() + if existing is None: + exception_id = new_id() + connection.execute( + """ + INSERT INTO exceptions ( + id, case_id, control_id, action_id, category, retryable, status, + summary, created_at, updated_at + ) VALUES (?, ?, ?, NULL, 'evidence_signing', 1, 'open', ?, ?, ?) + """, + ( + exception_id, + case_id, + control_id, + "Evidence signing configuration prevented completion", + now, + now, + ), + ) + else: + exception_id = str(existing["id"]) + connection.execute( + "UPDATE cases SET state='exception', updated_at=?, completed_at=NULL WHERE id=?", + (now, case_id), + ) + append_event( + connection, + case_id=case_id, + actor_id=actor_id, + event_type="evidence.failed", + payload={"exception_id": exception_id, "reason_code": "signing_configuration"}, + ) + def case_view(self, case_id: str, *, connection: sqlite3.Connection | None = None) -> dict[str, Any]: own_connection = connection is None connection = connection or connection_for(self.settings) @@ -617,6 +726,13 @@ def case_view(self, case_id: str, *, connection: sqlite3.Connection | None = Non "SELECT * FROM exceptions WHERE case_id=? ORDER BY created_at", (case_id,) ).fetchall() ] + payload["legal_holds"] = [ + {key: row[key] for key in row.keys()} + for row in connection.execute( + "SELECT * FROM legal_holds WHERE case_id=? ORDER BY created_at", + (case_id,), + ).fetchall() + ] return payload finally: if own_connection: @@ -643,10 +759,10 @@ def audit_events(self, case_id: str) -> list[dict[str, Any]]: finally: connection.close() - def metrics(self) -> dict[str, int]: + def metrics(self) -> dict[str, int | float]: connection = connection_for(self.settings) try: - result: dict[str, int] = {} + result: dict[str, int | float] = {} for row in connection.execute("SELECT state, COUNT(*) AS count FROM cases GROUP BY state").fetchall(): result[f"cases_{row['state']}"] = int(row["count"]) result["exceptions_open"] = int( @@ -655,6 +771,64 @@ def metrics(self) -> dict[str, int]: result["jobs_queued"] = int( connection.execute("SELECT COUNT(*) FROM jobs WHERE status='queued'").fetchone()[0] ) + for row in connection.execute("SELECT status, COUNT(*) AS count FROM jobs GROUP BY status").fetchall(): + result[f"jobs_{row['status']}"] = int(row["count"]) + for row in connection.execute( + "SELECT evidence_quality, state, COUNT(*) AS count FROM controls GROUP BY evidence_quality, state" + ).fetchall(): + result[f"controls_{row['evidence_quality']}_{row['state']}"] = int(row["count"]) + result["exceptions_overdue"] = int( + connection.execute( + "SELECT COUNT(*) FROM exceptions WHERE status='open' AND due_at IS NOT NULL AND due_at? AND retention_until<=?", + (now.isoformat(), horizon.isoformat()), + ).fetchone()[0] + ) + result["cases_retention_expired"] = int( + connection.execute( + "SELECT COUNT(*) FROM cases WHERE retention_until IS NOT NULL AND retention_until<=?", + (now.isoformat(),), + ).fetchone()[0] + ) + result["webhook_deliveries_accepted"] = int( + connection.execute("SELECT COUNT(*) FROM webhook_deliveries WHERE status='accepted'").fetchone()[0] + ) + result["webhook_replays_suppressed"] = int( + connection.execute("SELECT COUNT(*) FROM audit_events WHERE event_type='webhook.replayed'").fetchone()[ + 0 + ] + ) + result["webhook_conflicts"] = int( + connection.execute("SELECT COUNT(*) FROM audit_events WHERE event_type='webhook.conflict'").fetchone()[ + 0 + ] + ) + result["webhook_authenticated_rejections"] = int( + connection.execute("SELECT COUNT(*) FROM audit_events WHERE event_type='webhook.rejected'").fetchone()[ + 0 + ] + ) + oldest = connection.execute("SELECT MIN(available_at) FROM jobs WHERE status='queued'").fetchone()[0] + result["oldest_queued_job_age_seconds"] = ( + max(0.0, (now - datetime.fromisoformat(oldest)).total_seconds()) if oldest else 0.0 + ) return result finally: connection.close() diff --git a/src/offboardproof/webhooks/__init__.py b/src/offboardproof/webhooks/__init__.py new file mode 100644 index 0000000..8531953 --- /dev/null +++ b/src/offboardproof/webhooks/__init__.py @@ -0,0 +1 @@ +"""Authenticated, replay-safe webhook intake.""" diff --git a/src/offboardproof/webhooks/config.py b/src/offboardproof/webhooks/config.py new file mode 100644 index 0000000..f0ea8a9 --- /dev/null +++ b/src/offboardproof/webhooks/config.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from offboardproof.config import Settings +from offboardproof.errors import WebhookAuthenticationError +from offboardproof.util import parse_utc + +GENERIC_AUTH_MESSAGE = "Webhook authentication failed" + + +@dataclass(frozen=True) +class WebhookCredential: + actor_id: str + secret: bytes + + +def _fail() -> WebhookAuthenticationError: + return WebhookAuthenticationError(GENERIC_AUTH_MESSAGE) + + +def _load_secret(path_value: object) -> bytes: + if not isinstance(path_value, str): + raise _fail() + path = Path(path_value) + if not path.is_absolute() or not path.is_file(): + raise _fail() + secret = path.read_bytes() + if len(secret) < 32: + raise _fail() + return secret + + +def load_credential( + settings: Settings, + *, + source_id: str, + key_id: str, + now: datetime, +) -> WebhookCredential: + config_path = settings.webhook_config_file + if config_path is None or not config_path.is_absolute() or not config_path.is_file(): + raise _fail() + try: + document: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8")) + if document.get("schema_version") != 1: + raise _fail() + source = document["sources"][source_id] + if not isinstance(source, dict) or source.get("active") is not True: + raise _fail() + actor_id = source["actor_id"] + if not isinstance(actor_id, str) or not actor_id: + raise _fail() + + current = source.get("current_key") + if isinstance(current, dict) and current.get("key_id") == key_id: + return WebhookCredential(actor_id=actor_id, secret=_load_secret(current.get("secret_file"))) + + previous = source.get("previous_key") + if isinstance(previous, dict) and previous.get("key_id") == key_id: + accept_until = previous.get("accept_until") + if not isinstance(accept_until, str) or now > parse_utc(accept_until): + raise _fail() + return WebhookCredential(actor_id=actor_id, secret=_load_secret(previous.get("secret_file"))) + except WebhookAuthenticationError: + raise + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise _fail() from exc + raise _fail() diff --git a/src/offboardproof/webhooks/schemas.py b/src/offboardproof/webhooks/schemas.py new file mode 100644 index 0000000..2a27be0 --- /dev/null +++ b/src/offboardproof/webhooks/schemas.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from pydantic import Field, field_validator + +from offboardproof.enums import RiskTier +from offboardproof.schemas import StrictModel +from offboardproof.util import normalize_email + +IDENTIFIER_PATTERN = r"^[A-Za-z0-9._~-]{8,128}$" + + +class WebhookSubject(StrictModel): + email: str + display_name: str = Field(min_length=1, max_length=200) + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + return normalize_email(value) + + +class WebhookDeparture(StrictModel): + effective_at: datetime + risk_tier: RiskTier = RiskTier.STANDARD + transfer_owner_email: str + + @field_validator("effective_at") + @classmethod + def require_offset(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("effective_at must include an explicit UTC offset") + return value.astimezone(UTC) + + @field_validator("transfer_owner_email") + @classmethod + def validate_email(cls, value: str) -> str: + return normalize_email(value) + + +class WebhookWorkflow(StrictModel): + provider: str = Field(pattern=r"^[a-z][a-z0-9_-]{1,31}$") + + +class WebhookEvent(StrictModel): + schema_version: int = Field(ge=1, le=1) + event_type: str = Field(pattern=r"^employee\.departure\.authorized$") + event_id: str = Field(pattern=IDENTIFIER_PATTERN) + occurred_at: datetime + subject: WebhookSubject + departure: WebhookDeparture + workflow: WebhookWorkflow + + @field_validator("occurred_at") + @classmethod + def require_occurred_offset(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("occurred_at must include an explicit UTC offset") + return value.astimezone(UTC) + + def validate_clock(self, now: datetime) -> None: + if self.occurred_at > now.astimezone(UTC) + timedelta(minutes=5): + raise ValueError("occurred_at cannot be more than five minutes in the future") diff --git a/src/offboardproof/webhooks/service.py b/src/offboardproof/webhooks/service.py new file mode 100644 index 0000000..f718a4f --- /dev/null +++ b/src/offboardproof/webhooks/service.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from typing import Any + +from offboardproof.audit import append_event +from offboardproof.auth import Actor +from offboardproof.config import Settings +from offboardproof.db import connection_for, transaction +from offboardproof.enums import Role +from offboardproof.errors import ConflictError, WebhookAuthenticationError +from offboardproof.schemas import CaseCreate +from offboardproof.service import WorkflowService +from offboardproof.util import iso_now, new_id +from offboardproof.webhooks.schemas import WebhookEvent + + +@dataclass(frozen=True) +class WebhookResult: + delivery_id: str + event_id: str + case_id: str + case_state: str + replayed: bool + + def as_dict(self) -> dict[str, Any]: + return { + "delivery_id": self.delivery_id, + "event_id": self.event_id, + "case_id": self.case_id, + "case_state": self.case_state, + "replayed": self.replayed, + } + + +class WebhookService: + def __init__(self, settings: Settings, workflow: WorkflowService) -> None: + self.settings = settings + self.workflow = workflow + + @staticmethod + def _service_actor(connection: sqlite3.Connection, actor_id: str) -> Actor: + row = connection.execute("SELECT id, name, role, active FROM actors WHERE id=?", (actor_id,)).fetchone() + if row is None or not bool(row["active"]) or row["role"] != Role.SERVICE.value: + raise WebhookAuthenticationError("Webhook authentication failed") + return Actor(id=str(row["id"]), name=str(row["name"]), role=Role.SERVICE) + + def accept( + self, + *, + source_id: str, + actor_id: str, + delivery_id: str, + event: WebhookEvent, + payload_sha256: str, + ) -> WebhookResult: + connection = connection_for(self.settings) + conflict_message: str | None = None + result: WebhookResult | None = None + try: + with transaction(connection): + actor = self._service_actor(connection, actor_id) + existing = connection.execute( + """ + SELECT * FROM webhook_deliveries + WHERE source_id=? AND (delivery_id=? OR event_id=?) + ORDER BY received_at LIMIT 1 + """, + (source_id, delivery_id, event.event_id), + ).fetchone() + if existing is not None: + same_payload = existing["payload_sha256"] == payload_sha256 + same_event = existing["event_id"] == event.event_id + if not same_payload or not same_event: + append_event( + connection, + event_type="webhook.conflict", + actor_id=actor.id, + case_id=existing["case_id"], + payload={ + "source_id": source_id, + "delivery_id": delivery_id, + "event_id": event.event_id, + "original_payload_sha256": existing["payload_sha256"], + "received_payload_sha256": payload_sha256, + }, + ) + conflict_message = "Webhook delivery or event identifier was reused with different content" + else: + case = connection.execute( + "SELECT state FROM cases WHERE id=?", (existing["case_id"],) + ).fetchone() + if case is None: + raise ConflictError("Webhook replay references a missing case") + append_event( + connection, + event_type="webhook.replayed", + actor_id=actor.id, + case_id=existing["case_id"], + payload={ + "source_id": source_id, + "delivery_id": delivery_id, + "event_id": event.event_id, + "payload_sha256": payload_sha256, + "original_delivery_id": existing["delivery_id"], + }, + ) + result = WebhookResult( + delivery_id=delivery_id, + event_id=event.event_id, + case_id=str(existing["case_id"]), + case_state=str(case["state"]), + replayed=True, + ) + else: + request = CaseCreate( + subject_email=event.subject.email, + subject_name=event.subject.display_name, + transfer_owner=event.departure.transfer_owner_email, + effective_at=event.departure.effective_at, + risk_tier=event.departure.risk_tier, + provider=event.workflow.provider, + ) + case_id = self.workflow.create_case_in_transaction( + connection, + actor, + request, + f"webhook:{source_id}:{event.event_id}", + ) + now = iso_now() + connection.execute( + """ + INSERT INTO webhook_deliveries + (id, source_id, delivery_id, event_id, payload_sha256, case_id, + status, received_at, completed_at) + VALUES (?, ?, ?, ?, ?, ?, 'accepted', ?, ?) + """, + (new_id(), source_id, delivery_id, event.event_id, payload_sha256, case_id, now, now), + ) + append_event( + connection, + event_type="webhook.accepted", + actor_id=actor.id, + case_id=case_id, + payload={ + "source_id": source_id, + "delivery_id": delivery_id, + "event_id": event.event_id, + "payload_sha256": payload_sha256, + }, + ) + case = connection.execute("SELECT state FROM cases WHERE id=?", (case_id,)).fetchone() + result = WebhookResult( + delivery_id=delivery_id, + event_id=event.event_id, + case_id=case_id, + case_state=str(case["state"]), + replayed=False, + ) + if conflict_message is not None: + raise ConflictError(conflict_message) + if result is None: + raise RuntimeError("Webhook transaction produced no result") + return result + finally: + connection.close() + + def record_rejection( + self, + *, + source_id: str, + actor_id: str, + delivery_id: str, + payload_sha256: str, + reason_code: str, + ) -> None: + connection = connection_for(self.settings) + try: + with transaction(connection): + actor = self._service_actor(connection, actor_id) + append_event( + connection, + event_type="webhook.rejected", + actor_id=actor.id, + payload={ + "source_id": source_id, + "delivery_id": delivery_id, + "payload_sha256": payload_sha256, + "reason_code": reason_code, + }, + ) + finally: + connection.close() diff --git a/src/offboardproof/webhooks/signature.py b/src/offboardproof/webhooks/signature.py new file mode 100644 index 0000000..3110289 --- /dev/null +++ b/src/offboardproof/webhooks/signature.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import hashlib +import hmac +import re +from datetime import datetime + +from offboardproof.errors import WebhookAuthenticationError + +SIGNATURE_PATTERN = re.compile(r"^sha256=([0-9a-f]{64})$") +DELIVERY_PATTERN = re.compile(r"^[A-Za-z0-9._~-]{8,128}$") +SIGNED_DOMAIN = b"offboardproof.webhook.v1\n" + + +def parse_timestamp(value: str, *, now: datetime, skew_seconds: int) -> int: + if not value or not value.isascii() or not value.isdigit(): + raise WebhookAuthenticationError("Webhook authentication failed") + try: + timestamp = int(value) + except ValueError as exc: + raise WebhookAuthenticationError("Webhook authentication failed") from exc + if abs(int(now.timestamp()) - timestamp) > skew_seconds: + raise WebhookAuthenticationError("Webhook authentication failed") + return timestamp + + +def verify_signature( + *, + secret: bytes, + timestamp_text: str, + delivery_id: str, + raw_body: bytes, + signature_header: str, +) -> None: + if DELIVERY_PATTERN.fullmatch(delivery_id) is None: + raise WebhookAuthenticationError("Webhook authentication failed") + match = SIGNATURE_PATTERN.fullmatch(signature_header) + if match is None: + raise WebhookAuthenticationError("Webhook authentication failed") + signed_bytes = ( + SIGNED_DOMAIN + timestamp_text.encode("ascii") + b"\n" + delivery_id.encode("utf-8") + b"\n" + raw_body + ) + expected = hmac.new(secret, signed_bytes, hashlib.sha256).hexdigest() + if not hmac.compare_digest(expected, match.group(1)): + raise WebhookAuthenticationError("Webhook authentication failed") + + +def sign_for_testing(*, secret: bytes, timestamp: int, delivery_id: str, raw_body: bytes) -> str: + timestamp_text = str(timestamp) + signed_bytes = ( + SIGNED_DOMAIN + timestamp_text.encode("ascii") + b"\n" + delivery_id.encode("utf-8") + b"\n" + raw_body + ) + return "sha256=" + hmac.new(secret, signed_bytes, hashlib.sha256).hexdigest() diff --git a/src/offboardproof/worker.py b/src/offboardproof/worker.py index 83ce07b..101c85d 100644 --- a/src/offboardproof/worker.py +++ b/src/offboardproof/worker.py @@ -16,7 +16,6 @@ OutcomeKind, ) from offboardproof.errors import ConfigurationError, ConflictError, ProviderError -from offboardproof.evidence import write_evidence from offboardproof.providers.base import ActionRequest, Observation from offboardproof.providers.factory import get_provider from offboardproof.service import WorkflowService @@ -277,15 +276,8 @@ def _complete_action( }, ) new_state = self.service._refresh_case_state(connection, action["case_id"], actor_id=None) - if new_state.value == "completed": - _, digest = write_evidence(connection, self.settings, action["case_id"]) - with transaction(connection): - append_event( - connection, - case_id=action["case_id"], - event_type="evidence.generated", - payload={"sha256": digest}, - ) + if new_state.value == "completed": + self.service._write_completed_evidence(connection, action["case_id"], actor_id=None) def _handle_provider_error( self, diff --git a/tests/test_api.py b/tests/test_api.py index cd7dfe8..51d32cc 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -10,7 +10,9 @@ def test_health_auth_and_idempotent_create(world) -> None: # type: ignore[no-untyped-def] client = TestClient(create_app(world.settings)) - assert client.get("/health/live").status_code == 200 + live = client.get("/health/live", headers={"X-Request-ID": "request-0001"}) + assert live.status_code == 200 + assert live.headers["X-Request-ID"] == "request-0001" assert client.get("/health/ready").json() == {"status": "ready"} assert client.get("/v1/cases").status_code == 403 headers = { @@ -44,3 +46,32 @@ def test_api_validates_payload_and_role(world) -> None: # type: ignore[no-untyp headers={"Authorization": f"Bearer {world.tokens[Role.HR]}"}, ) assert response.status_code == 403 + + +def test_v2_operations_endpoints_and_generated_correlation(world) -> None: # type: ignore[no-untyped-def] + client = TestClient(create_app(world.settings)) + case = world.create(key="api-v2-operations") + security = {"Authorization": f"Bearer {world.tokens[Role.SECURITY]}"} + admin = {"Authorization": f"Bearer {world.tokens[Role.ADMIN]}"} + auditor = {"Authorization": f"Bearer {world.tokens[Role.AUDITOR]}"} + + created = client.post( + f"/v1/cases/{case['id']}/legal-holds", + headers=security, + json={"reason": "Security investigation is active"}, + ) + assert created.status_code == 201 + released = client.post( + f"/v1/cases/{case['id']}/legal-holds/release", + headers=admin, + json={"reason": "Security investigation is closed"}, + ) + assert released.status_code == 200 + assert client.get("/v1/retention/report", headers=auditor).status_code == 200 + metrics = client.get("/v1/metrics/summary", headers=auditor) + assert metrics.status_code == 200 + assert "webhook_replays_suppressed" in metrics.json() + + generated = client.get("/health/live", headers={"X-Request-ID": "bad"}) + assert generated.headers["X-Request-ID"] != "bad" + assert len(generated.headers["X-Request-ID"]) == 36 diff --git a/tests/test_backup.py b/tests/test_backup.py new file mode 100644 index 0000000..68a4e50 --- /dev/null +++ b/tests/test_backup.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest +from conftest import World +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from offboardproof.backup import BackupService +from offboardproof.enums import Role +from offboardproof.errors import ConfigurationError +from offboardproof.util import canonical_json, sha256_text +from offboardproof.worker import Worker + + +def _complete(world: World, *, key: str = "backup-case-0001") -> None: + case = world.create(key=key) + world.plan_and_approve(str(case["id"])) + worker = Worker(world.settings) + while worker.run_once(): + pass + view = world.service.case_view(str(case["id"])) + manual = next(control for control in view["controls"] if control["manual"] == 1) + world.service.complete_manual( + world.actors[Role.MANAGER], + str(case["id"]), + manual["id"], + "Synthetic backup evidence TEST-BACKUP", + ) + + +def test_backup_and_scratch_restore_verify(world: World) -> None: + _complete(world) + secret = world.settings.database_path.parent / "must-not-back-up.secret" + secret.write_text("not-a-real-secret", encoding="utf-8") + with TemporaryDirectory() as temporary: + root = Path(temporary) + backup = root / "backup" + scratch = root / "scratch" + service = BackupService(world.settings) + created = service.create(world.actors[Role.OPERATOR], backup) + assert created["file_count"] >= 5 + assert not any(path.name == secret.name for path in backup.rglob("*")) + verified = service.verify(backup) + assert verified.valid + assert verified.unsigned_evidence_count == 1 + restored = service.restore_verify(backup, scratch) + assert restored["database_integrity"] == "ok" + assert restored["foreign_key_violations"] == 0 + assert restored["audit_chain_valid"] is True + + +def test_backup_detects_changed_and_unsafe_inventory_entries(world: World) -> None: + with TemporaryDirectory() as temporary: + root = Path(temporary) + backup = root / "backup" + service = BackupService(world.settings) + service.create(world.actors[Role.SECURITY], backup) + config = backup / "config" / "settings.json" + config.write_text("changed", encoding="utf-8") + with pytest.raises(ConfigurationError, match="digest verification"): + service.verify(backup) + + second = root / "second" + service.create(world.actors[Role.SECURITY], second) + inventory_path = second / "inventory.json" + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + inventory["entries"][0]["path"] = "../escape" + stable = {"schema_version": inventory["schema_version"], "entries": inventory["entries"]} + inventory["inventory_sha256"] = sha256_text(canonical_json(stable)) + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + with pytest.raises(ConfigurationError, match="unsafe path"): + service.verify(second) + + +def test_signed_backup_verifies_public_trust_material(world: World) -> None: + private_key = Ed25519PrivateKey.generate() + key_path = world.settings.database_path.parent / "backup-signing.key" + key_path.write_bytes(private_key.private_bytes_raw()) + world.settings.signing_key_file = key_path.resolve() + world.settings.signing_key_id = "backup-signing-key" + world.settings.require_signed_evidence = True + _complete(world) + with TemporaryDirectory() as temporary: + service = BackupService(world.settings) + backup = Path(temporary) / "signed-backup" + service.create(world.actors[Role.ADMIN], backup) + verified = service.verify(backup) + assert verified.signed_evidence_verified == 1 + assert verified.unsigned_evidence_count == 0 + + +def test_backup_rejects_unsafe_destinations_and_extra_files(world: World) -> None: + service = BackupService(world.settings) + with pytest.raises(ConfigurationError, match="outside the repository"): + service.create(world.actors[Role.OPERATOR], Path.cwd() / "unsafe-backup") + with TemporaryDirectory() as temporary: + root = Path(temporary) + nonempty = root / "nonempty" + nonempty.mkdir() + (nonempty / "existing").write_text("occupied", encoding="utf-8") + with pytest.raises(ConfigurationError, match="new or empty"): + service.create(world.actors[Role.OPERATOR], nonempty) + backup = root / "backup" + service.create(world.actors[Role.OPERATOR], backup) + (backup / "extra").write_text("unlisted", encoding="utf-8") + with pytest.raises(ConfigurationError, match="unlisted"): + service.verify(backup) + + +def test_backup_preserves_public_keys_for_historical_rotation(world: World) -> None: + first_key = Ed25519PrivateKey.generate() + first_path = world.settings.database_path.parent / "first.key" + first_path.write_bytes(first_key.private_bytes_raw()) + world.settings.signing_key_file = first_path.resolve() + world.settings.signing_key_id = "rotation-first" + world.settings.require_signed_evidence = True + _complete(world, key="backup-rotation-first") + + second_key = Ed25519PrivateKey.generate() + second_path = world.settings.database_path.parent / "second.key" + second_path.write_bytes(second_key.private_bytes_raw()) + world.settings.signing_key_file = second_path.resolve() + world.settings.signing_key_id = "rotation-second" + _complete(world, key="backup-rotation-second") + + with TemporaryDirectory() as temporary: + backup = Path(temporary) / "rotated-backup" + service = BackupService(world.settings) + service.create(world.actors[Role.SECURITY], backup) + verified = service.verify(backup) + assert verified.signed_evidence_verified == 2 + assert (backup / "public-keys" / "rotation-first.pub").is_file() + assert (backup / "public-keys" / "rotation-second.pub").is_file() diff --git a/tests/test_evidence_v2.py b/tests/test_evidence_v2.py new file mode 100644 index 0000000..89a9ed3 --- /dev/null +++ b/tests/test_evidence_v2.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from conftest import World +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from offboardproof.db import connection_for +from offboardproof.enums import Role +from offboardproof.errors import ConfigurationError +from offboardproof.evidence_v2 import build_evidence_v2, write_evidence_v2 +from offboardproof.evidence_verification import EvidenceVerificationError, verify_evidence_v2 +from offboardproof.worker import Worker + + +def _complete(world: World, *, key: str) -> str: + case = world.create(key=key) + world.plan_and_approve(str(case["id"])) + worker = Worker(world.settings) + while worker.run_once(): + pass + view = world.service.case_view(str(case["id"])) + manual = next(item for item in view["controls"] if item["manual"] == 1) + completed = world.service.complete_manual( + world.actors[Role.MANAGER], + str(case["id"]), + manual["id"], + "Synthetic evidence confirmation TEST-V2", + ) + assert completed["state"] == "completed" + return str(case["id"]) + + +def _configure_key(world: World, tmp_path: Path) -> tuple[Ed25519PrivateKey, Path]: + private_key = Ed25519PrivateKey.generate() + key_file = tmp_path / "evidence-signing.key" + key_file.write_bytes( + private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + ) + world.settings.signing_key_file = key_file.resolve() + world.settings.signing_key_id = "pilot-2026-q3" + world.settings.require_signed_evidence = True + public_file = tmp_path / "evidence-signing.pub" + public_file.write_bytes( + private_key.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + return private_key, public_file + + +def _expect_code(code: int, evidence: Path, manifest: Path, public_key: Path) -> None: + with pytest.raises(EvidenceVerificationError) as raised: + verify_evidence_v2(evidence, manifest, public_key) + assert raised.value.exit_code == code + + +def test_signed_export_is_deterministic_and_verifies(world: World, tmp_path: Path) -> None: + _private_key, public_file = _configure_key(world, tmp_path) + case_id = _complete(world, key="evidence-v2-0001") + connection = connection_for(world.settings) + try: + payload = build_evidence_v2(connection, case_id) + evidence, manifest, first_digest = write_evidence_v2(connection, world.settings, case_id) + _, _, second_digest = write_evidence_v2(connection, world.settings, case_id) + artifact = connection.execute( + "SELECT * FROM evidence_artifacts WHERE case_id=? AND schema_version=2", + (case_id,), + ).fetchone() + finally: + connection.close() + + assert payload["schema_version"] == 2 + assert payload["completion"]["event_hash"] + assert payload["retention"]["retention_until"] + assert payload["retention"]["policy_snapshot"] == "default-365-days" + assert first_digest == second_digest + assert artifact is not None and artifact["signing_status"] == "signed" + result = verify_evidence_v2(evidence, manifest, public_file) + assert result["valid"] is True + assert result["origin"] == "trusted_key_match" + assert result["key_id"] == "pilot-2026-q3" + + +def test_tampering_and_wrong_trust_anchor_fail(world: World, tmp_path: Path) -> None: + _private_key, public_file = _configure_key(world, tmp_path) + case_id = _complete(world, key="evidence-v2-0002") + connection = connection_for(world.settings) + try: + evidence, manifest, _ = write_evidence_v2(connection, world.settings, case_id) + finally: + connection.close() + + original_evidence = evidence.read_bytes() + evidence.write_bytes(original_evidence + b" ") + _expect_code(3, evidence, manifest, public_file) + evidence.write_bytes(original_evidence) + + manifest_value = json.loads(manifest.read_text(encoding="utf-8")) + signature = manifest_value["signature"]["value"] + manifest_value["signature"]["value"] = ("A" if signature[0] != "A" else "B") + signature[1:] + manifest.write_text(json.dumps(manifest_value), encoding="utf-8") + _expect_code(4, evidence, manifest, public_file) + + connection = connection_for(world.settings) + try: + _, manifest, _ = write_evidence_v2(connection, world.settings, case_id) + finally: + connection.close() + wrong_key = Ed25519PrivateKey.generate() + wrong_public = tmp_path / "wrong.pub" + wrong_public.write_bytes( + wrong_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ) + _expect_code(5, evidence, manifest, wrong_public) + + connection = connection_for(world.settings) + try: + _, manifest, _ = write_evidence_v2(connection, world.settings, case_id) + finally: + connection.close() + manifest_value = json.loads(manifest.read_text(encoding="utf-8")) + embedded = manifest_value["public_key"]["value"] + manifest_value["public_key"]["value"] = ("A" if embedded[0] != "A" else "B") + embedded[1:] + manifest.write_text(json.dumps(manifest_value), encoding="utf-8") + _expect_code(5, evidence, manifest, public_file) + + +def test_signing_key_rotation_preserves_historical_manifest(world: World, tmp_path: Path) -> None: + _private_key, original_public = _configure_key(world, tmp_path) + case_id = _complete(world, key="evidence-v2-rotation") + connection = connection_for(world.settings) + try: + evidence, manifest, _ = write_evidence_v2(connection, world.settings, case_id) + original_manifest = manifest.read_bytes() + finally: + connection.close() + + replacement = Ed25519PrivateKey.generate() + replacement_file = tmp_path / "replacement.key" + replacement_file.write_bytes(replacement.private_bytes_raw()) + replacement_public = tmp_path / "replacement.pub" + replacement_public.write_bytes(replacement.public_key().public_bytes_raw()) + world.settings.signing_key_file = replacement_file.resolve() + world.settings.signing_key_id = "pilot-2026-q4" + connection = connection_for(world.settings) + try: + write_evidence_v2(connection, world.settings, case_id) + finally: + connection.close() + + assert manifest.read_bytes() == original_manifest + assert verify_evidence_v2(evidence, manifest, original_public)["valid"] is True + _expect_code(5, evidence, manifest, replacement_public) + + +def test_unsigned_export_is_explicit_and_not_implicitly_trusted(world: World, tmp_path: Path) -> None: + case_id = _complete(world, key="evidence-v2-0003") + connection = connection_for(world.settings) + try: + evidence, manifest, _ = write_evidence_v2(connection, world.settings, case_id) + artifact = connection.execute( + "SELECT signing_status FROM evidence_artifacts WHERE case_id=? AND schema_version=2", + (case_id,), + ).fetchone() + finally: + connection.close() + public_file = tmp_path / "unused.pub" + public_file.write_bytes(Ed25519PrivateKey.generate().public_key().public_bytes_raw()) + _expect_code(6, evidence, manifest, public_file) + assert artifact is not None and artifact["signing_status"] == "unsigned" + + +def test_required_signing_failure_does_not_complete_case(world: World) -> None: + world.settings.require_signed_evidence = True + case = world.create(key="evidence-required-0001") + world.plan_and_approve(str(case["id"])) + worker = Worker(world.settings) + while worker.run_once(): + pass + view = world.service.case_view(str(case["id"])) + manual = next(item for item in view["controls"] if item["manual"] == 1) + with pytest.raises(ConfigurationError, match="no signing key"): + world.service.complete_manual( + world.actors[Role.MANAGER], + str(case["id"]), + manual["id"], + "Synthetic required signing failure", + ) + failed = world.service.case_view(str(case["id"])) + assert failed["state"] == "exception" + assert failed["completed_at"] is None + assert any(item["category"] == "evidence_signing" for item in failed["exceptions"]) + assert not any(event["event_type"] == "case.completed" for event in world.service.audit_events(str(case["id"]))) diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..0cc76b5 --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import sqlite3 +from importlib.resources import files +from pathlib import Path + +import pytest + +from offboardproof.config import Settings +from offboardproof.db import SCHEMA_VERSION, connection_for, migrate + + +def _columns(connection: sqlite3.Connection, table: str) -> set[str]: + return {str(row["name"]) for row in connection.execute(f"PRAGMA table_info({table})").fetchall()} + + +def test_clean_install_and_repeated_migration(tmp_path) -> None: # type: ignore[no-untyped-def] + settings = Settings(database_path=tmp_path / "clean.db", evidence_dir=tmp_path / "evidence") + + assert migrate(settings) == SCHEMA_VERSION + assert migrate(settings) == SCHEMA_VERSION + + connection = connection_for(settings) + try: + assert int(connection.execute("PRAGMA user_version").fetchone()[0]) == 4 + assert {"retention_until", "retention_policy_id"} <= _columns(connection, "cases") + assert { + "schema_version", + "manifest_path", + "evidence_sha256", + "signing_status", + "signing_algorithm", + "signing_key_id", + "public_key_fingerprint", + "completion_event_hash", + } <= _columns(connection, "evidence_artifacts") + assert connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='webhook_deliveries'" + ).fetchone() + assert connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='legal_holds'").fetchone() + assert connection.execute("PRAGMA foreign_key_check").fetchall() == [] + finally: + connection.close() + + +def test_real_v1_fixture_upgrades_without_data_loss(tmp_path) -> None: # type: ignore[no-untyped-def] + database_path = tmp_path / "upgrade.db" + connection = sqlite3.connect(database_path) + try: + initial_sql = files("offboardproof").joinpath("migrations", "0001_initial.sql").read_text(encoding="utf-8") + connection.executescript(initial_sql) + connection.execute("PRAGMA user_version = 1") + connection.execute( + """ + INSERT INTO actors (id, name, role, token_digest, token_prefix, active, created_at) + VALUES ('actor-1', 'Fixture HR', 'hr', 'digest', 'prefix', 1, '2026-08-01T00:00:00Z') + """ + ) + connection.execute( + """ + INSERT INTO cases ( + id, organization_id, idempotency_key, subject_email, subject_name, transfer_owner, + effective_at, risk_tier, provider, state, created_by, created_at, updated_at + ) VALUES ( + 'case-1', 'local', 'fixture-key', 'person@example.test', 'Fixture Person', + 'owner@example.test', '2026-08-02T00:00:00Z', 'standard', 'mock', 'received', + 'actor-1', '2026-08-01T00:00:00Z', '2026-08-01T00:00:00Z' + ) + """ + ) + connection.execute( + """ + INSERT INTO evidence_artifacts (id, case_id, path, sha256, mime_type, created_at) + VALUES ('evidence-1', 'case-1', 'fixture.json', ?, 'application/json', '2026-08-03T00:00:00Z') + """, + ("a" * 64,), + ) + connection.execute( + """ + INSERT INTO audit_events ( + event_id, case_id, actor_id, event_type, payload_json, previous_hash, event_hash, created_at + ) VALUES ('event-1', 'case-1', 'actor-1', 'fixture.created', '{}', ?, ?, '2026-08-01T00:00:00Z') + """, + ("0" * 64, "b" * 64), + ) + connection.commit() + finally: + connection.close() + + settings = Settings(database_path=database_path, evidence_dir=tmp_path / "evidence") + assert migrate(settings) == SCHEMA_VERSION + + upgraded = connection_for(settings) + try: + case = upgraded.execute("SELECT * FROM cases WHERE id='case-1'").fetchone() + artifact = upgraded.execute("SELECT * FROM evidence_artifacts WHERE id='evidence-1'").fetchone() + event = upgraded.execute("SELECT * FROM audit_events WHERE event_id='event-1'").fetchone() + assert case is not None and case["subject_email"] == "person@example.test" + assert case["retention_until"] is None and case["retention_policy_id"] is None + assert artifact is not None and artifact["sha256"] == "a" * 64 + assert artifact["schema_version"] == 1 and artifact["signing_status"] == "legacy" + assert event is not None and event["event_hash"] == "b" * 64 + assert upgraded.execute("PRAGMA foreign_key_check").fetchall() == [] + finally: + upgraded.close() + + +def test_newer_schema_is_rejected(tmp_path) -> None: # type: ignore[no-untyped-def] + settings = Settings(database_path=tmp_path / "future.db", evidence_dir=tmp_path / "evidence") + connection = sqlite3.connect(settings.database_path) + try: + connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION + 1}") + finally: + connection.close() + + with pytest.raises(RuntimeError, match="newer than supported"): + migrate(settings) + + +def test_project_and_packaged_migrations_are_identical() -> None: + project_migrations = Path(__file__).resolve().parents[1] / "migrations" + for name in ( + "0002_webhook_intake.sql", + "0003_evidence_v2.sql", + "0004_retention_operations.sql", + ): + packaged = files("offboardproof").joinpath("migrations", name).read_bytes() + assert packaged == (project_migrations / name).read_bytes() diff --git a/tests/test_retention.py b/tests/test_retention.py new file mode 100644 index 0000000..4e46535 --- /dev/null +++ b/tests/test_retention.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +import pytest +from conftest import World + +from offboardproof.db import connection_for +from offboardproof.enums import Role +from offboardproof.errors import AuthorizationError, ConflictError +from offboardproof.retention import RetentionService +from offboardproof.schemas import CaseCreate + + +def test_default_and_explicit_retention_snapshots(world: World) -> None: + default_case = world.create(key="retention-default-0001") + default_until = datetime.fromisoformat(str(default_case["retention_until"])) + created_at = datetime.fromisoformat(str(default_case["created_at"])) + assert timedelta(days=364) < default_until - created_at <= timedelta(days=365) + assert default_case["retention_policy_id"] == "default-365-days" + + explicit_until = datetime.now(UTC) + timedelta(days=30) + explicit = world.service.create_case( + world.actors[Role.HR], + CaseCreate( + subject_email="second@example.test", + subject_name="Second Person", + transfer_owner="owner@example.test", + effective_at=datetime.now(UTC), + provider="mock", + retention_until=explicit_until, + retention_policy_id="contract-30-days", + ), + "retention-explicit-0001", + ) + assert datetime.fromisoformat(str(explicit["retention_until"])) == explicit_until + assert explicit["retention_policy_id"] == "contract-30-days" + + events = world.service.audit_events(str(explicit["id"])) + snapshot = next(event for event in events if event["event_type"] == "retention.snapshot_created") + assert "contract-30-days" in snapshot["payload_json"] + + +def test_legal_hold_authorization_uniqueness_and_release(world: World) -> None: + case = world.create(key="legal-hold-0001") + retention = RetentionService(world.settings) + with pytest.raises(AuthorizationError): + retention.create_hold(world.actors[Role.OPERATOR], str(case["id"]), "Operator cannot hold evidence") + + created = retention.create_hold( + world.actors[Role.SECURITY], + str(case["id"]), + "Security investigation remains open", + ) + assert created["status"] == "active" + with pytest.raises(ConflictError): + retention.create_hold( + world.actors[Role.SECURITY], + str(case["id"]), + "A second active hold is forbidden", + ) + released = retention.release_hold( + world.actors[Role.ADMIN], + str(case["id"]), + "Investigation closed with approval", + ) + assert released["status"] == "released" + assert released["released_by"] == world.actors[Role.ADMIN].id + + +def test_retention_report_is_non_mutating_digested_and_pii_free(world: World) -> None: + case = world.create(key="retention-report-0001") + connection = connection_for(world.settings) + try: + connection.execute( + "UPDATE cases SET state='completed', retention_until=? WHERE id=?", + ((datetime.now(UTC) - timedelta(days=1)).isoformat(), case["id"]), + ) + events_before = int(connection.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]) + finally: + connection.close() + + retention = RetentionService(world.settings) + as_of = datetime.now(UTC) + first = retention.report(world.actors[Role.AUDITOR], as_of) + second = retention.report(world.actors[Role.AUDITOR], as_of) + assert first["report_sha256"] == second["report_sha256"] + assert first["cases"][0]["eligibility"] == "expired_review_required" + rendered = json.dumps(first) + assert "alex@example.test" not in rendered + assert "Alex Morgan" not in rendered + + connection = connection_for(world.settings) + try: + assert int(connection.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]) == events_before + finally: + connection.close() + + metrics = world.service.metrics() + assert "cases_retention_expired" in metrics + assert "alex@example.test" not in json.dumps(metrics) + + +def test_retention_report_covers_non_destructive_categories(world: World) -> None: + incomplete = world.create(key="retention-category-incomplete") + future = world.create(key="retention-category-future") + missing = world.create(key="retention-category-missing") + held = world.create(key="retention-category-held") + now = datetime.now(UTC) + connection = connection_for(world.settings) + try: + connection.execute( + "UPDATE cases SET state='completed', retention_until=? WHERE id=?", + ((now + timedelta(days=1)).isoformat(), future["id"]), + ) + connection.execute("UPDATE cases SET retention_until=NULL WHERE id=?", (missing["id"],)) + connection.execute( + "UPDATE cases SET state='completed', retention_until=? WHERE id=?", + ((now - timedelta(days=1)).isoformat(), held["id"]), + ) + finally: + connection.close() + retention = RetentionService(world.settings) + retention.create_hold(world.actors[Role.SECURITY], str(held["id"]), "Active litigation preservation hold") + report = retention.report(world.actors[Role.OPERATOR], now) + categories = {item["case_id"]: item["eligibility"] for item in report["cases"]} + assert categories[str(incomplete["id"])] == "incomplete_case" + assert categories[str(future["id"])] == "not_expired" + assert categories[str(missing["id"])] == "missing_retention_policy" + assert categories[str(held["id"])] == "active_legal_hold" diff --git a/tests/test_security_edges.py b/tests/test_security_edges.py new file mode 100644 index 0000000..02c5af6 --- /dev/null +++ b/tests/test_security_edges.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from offboardproof.config import Settings +from offboardproof.errors import ConfigurationError, WebhookAuthenticationError +from offboardproof.evidence_signing import load_public_key, load_signing_key, write_public_key +from offboardproof.evidence_verification import EvidenceVerificationError, verify_evidence_v2 +from offboardproof.webhooks.config import load_credential +from offboardproof.webhooks.signature import parse_timestamp, verify_signature + + +def test_signing_key_configuration_formats_and_failures(tmp_path: Path) -> None: + private_key = Ed25519PrivateKey.generate() + raw_file = tmp_path / "raw.key" + raw_file.write_bytes(private_key.private_bytes_raw()) + settings = Settings(signing_key_file=raw_file.resolve(), signing_key_id="raw-key") + loaded = load_signing_key(settings) + assert loaded is not None and loaded.key_id == "raw-key" + + public_file = tmp_path / "public.pem" + write_public_key(loaded, public_file) + assert load_public_key(public_file).public_bytes_raw() == private_key.public_key().public_bytes_raw() + raw_public = tmp_path / "public.raw" + raw_public.write_bytes(private_key.public_key().public_bytes_raw()) + assert load_public_key(raw_public).public_bytes_raw() == private_key.public_key().public_bytes_raw() + + with pytest.raises(ConfigurationError, match="SIGNING_KEY_ID"): + load_signing_key(Settings(signing_key_file=raw_file.resolve())) + with pytest.raises(ConfigurationError, match="absolute readable"): + load_signing_key(Settings(signing_key_file=Path("relative.key"), signing_key_id="bad")) + with pytest.raises(ConfigurationError, match="no signing key"): + load_signing_key(Settings(require_signed_evidence=True)) + + password = b"correct horse battery staple" + encrypted = tmp_path / "encrypted.pem" + encrypted.write_bytes( + private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.BestAvailableEncryption(password), + ) + ) + password_file = tmp_path / "password.txt" + password_file.write_bytes(password + b"\n") + encrypted_settings = Settings( + signing_key_file=encrypted.resolve(), + signing_key_password_file=password_file.resolve(), + signing_key_id="encrypted-key", + ) + assert load_signing_key(encrypted_settings) is not None + password_file.write_text("wrong", encoding="utf-8") + with pytest.raises(ConfigurationError, match="could not be loaded"): + load_signing_key(encrypted_settings) + password_file.write_bytes(b"") + with pytest.raises(ConfigurationError, match="password file is empty"): + load_signing_key(encrypted_settings) + + missing_public = tmp_path / "missing.pub" + with pytest.raises(ValueError, match="does not exist"): + load_public_key(missing_public) + invalid_public = tmp_path / "invalid.pub" + invalid_public.write_bytes(b"invalid") + with pytest.raises(ValueError, match="could not be loaded"): + load_public_key(invalid_public) + + +def _webhook_config(tmp_path: Path, secret_file: Path) -> Path: + config = tmp_path / "webhook.json" + config.write_text( + json.dumps( + { + "schema_version": 1, + "sources": { + "source-one": { + "active": True, + "actor_id": "service-actor", + "current_key": {"key_id": "current", "secret_file": str(secret_file.resolve())}, + "previous_key": { + "key_id": "previous", + "secret_file": str(secret_file.resolve()), + "accept_until": (datetime.now(UTC) - timedelta(minutes=1)).isoformat(), + }, + } + }, + } + ), + encoding="utf-8", + ) + return config.resolve() + + +def test_webhook_configuration_and_signature_negative_paths(tmp_path: Path) -> None: + secret_file = tmp_path / "secret" + secret_file.write_bytes(b"x" * 32) + now = datetime.now(UTC) + settings = Settings(webhook_config_file=_webhook_config(tmp_path, secret_file)) + assert load_credential(settings, source_id="source-one", key_id="current", now=now).actor_id == "service-actor" + + for source, key in (("missing", "current"), ("source-one", "missing"), ("source-one", "previous")): + with pytest.raises(WebhookAuthenticationError, match="authentication failed"): + load_credential(settings, source_id=source, key_id=key, now=now) + + secret_file.write_bytes(b"short") + with pytest.raises(WebhookAuthenticationError): + load_credential(settings, source_id="source-one", key_id="current", now=now) + with pytest.raises(WebhookAuthenticationError): + load_credential(Settings(), source_id="source-one", key_id="current", now=now) + + document = json.loads(settings.webhook_config_file.read_text(encoding="utf-8")) # type: ignore[union-attr] + document["schema_version"] = 2 + settings.webhook_config_file.write_text(json.dumps(document), encoding="utf-8") # type: ignore[union-attr] + with pytest.raises(WebhookAuthenticationError): + load_credential(settings, source_id="source-one", key_id="current", now=now) + + document["schema_version"] = 1 + document["sources"]["source-one"]["active"] = False + settings.webhook_config_file.write_text(json.dumps(document), encoding="utf-8") # type: ignore[union-attr] + with pytest.raises(WebhookAuthenticationError): + load_credential(settings, source_id="source-one", key_id="current", now=now) + + for value in ("", " 1", "+1", "1.0", "not-time"): + with pytest.raises(WebhookAuthenticationError): + parse_timestamp(value, now=now, skew_seconds=300) + with pytest.raises(WebhookAuthenticationError): + parse_timestamp(str(int(now.timestamp()) - 301), now=now, skew_seconds=300) + with pytest.raises(WebhookAuthenticationError): + verify_signature( + secret=b"x" * 32, + timestamp_text=str(int(now.timestamp())), + delivery_id="short", + raw_body=b"{}", + signature_header="bad", + ) + with pytest.raises(WebhookAuthenticationError): + verify_signature( + secret=b"x" * 32, + timestamp_text=str(int(now.timestamp())), + delivery_id="delivery-valid", + raw_body=b"{}", + signature_header="sha256=" + "0" * 64, + ) + + +def test_evidence_verifier_rejects_malformed_and_unsupported_inputs(tmp_path: Path) -> None: + evidence = tmp_path / "evidence.json" + manifest = tmp_path / "manifest.json" + public_key = tmp_path / "public.key" + public_key.write_bytes(Ed25519PrivateKey.generate().public_key().public_bytes_raw()) + + evidence.write_text("not-json", encoding="utf-8") + manifest.write_text("{}", encoding="utf-8") + with pytest.raises(EvidenceVerificationError) as malformed: + verify_evidence_v2(evidence, manifest, public_key) + assert malformed.value.exit_code == 2 + + evidence.write_text('{"schema_version":1}', encoding="utf-8") + manifest.write_text('{"manifest_schema_version":1}', encoding="utf-8") + with pytest.raises(EvidenceVerificationError) as unsupported: + verify_evidence_v2(evidence, manifest, public_key) + assert unsupported.value.exit_code == 2 + + evidence.write_text('{"schema_version":2}', encoding="utf-8") + manifest.write_text('{"manifest_schema_version":1,"signing_status":"unsigned"}', encoding="utf-8") + with pytest.raises(EvidenceVerificationError) as unsigned: + verify_evidence_v2(evidence, manifest, public_key) + assert unsigned.value.exit_code == 6 diff --git a/tests/test_units.py b/tests/test_units.py index 587c472..732a92e 100644 --- a/tests/test_units.py +++ b/tests/test_units.py @@ -8,7 +8,7 @@ from offboardproof.auth import authenticate, create_actor from offboardproof.config import Settings -from offboardproof.db import connection_for, migrate +from offboardproof.db import SCHEMA_VERSION, connection_for, migrate from offboardproof.enums import Role from offboardproof.errors import AuthorizationError, ConfigurationError from offboardproof.providers.factory import get_provider @@ -32,8 +32,8 @@ def test_email_and_time_normalization() -> None: def test_authentication_and_migration(tmp_path) -> None: # type: ignore[no-untyped-def] settings = Settings(database_path=tmp_path / "db.sqlite", evidence_dir=tmp_path / "evidence") - assert migrate(settings) == 1 - assert migrate(settings) == 1 + assert migrate(settings) == SCHEMA_VERSION + assert migrate(settings) == SCHEMA_VERSION connection = connection_for(settings) try: actor, token = create_actor(connection, "Auditor", Role.AUDITOR) diff --git a/tests/test_webhooks.py b/tests/test_webhooks.py new file mode 100644 index 0000000..d0055ac --- /dev/null +++ b/tests/test_webhooks.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +from conftest import World +from fastapi.testclient import TestClient + +from offboardproof.api import create_app +from offboardproof.db import connection_for +from offboardproof.enums import Role +from offboardproof.util import canonical_json +from offboardproof.webhooks.signature import sign_for_testing + + +def _configure(world: World, tmp_path) -> bytes: # type: ignore[no-untyped-def] + secret = b"a-secure-webhook-secret-with-more-than-32-bytes" + secret_file = tmp_path / "webhook.secret" + secret_file.write_bytes(secret) + config_file = tmp_path / "webhooks.json" + config_file.write_text( + json.dumps( + { + "schema_version": 1, + "sources": { + "hr-primary": { + "active": True, + "actor_id": world.actors[Role.SERVICE].id, + "current_key": {"key_id": "current-key", "secret_file": str(secret_file.resolve())}, + } + }, + } + ), + encoding="utf-8", + ) + world.settings.webhook_config_file = config_file.resolve() + return secret + + +def _event(*, display_name: str = "Alex Morgan") -> dict[str, Any]: + return { + "schema_version": 1, + "event_type": "employee.departure.authorized", + "event_id": "hr-event-0192", + "occurred_at": datetime.now(UTC).isoformat(), + "subject": {"email": "alex@example.test", "display_name": display_name}, + "departure": { + "effective_at": "2026-08-20T17:00:00Z", + "risk_tier": "standard", + "transfer_owner_email": "owner@example.test", + }, + "workflow": {"provider": "mock"}, + } + + +def _headers(secret: bytes, raw_body: bytes, *, delivery_id: str, timestamp: int | None = None) -> dict[str, str]: + timestamp = timestamp if timestamp is not None else int(datetime.now(UTC).timestamp()) + return { + "Content-Type": "application/json", + "OffboardProof-Delivery": delivery_id, + "OffboardProof-Timestamp": str(timestamp), + "OffboardProof-Key-Id": "current-key", + "OffboardProof-Signature": sign_for_testing( + secret=secret, + timestamp=timestamp, + delivery_id=delivery_id, + raw_body=raw_body, + ), + } + + +def test_valid_delivery_replay_and_conflict(world: World, tmp_path) -> None: # type: ignore[no-untyped-def] + secret = _configure(world, tmp_path) + client = TestClient(create_app(world.settings)) + raw_body = canonical_json(_event()).encode() + + first = client.post( + "/v1/intake/webhooks/hr-primary", + content=raw_body, + headers=_headers(secret, raw_body, delivery_id="delivery-0192"), + ) + assert first.status_code == 202 + assert first.json()["replayed"] is False + + replay = client.post( + "/v1/intake/webhooks/hr-primary", + content=raw_body, + headers=_headers(secret, raw_body, delivery_id="delivery-0193"), + ) + assert replay.status_code == 200 + assert replay.json()["case_id"] == first.json()["case_id"] + assert replay.json()["replayed"] is True + + changed_body = canonical_json(_event(display_name="Changed Name")).encode() + conflict = client.post( + "/v1/intake/webhooks/hr-primary", + content=changed_body, + headers=_headers(secret, changed_body, delivery_id="delivery-0192"), + ) + assert conflict.status_code == 409 + + connection = connection_for(world.settings) + try: + assert connection.execute("SELECT count(*) FROM cases").fetchone()[0] == 1 + assert connection.execute("SELECT count(*) FROM webhook_deliveries").fetchone()[0] == 1 + event_types = [ + row["event_type"] + for row in connection.execute( + "SELECT event_type FROM audit_events WHERE event_type LIKE 'webhook.%' ORDER BY sequence_no" + ).fetchall() + ] + assert event_types == ["webhook.accepted", "webhook.replayed", "webhook.conflict"] + finally: + connection.close() + + +def test_authentication_and_schema_failures_create_no_case(world: World, tmp_path) -> None: # type: ignore[no-untyped-def] + secret = _configure(world, tmp_path) + client = TestClient(create_app(world.settings)) + raw_body = canonical_json(_event()).encode() + invalid_signature_headers = _headers(secret, raw_body, delivery_id="delivery-0200") + invalid_signature_headers["OffboardProof-Signature"] = "sha256=" + "0" * 64 + + invalid_signature = client.post( + "/v1/intake/webhooks/hr-primary", + content=raw_body, + headers=invalid_signature_headers, + ) + assert invalid_signature.status_code == 401 + assert "secret" not in invalid_signature.text.lower() + + invalid_event = _event() + invalid_event["unexpected"] = "forbidden" + invalid_body = canonical_json(invalid_event).encode() + invalid_schema = client.post( + "/v1/intake/webhooks/hr-primary", + content=invalid_body, + headers=_headers(secret, invalid_body, delivery_id="delivery-0201"), + ) + assert invalid_schema.status_code == 400 + + stale = int(datetime.now(UTC).timestamp()) - 1_000 + stale_response = client.post( + "/v1/intake/webhooks/hr-primary", + content=raw_body, + headers=_headers(secret, raw_body, delivery_id="delivery-0202", timestamp=stale), + ) + assert stale_response.status_code == 401 + + connection = connection_for(world.settings) + try: + assert connection.execute("SELECT count(*) FROM cases").fetchone()[0] == 0 + assert connection.execute("SELECT count(*) FROM webhook_deliveries").fetchone()[0] == 0 + rejected = connection.execute( + "SELECT payload_json FROM audit_events WHERE event_type='webhook.rejected'" + ).fetchone() + assert rejected is not None + assert "invalid_event_schema" in rejected["payload_json"] + assert "alex@example.test" not in rejected["payload_json"] + finally: + connection.close() + + +def test_body_and_content_type_limits(world: World, tmp_path) -> None: # type: ignore[no-untyped-def] + secret = _configure(world, tmp_path) + world.settings.webhook_max_body_bytes = 8_192 + client = TestClient(create_app(world.settings)) + raw_body = canonical_json(_event()).encode() + + unsupported = client.post( + "/v1/intake/webhooks/hr-primary", + content=raw_body, + headers={**_headers(secret, raw_body, delivery_id="delivery-0300"), "Content-Type": "text/plain"}, + ) + assert unsupported.status_code == 415 + + oversized_body = b"{" + b" " * 8_192 + b"}" + oversized = client.post( + "/v1/intake/webhooks/hr-primary", + content=oversized_body, + headers=_headers(secret, oversized_body, delivery_id="delivery-0301"), + ) + assert oversized.status_code == 413 + + +def test_previous_key_rotation_window_and_raw_body_integrity(world: World, tmp_path) -> None: # type: ignore[no-untyped-def] + current_secret = _configure(world, tmp_path) + previous_secret = b"previous-secure-webhook-secret-more-than-32-bytes" + previous_file = tmp_path / "previous.secret" + previous_file.write_bytes(previous_secret) + config = json.loads(world.settings.webhook_config_file.read_text(encoding="utf-8")) # type: ignore[union-attr] + config["sources"]["hr-primary"]["previous_key"] = { + "key_id": "previous-key", + "secret_file": str(previous_file.resolve()), + "accept_until": "2099-01-01T00:00:00Z", + } + world.settings.webhook_config_file.write_text(json.dumps(config), encoding="utf-8") # type: ignore[union-attr] + client = TestClient(create_app(world.settings)) + raw_body = canonical_json(_event()).encode() + headers = _headers(previous_secret, raw_body, delivery_id="delivery-0400") + headers["OffboardProof-Key-Id"] = "previous-key" + accepted = client.post("/v1/intake/webhooks/hr-primary", content=raw_body, headers=headers) + assert accepted.status_code == 202 + + changed_raw_body = raw_body + b"\n" + invalid = client.post( + "/v1/intake/webhooks/hr-primary", + content=changed_raw_body, + headers={ + **_headers(current_secret, raw_body, delivery_id="delivery-0401"), + "OffboardProof-Key-Id": "current-key", + }, + ) + assert invalid.status_code == 401 From e6cc893527514cd1ad736c8fc9b718a2c4e8f0ed Mon Sep 17 00:00:00 2001 From: DevFoundry-labs Date: Thu, 20 Aug 2026 17:30:41 +0400 Subject: [PATCH 3/3] Fix secure signing fixtures on Linux --- tests/conftest.py | 9 +++++++++ tests/test_backup.py | 8 ++++---- tests/test_evidence_v2.py | 9 +++++---- tests/test_security_edges.py | 16 ++++++++++++---- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e2da796..bb53629 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,10 @@ from __future__ import annotations +import os from collections.abc import Iterator from dataclasses import dataclass from datetime import UTC, datetime, timedelta +from pathlib import Path import pytest @@ -15,6 +17,13 @@ from offboardproof.service import WorkflowService +def write_owner_only(path: Path, value: bytes) -> None: + """Write secret test material with the same permissions production requires.""" + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(value) + + @dataclass class World: settings: Settings diff --git a/tests/test_backup.py b/tests/test_backup.py index 68a4e50..fe3ba56 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -5,7 +5,7 @@ from tempfile import TemporaryDirectory import pytest -from conftest import World +from conftest import World, write_owner_only from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from offboardproof.backup import BackupService @@ -78,7 +78,7 @@ def test_backup_detects_changed_and_unsafe_inventory_entries(world: World) -> No def test_signed_backup_verifies_public_trust_material(world: World) -> None: private_key = Ed25519PrivateKey.generate() key_path = world.settings.database_path.parent / "backup-signing.key" - key_path.write_bytes(private_key.private_bytes_raw()) + write_owner_only(key_path, private_key.private_bytes_raw()) world.settings.signing_key_file = key_path.resolve() world.settings.signing_key_id = "backup-signing-key" world.settings.require_signed_evidence = True @@ -113,7 +113,7 @@ def test_backup_rejects_unsafe_destinations_and_extra_files(world: World) -> Non def test_backup_preserves_public_keys_for_historical_rotation(world: World) -> None: first_key = Ed25519PrivateKey.generate() first_path = world.settings.database_path.parent / "first.key" - first_path.write_bytes(first_key.private_bytes_raw()) + write_owner_only(first_path, first_key.private_bytes_raw()) world.settings.signing_key_file = first_path.resolve() world.settings.signing_key_id = "rotation-first" world.settings.require_signed_evidence = True @@ -121,7 +121,7 @@ def test_backup_preserves_public_keys_for_historical_rotation(world: World) -> N second_key = Ed25519PrivateKey.generate() second_path = world.settings.database_path.parent / "second.key" - second_path.write_bytes(second_key.private_bytes_raw()) + write_owner_only(second_path, second_key.private_bytes_raw()) world.settings.signing_key_file = second_path.resolve() world.settings.signing_key_id = "rotation-second" _complete(world, key="backup-rotation-second") diff --git a/tests/test_evidence_v2.py b/tests/test_evidence_v2.py index 89a9ed3..b9be737 100644 --- a/tests/test_evidence_v2.py +++ b/tests/test_evidence_v2.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from conftest import World +from conftest import World, write_owner_only from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey @@ -37,12 +37,13 @@ def _complete(world: World, *, key: str) -> str: def _configure_key(world: World, tmp_path: Path) -> tuple[Ed25519PrivateKey, Path]: private_key = Ed25519PrivateKey.generate() key_file = tmp_path / "evidence-signing.key" - key_file.write_bytes( + write_owner_only( + key_file, private_key.private_bytes( serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption(), - ) + ), ) world.settings.signing_key_file = key_file.resolve() world.settings.signing_key_id = "pilot-2026-q3" @@ -149,7 +150,7 @@ def test_signing_key_rotation_preserves_historical_manifest(world: World, tmp_pa replacement = Ed25519PrivateKey.generate() replacement_file = tmp_path / "replacement.key" - replacement_file.write_bytes(replacement.private_bytes_raw()) + write_owner_only(replacement_file, replacement.private_bytes_raw()) replacement_public = tmp_path / "replacement.pub" replacement_public.write_bytes(replacement.public_key().public_bytes_raw()) world.settings.signing_key_file = replacement_file.resolve() diff --git a/tests/test_security_edges.py b/tests/test_security_edges.py index 02c5af6..2fde070 100644 --- a/tests/test_security_edges.py +++ b/tests/test_security_edges.py @@ -1,10 +1,12 @@ from __future__ import annotations import json +import os from datetime import UTC, datetime, timedelta from pathlib import Path import pytest +from conftest import write_owner_only from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey @@ -19,7 +21,7 @@ def test_signing_key_configuration_formats_and_failures(tmp_path: Path) -> None: private_key = Ed25519PrivateKey.generate() raw_file = tmp_path / "raw.key" - raw_file.write_bytes(private_key.private_bytes_raw()) + write_owner_only(raw_file, private_key.private_bytes_raw()) settings = Settings(signing_key_file=raw_file.resolve(), signing_key_id="raw-key") loaded = load_signing_key(settings) assert loaded is not None and loaded.key_id == "raw-key" @@ -40,21 +42,27 @@ def test_signing_key_configuration_formats_and_failures(tmp_path: Path) -> None: password = b"correct horse battery staple" encrypted = tmp_path / "encrypted.pem" - encrypted.write_bytes( + write_owner_only( + encrypted, private_key.private_bytes( serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.BestAvailableEncryption(password), - ) + ), ) password_file = tmp_path / "password.txt" - password_file.write_bytes(password + b"\n") + write_owner_only(password_file, password + b"\n") encrypted_settings = Settings( signing_key_file=encrypted.resolve(), signing_key_password_file=password_file.resolve(), signing_key_id="encrypted-key", ) assert load_signing_key(encrypted_settings) is not None + if os.name != "nt": + raw_file.chmod(0o644) + with pytest.raises(ConfigurationError, match="permissions"): + load_signing_key(settings) + raw_file.chmod(0o600) password_file.write_text("wrong", encoding="utf-8") with pytest.raises(ConfigurationError, match="could not be loaded"): load_signing_key(encrypted_settings)