diff --git a/.github/workflows/dependabot-regen.yml b/.github/workflows/dependabot-regen.yml index 2a925678..73dea33d 100644 --- a/.github/workflows/dependabot-regen.yml +++ b/.github/workflows/dependabot-regen.yml @@ -4,7 +4,7 @@ name: Dependabot Regen # dependency descriptors (`pyproject.toml`, `uv.lock`). Regenerates # the per-service `requirements.txt` files via # `scripts/gen_requirements.py` and commits the diff back onto the -# same PR branch, so Py Build's `gen_requirements.py --check` step +# same PR branch, so Inference Build's `gen_requirements.py --check` step # passes without a manual follow-up. # # **Why not GITHUB_TOKEN**: a push signed by the built-in workflow diff --git a/.github/workflows/py-build.yml b/.github/workflows/inference-build.yml similarity index 94% rename from .github/workflows/py-build.yml rename to .github/workflows/inference-build.yml index b5a81166..8f2dc20a 100644 --- a/.github/workflows/py-build.yml +++ b/.github/workflows/inference-build.yml @@ -1,4 +1,4 @@ -name: Py Build +name: Inference Build on: push: @@ -9,7 +9,7 @@ on: - "packages/**" - "pyproject.toml" - "uv.lock" - - ".github/workflows/py-build.yml" + - ".github/workflows/inference-build.yml" pull_request: branches: - "main" @@ -18,7 +18,7 @@ on: - "packages/**" - "pyproject.toml" - "uv.lock" - - ".github/workflows/py-build.yml" + - ".github/workflows/inference-build.yml" # Weekly real-model run (the real-models job is gated on non-PR events below). schedule: - cron: "0 7 * * 1" diff --git a/.github/workflows/py-release.yml b/.github/workflows/inference-release.yml similarity index 99% rename from .github/workflows/py-release.yml rename to .github/workflows/inference-release.yml index 671ed9d7..868ab81f 100644 --- a/.github/workflows/py-release.yml +++ b/.github/workflows/inference-release.yml @@ -1,4 +1,4 @@ -name: Py Release +name: Inference Release on: push: diff --git a/.github/workflows/py-security.yml b/.github/workflows/inference-security.yml similarity index 89% rename from .github/workflows/py-security.yml rename to .github/workflows/inference-security.yml index b39ac325..f5913d64 100644 --- a/.github/workflows/py-security.yml +++ b/.github/workflows/inference-security.yml @@ -1,4 +1,4 @@ -name: Py Security +name: Inference Security on: push: @@ -9,7 +9,7 @@ on: - "packages/**" - "pyproject.toml" - "uv.lock" - - ".github/workflows/py-security.yml" + - ".github/workflows/inference-security.yml" pull_request: branches: - "main" @@ -18,7 +18,7 @@ on: - "packages/**" - "pyproject.toml" - "uv.lock" - - ".github/workflows/py-security.yml" + - ".github/workflows/inference-security.yml" schedule: - cron: "0 6 * * 1" workflow_dispatch: diff --git a/.github/workflows/rs-build.yml b/.github/workflows/runtime-build.yml similarity index 97% rename from .github/workflows/rs-build.yml rename to .github/workflows/runtime-build.yml index 930b6b79..bfa71837 100644 --- a/.github/workflows/rs-build.yml +++ b/.github/workflows/runtime-build.yml @@ -1,4 +1,4 @@ -name: Rs Build +name: Runtime Build on: push: @@ -9,7 +9,7 @@ on: - "crates/**" - "Cargo.toml" - "Cargo.lock" - - ".github/workflows/rs-build.yml" + - ".github/workflows/runtime-build.yml" pull_request: branches: - "main" @@ -18,7 +18,7 @@ on: - "crates/**" - "Cargo.toml" - "Cargo.lock" - - ".github/workflows/rs-build.yml" + - ".github/workflows/runtime-build.yml" workflow_dispatch: env: diff --git a/.github/workflows/rs-release.yml b/.github/workflows/runtime-release.yml similarity index 98% rename from .github/workflows/rs-release.yml rename to .github/workflows/runtime-release.yml index 42dd5bea..d3928376 100644 --- a/.github/workflows/rs-release.yml +++ b/.github/workflows/runtime-release.yml @@ -1,4 +1,4 @@ -name: Rs Release +name: Runtime Release on: push: diff --git a/.github/workflows/rs-security.yml b/.github/workflows/runtime-security.yml similarity index 90% rename from .github/workflows/rs-security.yml rename to .github/workflows/runtime-security.yml index c164cdd9..060953f2 100644 --- a/.github/workflows/rs-security.yml +++ b/.github/workflows/runtime-security.yml @@ -1,4 +1,4 @@ -name: Rs Security +name: Runtime Security on: push: @@ -10,7 +10,7 @@ on: - "Cargo.toml" - "Cargo.lock" - "deny.toml" - - ".github/workflows/rs-security.yml" + - ".github/workflows/runtime-security.yml" pull_request: branches: - "main" @@ -20,7 +20,7 @@ on: - "Cargo.toml" - "Cargo.lock" - "deny.toml" - - ".github/workflows/rs-security.yml" + - ".github/workflows/runtime-security.yml" schedule: - cron: "0 6 * * 1" workflow_dispatch: diff --git a/README.md b/README.md index 7fb658ef..97301363 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Nvisy Runtime -[![Runtime](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/rs-build.yml?branch=main&label=runtime&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/rs-build.yml) -[![Inference](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/py-build.yml?branch=main&label=inference&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/py-build.yml) +[![Runtime](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/runtime-build.yml?branch=main&label=runtime&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/runtime-build.yml) +[![Inference](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/inference-build.yml?branch=main&label=inference&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/inference-build.yml) Multimodal redaction library and its inference services. @@ -25,8 +25,7 @@ pipeline) embed the engine directly. - **[nvisy-context](crates/nvisy-context/)**: wire schema for reference-data collections (analytic, biometric, document, geospatial, reference, temporal) - **[nvisy-policy](crates/nvisy-policy/)**: wire schema for redaction governance (rules, predicates, retention, audit) - **[nvisy-schema](crates/nvisy-schema/)**: umbrella crate re-exporting `nvisy-context` + `nvisy-policy` alongside plan and file types -- **[nvisy-core](crates/nvisy-core/)**: deployment-side runtime configuration (NER and LLM recognizer lineups, error vocabulary) -- **[nvisy-engine](crates/nvisy-engine/)**: stateless pipeline (decode, analyze, apply) wrapping elide and hosting the per-modality orchestrator +- **[nvisy-engine](crates/nvisy-engine/)**: stateless pipeline (decode, analyze, apply) wrapping elide, hosting the per-modality orchestrator and the deployment-side NER / LLM recognizer configuration - **[elide-bento](crates/elide-bento/)**: BentoML-hosted NER and OCR client implementing elide's recognizer traits Python packages ([`packages/`](packages/)) — BentoML services shipped diff --git a/crates/elide-bento/README.md b/crates/elide-bento/README.md index 047a1b6f..24cf4887 100644 --- a/crates/elide-bento/README.md +++ b/crates/elide-bento/README.md @@ -1,6 +1,6 @@ # elide-bento -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/rs-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/rs-build.yml) +[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/runtime-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/runtime-build.yml) Shared BentoML HTTP client wrapper for elide backends. diff --git a/crates/nvisy-context/README.md b/crates/nvisy-context/README.md index 34108e3a..235f65db 100644 --- a/crates/nvisy-context/README.md +++ b/crates/nvisy-context/README.md @@ -1,6 +1,6 @@ # nvisy-context -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/rs-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/rs-build.yml) +[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/runtime-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/runtime-build.yml) Wire schema for Nvisy contexts: persistent reference-data collections that tell detection *what to look for*. diff --git a/crates/nvisy-engine/README.md b/crates/nvisy-engine/README.md index b872770d..b2ab7f96 100644 --- a/crates/nvisy-engine/README.md +++ b/crates/nvisy-engine/README.md @@ -1,6 +1,6 @@ # nvisy-engine -[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/rs-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/rs-build.yml) +[![Build](https://img.shields.io/github/actions/workflow/status/nvisycom/runtime/runtime-build.yml?branch=main&label=build%20%26%20test&style=flat-square)](https://github.com/nvisycom/runtime/actions/workflows/runtime-build.yml) Stateless multimodal redaction pipeline over elide. @@ -9,10 +9,10 @@ Stateless multimodal redaction pipeline over elide. The engine bundles elide's codec, detection, recognition, redaction, and orchestration layers into a single per-request pipeline. Callers construct an engine paired with the deployment's NER and LLM lineups -(from [`nvisy-core`](../nvisy-core)), then drive documents through -analyze and apply verbs. Every call is self-contained: no persistence, -no HTTP layer, no long-running background tasks. Hosts (a SaaS -backend, a Tauri app, a CLI, an SDK) own their own workflow and +(configured through the `provider` module), then drive documents +through analyze and apply verbs. Every call is self-contained: no +persistence, no HTTP layer, no long-running background tasks. Hosts (a +SaaS backend, a Tauri app, a CLI, an SDK) own their own workflow and storage on top. Owns the per-request orchestrator constructor that wires elide's diff --git a/crates/nvisy-engine/src/analyzer/recognizer/guardrails.rs b/crates/nvisy-engine/src/analyzer/recognizer/guardrails.rs index d52bc712..c60f979c 100644 --- a/crates/nvisy-engine/src/analyzer/recognizer/guardrails.rs +++ b/crates/nvisy-engine/src/analyzer/recognizer/guardrails.rs @@ -15,12 +15,11 @@ use nvisy_schema::plan::MAX_REGEX_SOURCE_LEN; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -/// Deployment-side caps applied to -/// [`PatternRecognizerParams`]-driven recognizer builds. +/// Deployment-side caps on pattern-recognizer builds. /// -/// Loaded from the deployment's `[pattern.guardrails]` config -/// section; passed into the engine via -/// [`Engine::with_pattern_guardrails`]. +/// Applied to [`PatternRecognizerParams`]-driven builds. Loaded +/// from the deployment's `[pattern.guardrails]` config section; +/// passed into the engine via [`Engine::with_pattern_guardrails`]. /// /// [`PatternRecognizerParams`]: nvisy_schema::plan::PatternRecognizerParams /// [`Engine::with_pattern_guardrails`]: crate::Engine::with_pattern_guardrails diff --git a/crates/nvisy-engine/src/error.rs b/crates/nvisy-engine/src/error.rs deleted file mode 100644 index 309e3d87..00000000 --- a/crates/nvisy-engine/src/error.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Unified error types for the nvisy platform. -//! -//! All crates in the nvisy workspace use [`Error`] as their primary error -//! type and [`ErrorKind`] to classify failures. -//! -//! Construction goes through [`Error::new`] or one of the per-kind -//! shorthand fns ([`Error::validation`], [`Error::not_found`], …). -//! There is one helper per [`ErrorKind`] variant. Each shorthand -//! seeds the `retryable` flag with the sensible default for that -//! kind (timeouts and transient connection failures default to -//! retryable; everything else defaults to non-retryable). -//! -//! After construction, attach context via the builder methods: -//! [`with_source`], [`with_component`], [`with_retryable`]. Read -//! state back via the accessors ([`kind`], [`message`], [`component`], -//! [`is_retryable`]); the underlying cause is reachable through the -//! standard [`Error::source`] method. -//! -//! [`Error::source`]: std::error::Error::source -//! -//! [`with_source`]: Error::with_source -//! [`with_component`]: Error::with_component -//! [`with_retryable`]: Error::with_retryable -//! [`kind`]: Error::kind -//! [`message`]: Error::message -//! [`component`]: Error::component -//! [`is_retryable`]: Error::is_retryable - -use std::borrow::Cow; -use std::{error, io, result}; - -use elide_core::{Error as ElideError, ErrorKind as ElideErrorKind}; -use strum::Display; - -/// Trait-object alias for the [`Error`] cause chain. -/// -/// Wraps any `std::error::Error` that's safe to send across -/// threads: the usual bound for error sources in async code. -pub type ErrorSource = Box; - -/// Classification of error kinds. -/// -/// Used to tag every [`Error`] so callers can programmatically decide -/// how to handle a failure (e.g. retry on `Timeout`, surface to user -/// on `Validation`). Grouped by failure domain: -/// -/// - **Domain failures** (`Validation`, `Policy`, `NotFound`, -/// `Conflict`). The operation was well-formed but rejected by domain -/// logic. -/// - **Transport failures** (`Connection`, `Timeout`, `Cancellation`). -/// The operation never completed because the channel failed; often -/// retryable. -/// - **Infrastructure failures** (`Internal`, `Runtime`, `Serialization`). -/// Something inside the process or its immediate dependencies broke; -/// not the caller's fault. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Display)] -#[strum(serialize_all = "snake_case")] -pub enum ErrorKind { - /// Input or configuration failed validation checks. - Validation, - /// A policy rule was violated. - Policy, - /// The requested resource was not found. - NotFound, - /// The operation conflicts with the resource's current state. - /// - /// E.g. "already in terminal state", "cannot delete while - /// running". Maps to HTTP 409. Non-retryable: the caller has - /// to inspect the resource and pick a different operation. - Conflict, - /// Could not connect to an external service. - Connection, - /// An operation exceeded its time limit. - Timeout, - /// The operation was explicitly cancelled. - Cancellation, - /// An internal infrastructure error (filesystem, I/O, fjall). - Internal, - /// An internal runtime error inside an engine operation. - Runtime, - /// A serialization or encoding error. - Serialization, -} - -/// Unified error type for the nvisy platform. -/// -/// Carries a [`kind`], a human-readable [`message`], an optional -/// [`component`] tag identifying the producer (e.g. `"detection"`, -/// `"registry"`), an [`is_retryable`] flag, and an optional wrapped -/// source error reachable through [`Error::source`]. -/// -/// Fields are private; construct with [`Error::new`] or a per-kind -/// shorthand, then layer context via [`with_source`], [`with_component`], -/// [`with_retryable`]. -/// -/// [`kind`]: Self::kind -/// [`message`]: Self::message -/// [`component`]: Self::component -/// [`is_retryable`]: Self::is_retryable -/// [`with_source`]: Self::with_source -/// [`with_component`]: Self::with_component -/// [`with_retryable`]: Self::with_retryable -/// [`Error::source`]: std::error::Error::source -#[derive(Debug, thiserror::Error)] -#[error("{kind}: {message}")] -pub struct Error { - kind: ErrorKind, - message: String, - component: Option>, - retryable: bool, - #[source] - source: Option, -} - -impl Error { - /// Construct an error with the given kind and message. - /// - /// No component, no source, `retryable = false`. Prefer the - /// per-kind shorthand fns ([`Self::validation`], - /// [`Self::timeout`], …) when one matches; they set the right - /// `retryable` default for that kind. - pub fn new(kind: ErrorKind, message: impl Into) -> Self { - Self { - kind, - message: message.into(), - component: None, - retryable: false, - source: None, - } - } - - /// Attach an underlying cause to this error. - /// - /// Reachable downstream via [`Error::source`]. - /// - /// [`Error::source`]: std::error::Error::source - pub fn with_source(mut self, source: impl error::Error + Send + Sync + 'static) -> Self { - self.source = Some(Box::new(source)); - self - } - - /// Tag this error with the name of the producer component. - /// - /// E.g. `"detection"`, `"registry"`, `"ocr-bento"`. Accepts - /// `&'static str` (zero-alloc) or `String` (when the name is - /// computed at runtime). - pub fn with_component(mut self, component: impl Into>) -> Self { - self.component = Some(component.into()); - self - } - - /// Override the retryable flag. - /// - /// Per-kind shorthand fns set sensible defaults; use this - /// only when the call site has information the kind alone - /// can't express. - pub fn with_retryable(mut self, retryable: bool) -> Self { - self.retryable = retryable; - self - } - - /// The error's kind classification. - pub fn kind(&self) -> ErrorKind { - self.kind - } - - /// The human-readable message. - pub fn message(&self) -> &str { - &self.message - } - - /// The producer component tag, if attached. - pub fn component(&self) -> Option<&str> { - self.component.as_deref() - } - - /// Whether the operation that failed can be safely retried. - pub fn is_retryable(&self) -> bool { - self.retryable - } - - /// Validation failure. Non-retryable. - /// - /// Caller's input or configuration was rejected by domain - /// logic. - pub fn validation(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::Validation, message).with_component(component) - } - - /// Policy violation. Non-retryable. - /// - /// Detected data conflicts with an active policy rule. - pub fn policy(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::Policy, message).with_component(component) - } - - /// Resource not found. Non-retryable. - pub fn not_found(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::NotFound, message).with_component(component) - } - - /// Resource-state conflict. Non-retryable. - /// - /// E.g. an operation that requires a different status. Maps - /// to HTTP 409. - pub fn conflict(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::Conflict, message).with_component(component) - } - - /// Connection failure to an external service. - /// - /// `retryable` is caller-determined: transient network - /// glitches are retryable, permanent auth failures are not. - pub fn connection( - message: impl Into, - component: impl Into>, - retryable: bool, - ) -> Self { - Self::new(ErrorKind::Connection, message) - .with_component(component) - .with_retryable(retryable) - } - - /// Timeout. Always retryable. - pub fn timeout(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::Timeout, message) - .with_component(component) - .with_retryable(true) - } - - /// Explicit cancellation. Non-retryable. - /// - /// By definition the caller asked us to stop. - pub fn cancellation( - message: impl Into, - component: impl Into>, - ) -> Self { - Self::new(ErrorKind::Cancellation, message).with_component(component) - } - - /// Internal infrastructure failure (filesystem, I/O, database). - /// - /// Non-retryable by default: most internal failures need - /// investigation, not retry. - pub fn internal(message: impl Into, component: impl Into>) -> Self { - Self::new(ErrorKind::Internal, message).with_component(component) - } - - /// Runtime failure inside an engine operation. - /// - /// `retryable` is caller-determined: an LLM rate-limit is - /// retryable, a compile-time pattern error is not. - pub fn runtime( - message: impl Into, - component: impl Into>, - retryable: bool, - ) -> Self { - Self::new(ErrorKind::Runtime, message) - .with_component(component) - .with_retryable(retryable) - } - - /// Serialization / encoding failure. Non-retryable. - pub fn serialization( - message: impl Into, - component: impl Into>, - ) -> Self { - Self::new(ErrorKind::Serialization, message).with_component(component) - } -} - -impl From for Error { - fn from(err: io::Error) -> Self { - Self::internal(err.to_string(), "io").with_source(err) - } -} - -impl From for Error { - fn from(err: serde_json::Error) -> Self { - Self::serialization(err.to_string(), "serde_json").with_source(err) - } -} - -impl From for Error { - fn from(err: derive_builder::UninitializedFieldError) -> Self { - Self::validation( - format!("missing required field `{}`", err.field_name()), - "derive_builder", - ) - } -} - -impl From for Error { - /// Map elide's per-operation error into the runtime's shared vocabulary. - /// - /// The elide `ErrorKind` is preserved semantically via a - /// mapping onto the nearest [`ErrorKind`] variant; the - /// original elide error travels along as the source cause. - /// - /// Called at every `nvisy-engine` seam where an `elide::Error` - /// crosses into engine-land — pattern compile, recognizer - /// build, anonymizer attach, orchestrator analyze/anonymize. - fn from(err: ElideError) -> Self { - let kind = match err.kind() { - ElideErrorKind::OutOfRange | ElideErrorKind::Validation => ErrorKind::Validation, - ElideErrorKind::Transport => ErrorKind::Connection, - _ => ErrorKind::Runtime, - }; - Self::new(kind, err.to_string()).with_source(err) - } -} - -/// Convenience type alias for results using the Nvisy error type. -pub type Result = result::Result; diff --git a/crates/nvisy-engine/src/lib.rs b/crates/nvisy-engine/src/lib.rs index 4a4cc4c4..d3cb5c33 100644 --- a/crates/nvisy-engine/src/lib.rs +++ b/crates/nvisy-engine/src/lib.rs @@ -21,12 +21,17 @@ mod analyzer; mod anonymizer; -mod error; mod pipeline; mod provider; +#[doc(inline)] +pub use elide::recognition::Scope; +#[doc(inline)] +pub use elide_core::entity::Entity; +#[doc(inline)] +pub use elide_core::{Error, ErrorKind, Result}; + pub use self::analyzer::PatternGuardrails; -pub use self::error::{Error, ErrorKind, ErrorSource, Result}; pub use self::pipeline::{ AnalyzedDocument, AnonymizedDocument, Engine, EntityRecord, RecognizedGroup, }; diff --git a/crates/nvisy-engine/src/pipeline/mod.rs b/crates/nvisy-engine/src/pipeline/mod.rs index 1df23c61..53537b53 100644 --- a/crates/nvisy-engine/src/pipeline/mod.rs +++ b/crates/nvisy-engine/src/pipeline/mod.rs @@ -91,6 +91,7 @@ use elide_core::modality::image::Image; #[cfg(feature = "internal_tabular")] use elide_core::modality::tabular::Tabular; use elide_core::modality::text::Text; +use elide_core::{Error, ErrorKind, Result}; use nvisy_schema::context::Context; use nvisy_schema::file::Document; use nvisy_schema::plan::AnalyzerParams; @@ -99,11 +100,9 @@ use uuid::Uuid; pub use self::analyzed::{AnalyzedDocument, EntityRecord, RecognizedGroup}; use self::report::{take_body, take_part}; +use crate::PatternGuardrails; use crate::provider::llm::LlmConfig; use crate::provider::ner::NerConfig; -use crate::{Error, PatternGuardrails, Result}; - -const COMPONENT: &str = "pipeline"; /// Cheaply-cloneable pipeline adapter over [`elide`]. /// @@ -239,12 +238,12 @@ impl Engine { let body_group = body_group.or_else(|| take_body::