diff --git a/.opencode/.gitignore b/.opencode/.gitignore index c072cfe070..fa791daac7 100644 --- a/.opencode/.gitignore +++ b/.opencode/.gitignore @@ -5,3 +5,5 @@ bun.lock .gitignore package-lock.json references/ +!skills/snowflake-setup/references/ +!skills/snowflake-setup/references/** diff --git a/.opencode/skills/snowflake-setup/SKILL.md b/.opencode/skills/snowflake-setup/SKILL.md new file mode 100644 index 0000000000..395eaca8a8 --- /dev/null +++ b/.opencode/skills/snowflake-setup/SKILL.md @@ -0,0 +1,428 @@ +--- +name: snowflake-setup +description: Bootstrap or audit a Snowflake account end-to-end — topology, RBAC, ingestion, PII governance, cost controls, DR, sharing, Cortex. Supports greenfield setup, brownfield audit + remediation, and hybrid mode. Emits idempotent SQL, HCL/Terraform, and rollback scripts. +--- + +# Snowflake Setup and Audit + +Guide the user through standing up a production-ready Snowflake account, or auditing an existing one and remediating gaps. Produces DDL (or Terraform HCL) grouped by executing role, a rollback script, and a post-setup checklist. + +## Requirements + +**Agent:** any +**Tools used:** +- **Baseline:** `read`, `write`, `bash` +- **Connection lifecycle:** `warehouse_list`, `warehouse_add`, `warehouse_discover` (auto-detect existing accounts), `warehouse_test` (validate role + warehouse + database access) +- **Query execution:** `sql_execute` (required for audit; optional for greenfield `guided-execute`) +- **Audit intelligence:** `finops_analyze_credits` (credit breakdown + anomalies), `finops_role_hierarchy` + `finops_role_grants` + `finops_user_roles` (role tree, per-role grants, per-user role assignments — use all three together for the RBAC audit), `finops_warehouse_advice` (data-driven sizing) +- **Live schema scanning:** `schema_inspect` (inspect a specific table's columns/types on the target warehouse), `schema_detect_pii` (scan warehouse/schema/table for PII columns via heuristic + sampling) +- **Schema-file analysis (offline, not against live account):** `altimate_core_classify_pii` (classify PII in a YAML/JSON schema definition — usable in greenfield mode when the user provides a schema file, or in audit mode after exporting live schema via `schema_inspect`), `altimate_core_grade` (grade a SQL query against a schema), `altimate_core_policy` (check a SQL query against a policy JSON), `altimate_core_export_ddl` (emit DDL from a schema definition) +**References:** +- `references/topology-patterns.md` — database and warehouse topologies (Medallion, Functional, Domain-per-DB, Data Vault 2.0) +- `references/data-vault-patterns.md` — hub/link/satellite modeling, loader patterns, DV2 RBAC and governance (loaded only when DV2 topology selected) +- `references/rbac-patterns.md` — roles, grants, service accounts +- `references/ingestion-patterns.md` — Snowpipe, Task+COPY, Fivetran, Snowpipe Streaming +- `references/governance-patterns.md` — masking, row access, tags +- `references/cost-governance.md` — resource monitors, cost attribution +- `references/idempotency-patterns.md` — emission modes and rollback rules +- `references/audit-queries.md` — brownfield diagnostic queries +- `references/terraform-mapping.md` — DDL → HCL mapping +- `references/advanced-features.md` — DR, sharing, Cortex, network/SSO, cost forecasting + +## When to Use This Skill + +**Use when the user wants to:** +- Bootstrap a brand-new Snowflake account +- Audit an existing account for gaps (missing FUTURE grants, unmasked PII, no resource monitors, orphaned roles, etc.) and generate a remediation plan +- Extend a partially-configured account with missing pieces (RBAC, cost controls, governance) +- Get IaC (Terraform) equivalents of the setup +- Produce a rollback plan before a risky setup change + +**Do NOT use for:** +- Optimizing a single warehouse or query → `query-optimize` +- Investigating cost anomalies day-to-day → `cost-report` +- Migrating queries from another warehouse → `sql-translate` + `schema-migration` +- PII detection alone → `pii-audit` + +## Workflow + +### 1. Auto-detect Existing Accounts + Confirm Mode + +Before asking anything, call `warehouse_discover` to enumerate warehouses the user may already have configured elsewhere (e.g. via Datamate, `~/.snowsql/config`, or environment variables). If a Snowflake connection is detected, mention it in the mode question so the user can choose knowingly: + +> "I detected an existing Snowflake connection: ` / / `. Are you starting fresh (`greenfield`), auditing this one (`audit`), or extending it (`hybrid`)?" + +If nothing is detected, proceed with the standard mode question below. + +**First triage question** — always asked: + +- `greenfield` — brand-new account, generate full setup from scratch +- `audit` — scan existing account, produce scored gap report + remediation DDL +- `hybrid` — audit existing account, then extend it toward a target topology + +Audit and hybrid modes **require** a configured Snowflake warehouse the agent can reach via `sql_execute`. The signed-in Snowflake user must have `ACCOUNTADMIN` or a role granted `IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE`. See the warehouse-preflight step below for how the skill discovers or registers a connection — do not fall back to greenfield if audit is chosen but no warehouse exists. + +### 2. Gather Requirements (Greenfield / Hybrid) + +Ask all blocking questions **in one batch** — do not proceed until answered. + +**Blocking:** + +1. **Topology?** — Medallion / Functional / Domain-per-Database / Data Vault 2.0 (see `topology-patterns.md`, and `data-vault-patterns.md` if DV2 chosen) +2. **RBAC model?** — small-team single-layer / large-team functional+access two-layer (see `rbac-patterns.md`) +3. **Environments?** — prod-only / prod+dev / prod+staging+dev +4. **Ingestion sources?** (multi-select) — Fivetran or Airbyte, Snowpipe (event-driven), Task+COPY (batch), Snowpipe Streaming (Kafka), CDC from operational DBs +5. **Cloud provider for external stages?** — AWS S3 / GCS / Azure Blob +6. **Emission mode?** — `strict` / `idempotent` (default) / `additive` (see `idempotency-patterns.md`) +7. **Output format?** — `sql` (default) / `terraform` / `both` + +**Optional (defaults available):** + +8. **Monthly credit budget?** — number, or `forecast` to trigger the cost-forecasting workflow (see `advanced-features.md` § Cost Forecasting) +9. **PII discovery mode?** — `declared` (user lists categories) / `discover` (call `schema_detect_pii` on RAW schemas after they're populated; optionally cross-check with `altimate_core_classify_pii` if a schema file is provided) / `both` +10. **Multi-tenant data?** — yes/no; if yes, plan row access policies +11. **Existing account or new signup?** — affects ACCOUNTADMIN bootstrap DDL + +**Advanced-feature triggers (each opt-in):** + +12. **Restrict access by IP?** — enables network policy section +13. **Federate authentication (SSO)?** — Okta / Azure AD / Google Workspace / none +14. **Enable Cortex (LLM functions, Cortex Search)?** — yes/no +15. **Cross-region disaster recovery?** — none / replication-only / failover-group +16. **Data sharing?** — none / outbound-share / reader-account + +### 2b. Warehouse Preflight (Audit / Hybrid, and Greenfield if `guided-execute`) + +Before running any diagnostic query or execute step, verify a Snowflake warehouse connection is configured: + +1. Call `warehouse_list` and look for an entry with `type: snowflake` +2. If **none is found**: + - Fail fast for `audit` mode — tell the user the exact next step: "Run `warehouse_add` with your Snowflake account, user, role (ACCOUNTADMIN or one with `IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE`), warehouse, and either password or private-key path. Rerun `/snowflake-setup` once configured." + - For `hybrid`, offer the same. Do not silently fall back to greenfield. + - For `greenfield` with `guided-execute` selected, offer to either (a) add the warehouse now via `warehouse_add`, or (b) downgrade the execution path to `review-only`. +3. If **multiple Snowflake warehouses** exist, prompt the user to pick one before proceeding. Present the list with the fields returned by `warehouse_list` (name, account, role, warehouse) so the user can distinguish between them — never guess or auto-select even if one looks "primary". Wait for an explicit selection; do not run any query until the user answers. Remember the selection for every subsequent `sql_execute` call in this session. +4. If **exactly one Snowflake warehouse** exists, use it — but surface which one you're using ("Using Snowflake warehouse: `` (account: ``, role: ``)") so the user can catch a wrong-account mistake before any query runs. +5. Confirm the selected warehouse works before proceeding: call `warehouse_test` with the chosen warehouse. This validates connectivity, role permissions, warehouse USAGE, and database access as a single package (superior to a bare `SELECT CURRENT_ACCOUNT()` because it catches missing grants that a scalar query wouldn't). Surface the result. If it fails, do not proceed — surface the error and ask the user to fix credentials via `warehouse_add`. +6. Cache the chosen warehouse name (and current role) for the rest of the workflow; every downstream `sql_execute` call must target it explicitly. + +### 3. (Audit / Hybrid) Run Diagnostic Queries + +Run each section of `references/audit-queries.md` via `sql_execute` (targeting the warehouse chosen in step 2b). **Prefer altimate tools over hand-written SQL** where they cover the same ground — they return structured data and their queries are versioned in the altimate-core: + +| Section | Tool to use first | Fall back to raw SQL when | +|---------|-------------------|----------------------------| +| 1. Topology and databases | `sql_execute` (queries from `audit-queries.md`) | Always — no dedicated tool | +| 2. Warehouses | `sql_execute` | Always — no dedicated tool | +| 3. RBAC | **`finops_role_hierarchy`** (role tree) + **`finops_role_grants`** (per-role object grants) + **`finops_user_roles`** (which users have which roles) — use all three to find orphaned roles, over-privileged roles, and DEFAULT_ROLE = ACCOUNTADMIN | Any tool unavailable or returned partial data | +| 4. Ingestion | `sql_execute` | Always — no dedicated tool | +| 5. Governance (PII) | Handled in step 5 (PII Discovery) — do not duplicate here | — | +| 6. Cost controls | **`finops_analyze_credits`** — credit breakdown, anomaly detection, per-warehouse spend | Tool unavailable | +| 7. Network and security | `sql_execute` | Always — no dedicated tool | +| 8. Data sharing and replication | `sql_execute` | Always — no dedicated tool | + +For each finding, score CRITICAL / WARNING / INFO per the severity rubric. Compute maturity score: + +``` +maturity = 100 - (CRITICAL × 10) - (WARNING × 3) - (INFO × 1) +``` + +Deliver the audit report as a table sorted by severity: + +``` +CRITICAL FINDINGS (must fix before production): + [RBAC] 3 users have DEFAULT_ROLE = ACCOUNTADMIN + → alice, bob, service_x + Remediation: ALTER USER SET DEFAULT_ROLE = ; + + [GOVERNANCE] 12 columns look like PII but have no masking + → RAW.SALESFORCE.CONTACTS.email, RAW.SALESFORCE.CONTACTS.phone, ... + Remediation: apply mask_email / mask_phone from governance-patterns.md + + [COST] 2 warehouses have no resource monitor + → TRANSFORM_WH, DEV_WH + Remediation: see cost-governance.md § Resource Monitor Architecture + +WARNING FINDINGS: + ... + +INFO FINDINGS: + ... + +Maturity: 63/100 (< 70: requires remediation before relying on for production) +``` + +**Maturity score is authoritative for this skill.** (Earlier drafts proposed a second-opinion via `altimate_core_grade`, but that tool grades SQL queries against a schema — not audit findings. No second-opinion mechanic is needed; keep the CRITICAL/WARNING/INFO rubric.) + +In hybrid mode, use audit findings to seed the greenfield questionnaire — pre-fill answers based on what already exists (topology detected, ingestion sources present, existing warehouse sizes from `finops_analyze_credits`, role structure from `finops_role_hierarchy` + `finops_role_grants`) and only ask for gaps. + +### 4. Generate the Plan + +Produce a single markdown plan with the following sections, in this order. For each section not needed given the user's choices, state "Not needed because …" explicitly — do not silently skip. + +1. **Databases and Schemas** ← `topology-patterns.md` (placeholder-driven; see 4a). **If topology = `data-vault-2`**, also emit `RAW_VAULT`, `BUSINESS_VAULT`, `INFO_MARTS` per `data-vault-patterns.md`, plus the HUBS/LINKS/SATELLITES schema pattern inside `RAW_VAULT`. +2. **Warehouses** ← `topology-patterns.md` § Warehouse Sizing Guide (placeholder-driven; see 4a). **In hybrid mode**, call `finops_warehouse_advice` on the existing account and use its recommendations to override the static sizing table where they differ. Present the delta to the user before emitting. **If topology = `data-vault-2`**, apply the DV2 cost adjustment table from `data-vault-patterns.md` (typically 2–3× LOADING_WH and TRANSFORM_WH baselines). +3. **RBAC** ← `rbac-patterns.md` (placeholder-driven; see 4a). **In hybrid mode**, use `finops_role_hierarchy` + `finops_role_grants` output to detect existing roles and only emit DDL for missing ones. **If topology = `data-vault-2`**, add the `VAULT_LOADER_ROLE`, `BUSINESS_VAULT_BUILDER_ROLE`, and `MART_BUILDER_ROLE` from `data-vault-patterns.md`, and emit the insert-only enforcement `REVOKE UPDATE, DELETE` statements on `RAW_VAULT.*`. +4. **Ingestion** (one subsection per selected source) ← `ingestion-patterns.md` (requires detail questions; see 4b) +5. **Governance** ← `governance-patterns.md` (placeholder-driven for defaults; see PII discovery in step 5). **Before emitting any masking / row-access policy DDL**, verify each target column actually exists by calling `schema_inspect` on the target table (needs the warehouse name from step 2b). If the column is missing or has an unexpected type, refuse to emit that policy and surface the error. **If topology = `data-vault-2`**, apply masking at the layer chosen in the DV2 detail questions (RAW_VAULT satellites / BUSINESS_VAULT+INFO_MARTS / hybrid) per `data-vault-patterns.md` § PII placement. +6. **Cost Controls** ← `cost-governance.md` (placeholder-driven). **In hybrid mode**, seed monitor thresholds from `finops_analyze_credits` (30-day p95 usage × 1.5 = suggested quota). +7. **Environment Promotion** (zero-copy clones) ← `topology-patterns.md` (placeholder-driven) +8. **Network Security + SSO** (if enabled) ← `advanced-features.md` (requires detail questions; see 4b) +9. **Disaster Recovery** (if enabled) ← `advanced-features.md` (requires detail questions; see 4b) +10. **Data Sharing** (if enabled) ← `advanced-features.md` (requires detail questions; see 4b) +11. **Cortex / ML** (if enabled) ← `advanced-features.md` (requires detail questions; see 4b) +12. **Cost Forecast** (if budget = `forecast`) ← `advanced-features.md` (requires detail questions; see 4b) +13. **Rollback Script** ← `idempotency-patterns.md`. **In audit/hybrid mode**, before generating the DROP sequence, enumerate existing objects via `SHOW DATABASES`, `SHOW WAREHOUSES`, `SHOW ROLES`, `SHOW USERS`, `SHOW INTEGRATIONS`, and `SHOW RESOURCE MONITORS` (via `sql_execute`). Cross-reference with the objects the skill created in this session and emit `DROP … IF EXISTS` only for the intersection. This ensures the rollback covers exactly what exists — no more, no less. + + **Delivery split — DROP DATABASE / DROP SCHEMA / TRUNCATE cannot run via altimate-code.** The `sql_execute` tool has a non-bypassable safety guard blocking these three statement types (see `packages/opencode/src/altimate/tools/sql-classify.ts`). This means `guided-execute` **cannot** apply a full rollback on its own. The skill's rollback file must therefore split into two parts: + - `rollback-tool-safe.sql` — every DROP except DATABASE / SCHEMA / TRUNCATE (roles, warehouses, users, monitors, policies, tags, pipes, tasks, stages, integrations). Runs cleanly via `sql_execute`. + - `rollback-manual.sql` — the `DROP DATABASE` and `DROP SCHEMA` statements. Must be executed via `snowsql`, Snowsight SQL Worksheet, or a direct `snowflake-sdk` script by the user. + Emit both files, and tell the user explicitly at the end of the rollback plan: *"After running rollback-tool-safe.sql, execute rollback-manual.sql through Snowsight or snowsql — this file contains DROP DATABASE / DROP SCHEMA statements that altimate-code cannot execute for you."* + +### 4a. Placeholder-Driven Sections + +Sections 1, 2, 3, 5 (non-PII parts), 6, 7 use safe defaults derived from the triage answers. Any value the skill cannot safely default is emitted as a **clearly-marked placeholder** (e.g. ``, ``, ``) with a comment `-- REPLACE BEFORE RUNNING` inline. For Terraform output, use `var.` and declare the variable in `variables.tf` with a description that says what to set. + +Every placeholder must appear in a **"Configure Before Running" checklist** at the top of the delivered SQL / HCL file so users can't miss them. Never emit a bucket name, IAM ARN, IP range, cert body, private key path, or SSO URL as a guess or plausible-looking default — always a placeholder. + +Naming defaults the skill uses without asking (safe because they're conventional and can be renamed later): +- Database names: from topology choice (BRONZE/SILVER/GOLD or RAW/TRANSFORM/ANALYTICS) +- Warehouse names: `_WH` (LOADING_WH, TRANSFORM_WH, ANALYTICS_WH, DEV_WH) +- Role names: `_ROLE` per the RBAC reference topology +- Schema names within RAW: one per selected ingestion source + +### 4b. Detail Questions (Section-by-Section Confirmation) + +For sections where wrong defaults are dangerous (real bucket names, IP ranges, IdP URLs, region choices, target accounts), ask a small batch of detail questions **just before emitting that section**. Present the current plan for the section, then ask. + +**Data Vault 2.0 detail questions (asked only if topology = `data-vault-2`, before Section 1 is emitted):** + +- Which dbt vault package to configure? — `AutomateDV` (recommended) / `dbtvault` (legacy) / `custom` (no package, hand-rolled macros) +- Hash algorithm? — `MD5` (default, faster) / `SHA256` (compliance-grade, slower) +- Business key naming convention? — `BK__ID` (default) / `NK_` / custom pattern +- Which source systems feed the initial vault? (one hub per business entity per source; typical: customer, order, product) +- PII placement — mask in `RAW_VAULT` at load / mask only in `BUSINESS_VAULT` + `INFO_MARTS` (default for regulated industries) / hybrid (column mask in vault + row access in marts) +- Vault refresh cadence — how often should `RAW → RAW_VAULT` run? (default: hourly for hot, nightly for reference data) + +**Section 4 — Ingestion (per selected source):** + +- If Snowpipe or Task+COPY was selected: + - S3 / GCS / Azure Blob URL(s)? (e.g. `s3://my-raw-data/salesforce/`) + - IAM role ARN or GCP service account or Azure app registration for the storage integration? + - Which schemas should each stage point at? (default: one schema per source per `topology-patterns.md`) + - For Task+COPY: cron schedule (default `0 * * * * UTC` = hourly) +- If Fivetran/Airbyte was selected: + - Which sources? (Salesforce, HubSpot, Stripe, GitHub, Postgres, other) + - Fivetran/Airbyte-generated schema names, or use `_` convention? + - Loader service account name (default: `_loader`) +- If Snowpipe Streaming was selected: + - Kafka topic names? + - Which target tables and clustering keys? + +**Section 8 — Network Security + SSO:** + +- If IP restriction was enabled (triage Q12=yes): + - CIDR blocks to allow? (comma-separated list) + - Should the same policy apply to service accounts, or a separate CI-runner policy? +- If SSO was enabled (triage Q13): + - Full IdP issuer URL? (Okta / Azure AD tenant / Google Workspace) + - SSO endpoint URL? + - X509 signing cert (paste body or provide path)? + - Enforce SSO-only login (block password auth for humans)? +- Always ask (regardless of triage answers): + - Enable MFA enforcement for all human users? (recommended: yes) + - Generate key-pair auth for service accounts now, or emit placeholders for user-supplied keys? + +**Section 9 — Disaster Recovery (if triage Q15 ≠ none):** + +- Target region for the replica account? (e.g. `AWS_US_WEST_2`) +- Target account locator (must already exist and be in the same org)? +- Replication schedule? (default: `60 MINUTE` for replication-only, `15 MINUTE` for failover) +- RPO target? (informational — used to validate schedule) +- For failover-group: client connection name? (default: `prod_connection`) + +**Section 10 — Data Sharing (if triage Q16 ≠ none):** + +- Which schemas or specific objects to share? (must be secure views, not raw tables) +- Consumer account identifiers? +- For reader account: reader account name and initial admin password (placeholder, not asked)? + +**Section 11 — Cortex / ML (if triage Q14=yes):** + +- Cortex warehouse size? (default: MEDIUM Snowpark-optimized) +- Which roles need `USAGE` on Cortex functions? (default: ANALYST_ROLE) +- Set up a Cortex Search service? If yes, target table + attribute columns + target lag? + +**Section 12 — Cost Forecast (if triage Q8=forecast):** + +- Expected data volume ingested per day (GB)? +- Expected number of dbt models? +- Expected concurrent BI users? +- Expected ad-hoc analyst sessions per week? +- Expected credit price ($/credit) for cost estimation? (default: $3) + +**In hybrid mode**, populate answers to these detail questions from audit findings where possible (e.g. detected bucket URLs from existing storage integrations, current network policies, current credit consumption from `finops_query_history`). Only ask about deltas. + +### 5. PII Discovery (if `discover` or `both`) + +After the RAW databases and schemas are created (or in audit mode, before generating masking DDL): + +1. Call `schema_detect_pii` with the warehouse from step 2b, scoped to the RAW schema(s). This scans live tables/columns via heuristic + sampling and returns PII candidates with confidence scores. +2. Optionally: if the user has a schema YAML/JSON file (e.g. from dbt sources.yml), also run `altimate_core_classify_pii` with that `schema_path` for a cross-check. +3. Merge results with any user-declared categories. +4. Auto-populate masking policy targets in section 5 (Governance). +5. Report the classification confidence per column in the plan output. + +Fall back gracefully: +- If `schema_detect_pii` returns no results (RAW schemas empty in greenfield), use declared categories only +- If neither `schema_detect_pii` nor `altimate_core_classify_pii` is available, fall back to the name-based heuristic from `audit-queries.md` § Section 5 and note the reduced confidence + +### 6. Emit the DDL / HCL + +Apply the emission mode from question 6 to every `CREATE` statement: + +- `strict` → plain `CREATE` +- `idempotent` (default) → `CREATE IF NOT EXISTS` for allowed objects; use `ALTER` (not `OR REPLACE`) for objects on the danger list in `idempotency-patterns.md` +- `additive` → run existence checks first via `SHOW` / `INFORMATION_SCHEMA`, skip anything present + +Apply the output format from question 7: +- `sql` → grouped SQL blocks (see below) +- `terraform` → HCL files per `terraform-mapping.md` (`providers.tf`, `variables.tf`, one `.tf` per section) +- `both` → both, in separate output files + +**SQL grouping — always** organize by executing role so the user knows which `USE ROLE` to run before each block: + +``` +==== Run as ACCOUNTADMIN ==== +1. Databases and schemas +2. Warehouses +3. Resource monitors (account-level + per-warehouse) +4. Storage integrations (if S3/GCS/Azure) +5. Custom roles +6. Grant custom roles to SYSADMIN (idempotency check: skip if already granted) +7. Network policies (if enabled) +8. Security integrations for SSO/SCIM (if enabled) +9. Replication / failover groups (if DR enabled) +10. Cortex warehouse (if enabled) + +==== Run as SECURITYADMIN ==== +11. Service accounts (loader, dbt, streaming, cortex) +12. DEFAULT_ROLE / DEFAULT_WAREHOUSE bindings +13. Network policy application to service accounts + +==== Run as SYSADMIN ==== +14. Object privileges to functional roles +15. FUTURE privileges (CRITICAL — always include) +16. File formats, stages, pipes, tasks +17. Masking policies and row access policies +18. Apply masking to PII columns (from PII discovery output) +19. Tag taxonomy and tag applications +20. Shares (if data sharing enabled) + +==== Manual (outside Snowflake) ==== +- AWS IAM role trust policy for storage integrations (or GCS / Azure equivalent) +- S3 bucket event notifications → SQS (if Snowpipe) +- Fivetran / Airbyte destination configuration +- IdP-side SCIM / SAML setup (Okta app, Azure AD enterprise app, etc.) +- Client connection URL configuration (if failover group enabled) +``` + +Always append a **validation query pack** the user can run after each block to confirm it worked (queries pulled from the appropriate reference file). + +**Rule: validation queries must only reference objects the emitted DDL actually creates.** If a section's DDL was skipped because it depends on placeholders the user did not fill in (e.g. no S3 bucket URL was provided, so the storage integration / external stage / pipe DDL was omitted), the corresponding validation queries — `SYSTEM$PIPE_STATUS(...)`, `SHOW STAGES`, `DESC INTEGRATION`, `SHOW PIPES`, or `SELECT ... FROM RAW..` — must **also** be omitted. Emitting validation queries for objects that were never created leaves the user chasing false "0 rows" reports on the tail of the file. The skill should track which sections it emitted and pass that flag into the validation-query emission. + +### 7. Emit the Rollback Script + +Regardless of mode or format, always generate a companion rollback SQL file per `idempotency-patterns.md` § Rollback Script Generation: + +- DROPs in strict reverse dependency order (policies → tags → pipes/tasks → stages → tables → schemas → databases → warehouses → resource monitors → grants → roles → users → integrations) +- Every DROP uses `IF EXISTS` +- Never emits `DROP ... CASCADE` +- Never drops built-in roles or the `SNOWFLAKE` database +- SUSPEND tasks and UNSET policies/tags/monitors before dropping +- Interactive confirmation guard at the top requiring the user to paste the account locator before destructive statements execute + +### 8. Execute per the User's Choice + +The execution path was already selected in triage Q3. Do not re-ask; just proceed: + +- `review-only` — hand over the plan + DDL/HCL as files; do not offer to execute +- `guided-execute` — run DDL directly via `sql_execute` against the warehouse from step 2b, section by section, pausing for user confirmation between role switches. If no Snowflake warehouse is configured (step 2b failed), fall back to `review-only` and warn the user. **Caveat:** `sql_execute` cannot run `DROP DATABASE`, `DROP SCHEMA`, or `TRUNCATE` (non-bypassable safety guard). Rollback delivery is therefore split into `rollback-tool-safe.sql` (auto-executable) and `rollback-manual.sql` (user runs via snowsql/Snowsight). See plan section 13. +- `dbt-integrate` — same as `guided-execute` (or `review-only` if no connection), and additionally emit a `profiles.yml` snippet configured for the created databases, warehouses, and service account +- `terraform-apply` — emit HCL files and initialize a working directory with `terraform init`; user runs `terraform plan` / `terraform apply` themselves + +### 9. Post-Setup Checklist + +Regardless of path, remind the user of manual steps that can't be scripted: + +- [ ] Configure AWS / GCS / Azure IAM trust policy for storage integrations (use the `storage_aws_iam_user_arn` and `storage_aws_external_id` outputs) +- [ ] Set up S3 event notifications → SQS (if using Snowpipe) +- [ ] Configure Fivetran / Airbyte destinations with the new loader service account credentials +- [ ] Distribute service-account private keys to CI systems (never commit to source control) +- [ ] Set `DEFAULT_ROLE` on all human users to their least-privilege role — never ACCOUNTADMIN +- [ ] Configure notification integrations for resource monitor alerts (email, Slack via webhook) +- [ ] If PII discovery ran on empty RAW: rerun after first ingestion, then apply any newly-discovered masking policies +- [ ] Test masking as `ANALYST_ROLE` — confirm masked output, not plaintext +- [ ] Set up IdP-side SCIM app (paste SCIM access token from setup output) +- [ ] Test SSO login flow for one human user before enforcing `SSO_LOGIN_PAGE = TRUE` +- [ ] Store the rollback script somewhere retrievable (private ops repo) in case the setup needs to be undone +- [ ] Document topology, RBAC, and DR posture decisions in the team wiki +- [ ] Schedule a re-audit in 30 days (rerun `/snowflake-setup` and choose `audit`) to catch drift + +## Usage + +**Single entry point:** `/snowflake-setup` + +The skill always starts by asking a small set of triage questions to establish scope, then routes into the appropriate workflow. There are no flags, positional args, or command variants — everything is driven by the interactive prompt. + +### Turn 1 — Triage Questions (always ask, in this order) + +Ask these upfront, before reading any references or generating any DDL. Present them as a single batch so the user can answer all at once. + +1. **What kind of work is this?** + - `greenfield` — brand-new Snowflake account, generate full setup + - `audit` — existing account, scan for gaps and produce a scored remediation plan + - `hybrid` — audit first, then extend the account toward a target topology + +2. **Output format?** + - `sql` — grouped executable SQL blocks (default) + - `terraform` — HCL files for the Snowflake-Labs Terraform provider + - `both` — emit both simultaneously + +3. **How much control do you want at execution time?** + - `review-only` — deliver plan + DDL/HCL as files; never offer to execute + - `guided-execute` — after review, offer to run DDL via `sql_execute` against a configured Snowflake warehouse, pausing at each role switch + - `dbt-integrate` — also emit a matching `profiles.yml` snippet + - `terraform-apply` — emit HCL and prepare a `terraform init`-ready working directory (only offered if output format includes terraform) + +### Turn 2 — Mode-Specific Questions + +Based on the answer to triage Q1, ask the appropriate detailed questions: + +- **`greenfield` or `hybrid`** → go to workflow step **2. Gather Requirements** and ask the 5 blocking + 4 optional + 5 feature-trigger questions listed there +- **`audit`** → go to workflow step **2b. Warehouse Preflight** first (verify a Snowflake warehouse is configured via `warehouse_list`; if not, prompt the user to run `warehouse_add` and stop). Then proceed to step **3. Run Diagnostic Queries**. Do not silently degrade to greenfield. + +### Turn 3+ — Execute the Workflow + +Follow workflow steps 4–9 in order. Never batch a workflow step with a triage question; each turn does one thing so the user can course-correct. + +### Rules for the Triage Prompt + +- **Never assume** — always ask Q1 even if the user's opening message hints at a mode. A message like "help me set up Snowflake" could mean greenfield or hybrid. +- **Never ask more than the triage batch on turn 1.** If the user typed a message like "audit my snowflake and generate terraform", accept those as answers to Q1 and Q2 and only ask the remaining triage question(s). +- **Confirm before proceeding** if the user's initial message conflicts with a triage answer (e.g. "greenfield" + "audit my existing"). Clarify which they meant. + +## Guardrails + +Non-negotiable behaviors the skill must enforce: + +1. **No ACCOUNTADMIN to service accounts.** Any DDL granting ACCOUNTADMIN (directly or via role hierarchy) to a service account is a bug — refuse to emit it. +2. **Always include FUTURE grants** alongside object grants — the #1 forgotten step; silent access failures on new tables. +3. **Never `CREATE OR REPLACE`** for WAREHOUSE, ROLE, USER, DATABASE, SCHEMA, TABLE (non-transient), STORAGE INTEGRATION, PIPE, or RESOURCE MONITOR — use `ALTER` instead. See `idempotency-patterns.md` § DANGER. +4. **Storage integration DDL requires ACCOUNTADMIN** and one-time manual IAM setup — flag this in the plan header, not buried in a note. +5. **Prefer key-pair auth over passwords** for all service accounts. If the user asks for passwords, emit the DDL but include the upgrade path in the checklist. +6. **Never drop built-in roles or the `SNOWFLAKE` database** in rollback scripts. +7. **Audit mode requires a live connection.** Warehouse preflight (step 2b) must find a Snowflake warehouse via `warehouse_list` and pass a `SELECT CURRENT_ACCOUNT()` smoke test via `sql_execute` before any diagnostic query runs. Fail fast with `warehouse_add` guidance if not — do not silently degrade. +8. **PII discovery results are advisory, not authoritative.** Present classifications with confidence scores; require explicit user confirmation before applying masking policies to columns flagged as PII. +9. **Rollback scripts are gated.** Interactive confirmation prompt at the top; refuse to run without the account locator matching `CURRENT_ACCOUNT()`. +10. **Terraform state must not contain secrets.** If emitting Terraform, remind the user to use a remote backend and mark sensitive outputs. +11. **One role per GRANT statement — never comma-separate roles.** The comma-separated form (e.g. `GRANT USAGE ON DATABASE X TO ROLE A, B, C`) is silently dropped by the current `snowflake-sdk` driver — the query returns `(0 rows)` with no error and the roles receive nothing. See `references/rbac-patterns.md` § GRANT emission rule. Same rule applies to `REVOKE`. diff --git a/.opencode/skills/snowflake-setup/TESTING.md b/.opencode/skills/snowflake-setup/TESTING.md new file mode 100644 index 0000000000..dcfd28c1dd --- /dev/null +++ b/.opencode/skills/snowflake-setup/TESTING.md @@ -0,0 +1,387 @@ +# Testing the `snowflake-setup` skill + +## Current state + +| Tier | Status | Location | +|------|--------|----------| +| **Tier 1 — prompt contract** | ✅ Implemented (74 tests, ~1s, runs on every PR) | `packages/opencode/test/altimate/snowflake-setup-contract.test.ts` | +| **Tier 2/3 — model-in-the-loop eval** | ✅ Implemented (13 tests, ~90s, opt-in) | `packages/opencode/test/altimate/snowflake-setup-eval.test.ts` | +| **Tier 3 — live Snowflake apply/rollback cycle** | ✅ Executed 2026-08-25 + 2026-08-26 against `DKZPOBS-TQ14188` — see `LIVE-EVAL-RESULTS.md`. Not codified as a repeatable test (would need reusable operator credentials). | `test/altimate/fixtures/snowflake-setup/LIVE-EVAL-RESULTS.md` | + +Run Tier 1 (always safe): +```bash +cd packages/opencode +bun test test/altimate/snowflake-setup-contract.test.ts +``` + +Run Tier 2/3 model-in-the-loop eval (requires Altimate LLM Gateway login, costs API credits): +```bash +cd packages/opencode +SNOWFLAKE_SETUP_EVAL=1 bun test test/altimate/snowflake-setup-eval.test.ts +``` + +The eval test boots real `altimate-code run --yolo`, feeds a scripted greenfield +scenario, captures the emitted SQL files, and grades them against SKILL.md's +non-negotiable guardrails. See §"Tier 2/3 as-implemented" below for details. + +Golden reference files from a prior successful run: +- `packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-medallion.expected.sql` +- `packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-medallion.expected.sql` + +These are for human eyeballing when the eval starts failing — they are not +asserted verbatim (the LLM emits slightly different phrasing each run). + +## What Tier 1 covers + +Tier 1 is a **static analysis of the skill's markdown** — deterministic string / AST assertions that pin non-negotiable invariants. Whitespace-normalized so prose reflow doesn't break tests; substantive removal of a rule does. + +Categories (74 assertions total): + +1. Structural — frontmatter valid, name = `snowflake-setup`, all 10 reference files present +2. Tool references resolve — every `` `tool_name` `` mention matches a `Tool.define("", ...)` under `src/altimate/tools/` +3. Reference file mentions resolve — every `` `references/*.md` `` mention exists on disk +4. No stale names — `snowflake_sql`, `finops_role_access`, `snowflake-greenfield-setup` never appear anywhere in the skill dir +5. Triage flow — mode / output-format / execution-control question options intact +6. Topology options — all 4 present (Medallion / Functional / Domain-per-DB / Data Vault 2.0), DV2 branches wired in plan sections 1/3/5 +7. Emission modes — strict / idempotent (default) / additive present +8. Warehouse preflight — step 2b present with multi-warehouse prompt, `warehouse_test` smoke check, audit fail-fast +9. Guardrails — all 10 non-negotiable guardrails present +10. Rollback safety — `IF EXISTS` on drops, no CASCADE, built-in roles protected, account-locator confirmation guard, full CREATE OR REPLACE danger list of 8 object types +11. DV2 reference completeness — hub/link/satellite patterns, HASHDIFF, insert-only enforcement, 3 DV2 roles, PII placement decision +12. Audit queries — target `sql_execute` (not the hallucinated `snowflake_sql`), severity rubric, maturity formula + +## What Tier 1 does NOT cover + +- Whether emitted DDL is syntactically valid Snowflake SQL +- Whether the audit workflow actually finds real misconfigurations +- Whether Terraform HCL passes `terraform validate` +- Whether the skill loads and runs in the OpenCode session runtime +- Whether tool argument shapes match how the skill instructs the agent to invoke them (verified manually against Zod schemas in an earlier audit; NOT continuously enforced by a test) + +Tiers 2 and 3 are designed to close those gaps. + +--- + +## Tier 2/3 as-implemented (model-in-the-loop) + +`snowflake-setup-eval.test.ts` boots the real `altimate-code run` subcommand +with `--yolo` (auto-approve permissions), feeds it the prompt below with all +14 skill answers pre-provided, and grades the two SQL files the skill emits. + +**Prompt used (verbatim in the test file):** + +> Invoke the snowflake-setup skill. Pre-answered triage: mode=greenfield, +> format=sql, execution=review-only. Detail answers: topology=Medallion, +> RBAC=small-team, envs=prod-only, ingestion=Snowpipe(AWS S3), cloud=AWS, +> emission=idempotent, budget=500 credits, PII discovery=declared with +> categories email/first_name/last_name, no multi-tenancy, no advanced +> features. Emit the greenfield SQL to eval-artifacts/greenfield.sql and +> rollback SQL to eval-artifacts/rollback.sql. Placeholders for S3 ARN etc +> are fine. Do NOT execute anything against Snowflake. + +**Assertions (13, all shape-based, resilient to LLM phrasing drift):** + +| # | Assertion | +|---|-----------| +| 1 | `bun run dev run --yolo` completes and emits both SQL files (>100 / >50 lines) | +| 2 | Greenfield: Medallion → BRONZE / SILVER / GOLD databases present | +| 3 | Greenfield: emission=idempotent → ≥85% of CREATE statements use IF NOT EXISTS | +| 4 | Greenfield: guardrail #2 — ≥4 FUTURE grants present | +| 5 | Greenfield: guardrail #1 — no ACCOUNTADMIN granted to service accounts; no service account has DEFAULT_ROLE=ACCOUNTADMIN | +| 6 | Greenfield: guardrail #4 — all 3 role-switch blocks present (ACCOUNTADMIN / SECURITYADMIN / SYSADMIN) | +| 7 | Greenfield: PII masking policies emitted for email and name | +| 8 | Greenfield: ≥2 resource monitor CREATE statements | +| 9 | Greenfield: "CONFIGURE BEFORE RUNNING" placeholder checklist present with `` markers | +| 10 | Rollback: ≥95% of DROP statements use IF EXISTS | +| 11 | Rollback: no DROP ... CASCADE | +| 12 | Rollback: never drops built-in roles (ACCOUNTADMIN, SECURITYADMIN, SYSADMIN, USERADMIN, PUBLIC, ORGADMIN) or SNOWFLAKE database | +| 13 | Rollback: account-locator confirmation guard with CURRENT_ACCOUNT() check present | + +**Cost per run:** ~120K input tokens + ~2–5K output tokens against the Altimate +LLM gateway. Not billed to the runner if using an Altimate-provided API key; +budget accordingly for CI. + +**Runtime:** ~90 seconds cold; primary cost is LLM inference. + +**Gating:** `SNOWFLAKE_SETUP_EVAL=1` env var. Without it, all 13 tests are +`describe.skipIf`'d — no LLM cost on regular CI runs. + +**Known variance:** LLM emits slightly different phrasing, comments, and +non-essential SQL each run. Assertions match on shape and required primitives +(role switches, IF NOT EXISTS ratios, PII policy families) rather than exact +strings. First failure to investigate before assuming a regression: rerun and +check if the LLM produced substantively different output vs cosmetically +different. + +**What it does NOT test:** +- Applying the emitted DDL to a real Snowflake account (see §"Tier 3 live + apply — future work" below) +- Rollback correctness end-to-end (DDL → apply → rollback → verify blank) +- Audit mode against a broken account +- Terraform HCL emission or `terraform validate` +- External integrations (S3, SSO, DR, sharing, Cortex) +- Non-Medallion topologies (DV2, Functional, Domain-per-DB) — the eval script + covers Medallion only; add more scenarios in follow-up PRs + +--- + +## Tier 3 — Live Snowflake apply/rollback cycle (future work) + +The remaining eval work is to actually **apply** the emitted DDL to a Snowflake +account, verify state, run the rollback, and verify the account returns to +blank. Everything below is designed but not implemented. + +--- + +## Tier 2 — Original mocked-execution eval design (superseded by Tier 2/3 above) + +> This section was the original Tier 2 design. It was superseded when +> the model-in-the-loop approach turned out to be feasible directly. Kept +> here for reference — the mock-harness approach may still be useful if a +> future team wants to run scenario coverage without LLM calls. + +**Goal:** verify the skill's workflow logic — the decisions it makes given a scripted account state — without needing a live Snowflake account. + +**Suggested location:** `packages/opencode/test/altimate/snowflake-setup-eval.test.ts` +**Fixture location:** `packages/opencode/test/altimate/fixtures/snowflake-setup/` +**Estimated build effort:** 2–3 days of focused work (this is the harness work, mostly) +**Estimated runtime once built:** ~30s per suite + +### What has to be built first (blocking dependencies) + +**1. Skill-invocation harness.** +No existing altimate-code test file drives a skill through its markdown Q&A flow with tool calls. Existing skill tests (`packages/opencode/test/skill/*.test.ts`) only verify skill *discovery* — did the loader see it, is the frontmatter valid, does the slash-command register? They do not simulate a user turn or capture tool invocations. + +The harness needs to: +- Load a skill by name via the existing `Skill.defaultLayer` +- Accept a scripted `turnScript` — array of `{ userTurn: string, expectAsks: string[], provideAnswers: Record }` +- Intercept every tool invocation the agent attempts +- Route interceptions through registered mocks +- Capture the final emitted output (SQL / HCL / plan markdown) +- Provide assertion helpers: `expectEmittedContains`, `expectRolePresent`, `expectDDLParsesAs`, `expectSectionOrder` + +This is arguably its own PR — testable infrastructure for **every** skill, not just this one. That's a design conversation worth having with maintainers first (probably file a GH issue proposing the harness API before building it). + +**2. Tool mocks.** +Each mock is a function matching the tool's Zod input schema and returning fixture data matching the tool's actual response shape. Faithful shapes are the entire game — a mock that returns "here are 3 warehouses" but with the wrong field names gives false confidence. + +Tools that need mocks: + +| Tool | Purpose in skill | Mock complexity | +|------|------------------|-----------------| +| `warehouse_list` | Preflight discovery | Low — array of connection records | +| `warehouse_add` | Never called in tests (fixture pre-supplies connections) | Low — no-op | +| `warehouse_discover` | Turn-1 auto-detect | Low — array of discovered accounts | +| `warehouse_test` | Preflight smoke check | Low — success/fail with message | +| `sql_execute` | Audit queries + guided-execute | **High** — must match query patterns and return realistic result shapes from `SNOWFLAKE.ACCOUNT_USAGE` | +| `finops_analyze_credits` | Audit § Cost | Medium — credit history object | +| `finops_role_hierarchy` | Audit § RBAC | Medium — role tree object | +| `finops_role_grants` | Audit § RBAC | Medium — grant list | +| `finops_user_roles` | Audit § RBAC | Low — user→roles map | +| `finops_warehouse_advice` | Hybrid sizing | Medium — sizing recommendation object | +| `schema_inspect` | Governance § column existence check | Medium — column metadata | +| `schema_detect_pii` | PII discovery | Medium — PII candidates with confidence | +| `altimate_core_classify_pii` | PII cross-check | Low — same shape as schema_detect_pii | + +Highest risk is `sql_execute` — the skill emits many different query shapes and each needs its own mock branch. A pattern-matching approach (e.g. "if query contains `SHOW WAREHOUSES`, return warehouse fixture") is fragile; a query-parser-based dispatch is better but requires more upfront work. + +**3. Fixture set.** +JSON files representing account state. Each scenario has its own directory: + +``` +fixtures/snowflake-setup/ +├── blank/ +│ ├── warehouses.json # [] +│ ├── roles.json # only built-ins +│ ├── databases.json # only SNOWFLAKE, SNOWFLAKE_SAMPLE_DATA +│ ├── resource_monitors.json # [] +│ └── ... +├── partially-configured/ +│ ├── warehouses.json # 2 warehouses, one without monitor +│ ├── roles.json # some custom roles, 1 orphaned +│ └── ... +├── production-ready/ +│ ├── warehouses.json # 4 warehouses, all monitored +│ ├── roles.json # full RBAC topology +│ └── ... +├── broken-with-7-issues/ +│ ├── warehouses.json # 3 warehouses, 1 no auto-suspend +│ ├── roles.json # 2 orphaned, 3 users w/ DEFAULT_ROLE=ACCOUNTADMIN +│ ├── grants.json # 4 schemas missing FUTURE +│ ├── users.json # 2 service accounts w/ passwords +│ ├── policies.json # 12 PII columns, no masking +│ ├── monitors.json # [] +│ └── ANSWER_KEY.md # expected: 7 CRITICAL findings, maturity < 30 +└── ... +``` + +Answer keys live **inside** each scenario dir so a directory-copy operation preserves the expected outcome with the input. Alternative: keep them **outside** the scanned dir (as PR #1092 does with `optimizer-project-answer-key.md`) so the skill can't accidentally read the answers during a test. Either is defensible — pick one and document it. + +### Test scenarios + +**Greenfield matrix (partial coverage of 36 combinations):** + +| # | Topology | Emission | Format | Assertion | +|---|----------|----------|--------|-----------| +| G1 | Medallion | idempotent | sql | DDL emits BRONZE/SILVER/GOLD databases, warehouses sized per guide, RBAC per reference topology; `CREATE ... IF NOT EXISTS` used throughout; rollback file emitted alongside; matches golden file `G1.expected.sql` | +| G2 | Functional | strict | sql | plain `CREATE`, no `IF NOT EXISTS`; RAW/TRANSFORM/ANALYTICS databases | +| G3 | Domain-per-DB | idempotent | sql | `ANALYTICS_FINANCE`, `ANALYTICS_MARKETING`, `ANALYTICS_ENGINEERING` databases; per-domain roles | +| G4 | Data Vault 2.0 | idempotent | sql | `RAW_VAULT`, `BUSINESS_VAULT`, `INFO_MARTS` databases; hub/link/satellite schemas; `VAULT_LOADER_ROLE`, `BUSINESS_VAULT_BUILDER_ROLE`, `MART_BUILDER_ROLE`; insert-only `REVOKE UPDATE, DELETE` present | +| G5 | Medallion | idempotent | terraform | HCL emitted; `providers.tf` + `variables.tf` + resource files; `prevent_destroy = true` on prod databases; sensitive outputs marked | +| G6 | Medallion | idempotent | both | both `.sql` and `.tf` files emitted; content matches G1 + G5 | +| G7 | Medallion | additive (fixture: `partially-configured/`) | sql | DDL emits only for missing objects; existing 2 warehouses NOT recreated | + +**Audit scenarios:** + +| # | Fixture | Assertion | +|---|---------|-----------| +| A1 | `blank/` | 0 findings, maturity 100, "This looks like a new/empty account — did you mean greenfield?" nudge appears | +| A2 | `production-ready/` | 0 CRITICAL findings, maturity ≥ 90 | +| A3 | `broken-with-7-issues/` | ≥ 6/7 planted issues surfaced with correct severity; maturity ≤ 30; each finding maps to a remediation snippet from the correct reference file | +| A4 | `broken-with-7-issues/` | Fail-fast if `warehouse_list` returns no snowflake entry — no `sql_execute` is ever called | + +**Hybrid scenarios:** + +| # | Fixture | Assertion | +|---|---------|-----------| +| H1 | `partially-configured/` (has Medallion topology, 2 warehouses, no RBAC) | Detects Medallion topology; pre-fills Q1 answer; skips warehouse questions; asks only for missing RBAC and cost controls | +| H2 | `partially-configured/` with existing FUTURE grants | RBAC section detects them and doesn't re-emit; only missing grants emit | + +**Rollback scenarios:** + +| # | Setup | Assertion | +|---|-------|-----------| +| R1 | Apply G1 to `blank/` → capture state → run rollback | Post-rollback state matches `blank/` exactly (deep-equal) | +| R2 | Rollback against a mismatched account locator | Confirmation guard fires; no DROPs execute | +| R3 | Rollback where a task is running | `ALTER TASK ... SUSPEND` emitted before `DROP TASK` | + +### Grading rubric + +- **Snapshot-based** for emitted SQL/HCL: golden files in `fixtures/snowflake-setup/expected/`, regenerated with `UPDATE_SNAPSHOTS=1 bun test` +- **Assertion-based** for audit findings: match on `{ category, severity, remediation_ref }` tuples; order-independent +- **Deep-equal** for rollback state comparison + +Regressions from any golden file fail the test. Non-substantive DDL reordering (like putting `ALTER USER` before `GRANT USAGE`) may false-positive — mitigate by canonicalizing statement order before comparison. + +### Known false-negative modes + +Mocked evals only test what the mocks return. Real failure modes NOT caught: + +- Snowflake introducing a new column in `ACCOUNT_USAGE.WAREHOUSES` that breaks a query (the mock still returns the old shape) +- `sql_execute` in production requiring different permissions than mocked +- Terraform provider version incompatibility +- Actual query performance at scale + +Tier 3 is the only way to catch these. + +--- + +## Tier 3 live apply/rollback — full design (superseded intro; kept for reference) + +**Goal:** end-to-end verification against a real Snowflake account. + +**Suggested location:** `packages/opencode/test/altimate/snowflake-setup-live.test.ts` +**Gating:** `SNOWFLAKE_LIVE_EVAL=1` env var + credential env vars required +**Estimated build effort:** 1 day, after Tier 2 harness exists (Tier 3 reuses the harness) +**Estimated runtime:** 15–30 min per suite run +**Cost:** ~5–20 Snowflake credits per full run (rollback + audit cycles) + +### Prerequisites + +1. **A dedicated blank Snowflake account** — not a shared team account. Full teardown between runs. +2. **Credentials** set via env vars: + - `SNOWFLAKE_ACCOUNT` — account locator + - `SNOWFLAKE_USER` — user with ACCOUNTADMIN + - `SNOWFLAKE_PRIVATE_KEY_PATH` — path to key-pair auth private key (never a password in CI) + - `SNOWFLAKE_WAREHOUSE` — an already-existing warehouse to run the test queries from (typically `COMPUTE_WH`) +3. **Terraform** installed and on `PATH` +4. **`SNOWFLAKE_LIVE_EVAL=1`** set — the `describe.skipIf` gate + +### Test flow + +**Phase 1 — Preflight** (~ 30s) +- Register the test account via `warehouse_add` +- Call `warehouse_test` — must return success +- Verify account is genuinely blank (no user databases, no custom roles) +- Fail fast if any prior test state exists + +**Phase 2 — Greenfield → verify → rollback cycle** (~ 10 min for one topology) + +For each topology in [Medallion, Data Vault 2.0] (skip Functional and Domain-per-DB — same shape as Medallion; not worth the credits): + +1. Invoke the skill with pre-scripted answers via the same harness Tier 2 uses +2. Skill emits DDL to a temp file +3. Test executes the DDL via `snowsql` CLI (or `sql_execute` in a loop with role switches) +4. Verify each expected object exists: + - `SHOW DATABASES` — matches expected set + - `SHOW WAREHOUSES` — matches expected set with sizes, auto_suspend + - `SHOW ROLES` — matches expected topology + - `SHOW GRANTS TO ROLE ANALYST_ROLE` — future grants present + - `SHOW RESOURCE MONITORS` — account monitor + per-warehouse monitors +5. Execute the emitted rollback script +6. Verify account is back to genuinely blank state — same checks as Phase 1 +7. If any step fails, run rollback anyway (as a defensive teardown) and fail the test + +**Phase 3 — Audit against a broken state** (~ 5 min) + +1. Apply a "known-broken" DDL script (planted with the 7 issues from Tier 2 fixture) +2. Run skill in audit mode +3. Assert same finding recall as Tier 2 test A3 (≥ 6/7 issues), but this time against real `ACCOUNT_USAGE` queries +4. Run rollback of the known-broken state + +**Phase 4 — Terraform validate** (~ 2 min) + +1. Invoke skill with `terraform` output format +2. Skill emits HCL files +3. Run `terraform init` on the emitted directory (requires network to Snowflake registry) +4. Run `terraform validate` — must pass +5. Run `terraform plan` against the blank account — must generate expected resource count +6. Optional: `terraform apply` for one topology if time permits + +**Phase 5 — Teardown** (~ 30s) + +1. Explicit `DROP` of every database, warehouse, role, and integration that could have been created +2. Verify blank state one more time + +### Scope explicitly OUT + +Live-eval Phase 2/3 verify the core lifecycle. Live-eval does NOT verify: + +- Snowpipe / Task+COPY against a real S3 bucket — needs external infra (S3 + IAM) +- Fivetran / Airbyte — needs external accounts +- SSO federation (Okta, Azure AD, Google Workspace) — needs an IdP tenant +- SCIM provisioning — same +- Cross-region DR (replication / failover groups) — needs a **second** Snowflake account in a different region +- Data sharing to another account — needs a second account +- Cortex functions in production — accrues charges; may not be available in all regions + +These paths remain "generated-but-runtime-unverified" and should be marked as such in the emitted plan header ("Note: this section covers configuration that has been unit-tested but not verified end-to-end against live infrastructure. Test manually against your own environment before production use."). + +### CI wiring + +Tier 3 should NOT run on every PR. Suggested: + +- Nightly: run against a dedicated `altimate-code-snowflake-eval` test account +- On-demand: `gh workflow run snowflake-live-eval.yml` for a manual trigger +- Never on forks: gate via `if: github.repository == 'AltimateAI/altimate-code'` + +Credentials via GitHub Actions Secrets (`SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_PRIVATE_KEY_B64` decoded to a temp file at runtime). + +--- + +## Priority for future work + +If someone picks this up, sequence the work like this: + +1. **Design + review the skill-invocation harness API first** (file a GH issue). This is the load-bearing dependency for both Tier 2 and Tier 3. Getting the harness API right unblocks eval work for every other skill in the repo too. +2. **Build the harness in a separate PR.** Get it reviewed and merged before Tier 2 depends on it. +3. **Build Tier 2 in a third PR** using the harness. Fixtures + mocks + graded scenarios. +4. **Build Tier 3 in a fourth PR** after acquiring a dedicated test account. Reuses the harness; adds live-connectivity plumbing. + +Attempting all four in one PR is not recommended — the surface area is too large for effective review. + +## Contact / provenance + +- Tier 1 test pattern borrowed from PR #1092 (`optimizer-prompt-contract.test.ts`, dbt-optimizer agent). +- Original skill design + Tier 1 implementation: this PR. +- Tier 2 / Tier 3 designs: this doc; not implemented. diff --git a/.opencode/skills/snowflake-setup/references/advanced-features.md b/.opencode/skills/snowflake-setup/references/advanced-features.md new file mode 100644 index 0000000000..f3721f55ae --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/advanced-features.md @@ -0,0 +1,354 @@ +# Advanced Features + +Optional sections the skill emits when specific opt-in questions are answered. Each has its own trigger question and can be enabled independently. + +## Network Security and SSO + +### Trigger questions +- "Restrict Snowflake access to specific IP ranges?" (VPN, office, cloud VPC) +- "Federate authentication via Okta / Azure AD / Google Workspace?" + +### Account-level network policy + +```sql +USE ROLE SECURITYADMIN; + +CREATE NETWORK POLICY office_and_vpn + ALLOWED_IP_LIST = ('203.0.113.0/24', '198.51.100.5/32', '10.100.0.0/16') + BLOCKED_IP_LIST = () + COMMENT = 'Corporate office + VPN ranges'; + +-- Apply to entire account (default policy) +ALTER ACCOUNT SET NETWORK_POLICY = office_and_vpn; + +-- Or apply per-user (overrides account default) +ALTER USER alice SET NETWORK_POLICY = office_and_vpn; + +-- Service accounts often need a different policy (CI runner IPs) +CREATE NETWORK POLICY ci_runners + ALLOWED_IP_LIST = ('52.0.0.0/8', '54.0.0.0/8'); -- GitHub-hosted runners; tighten in production +ALTER USER dbt_service SET NETWORK_POLICY = ci_runners; +``` + +### Key-pair auth for service accounts + +Password auth for service accounts is a critical audit finding. Generate a keypair: + +```bash +# Generate encrypted private key (recommended) +openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out fivetran_loader.p8 -v2 aes-256-cbc +# Extract the public key +openssl rsa -in fivetran_loader.p8 -pubout -out fivetran_loader.pub +# Strip PEM headers for Snowflake ALTER USER +cat fivetran_loader.pub | grep -v -- ----- | tr -d '\n' +``` + +```sql +ALTER USER fivetran_loader SET RSA_PUBLIC_KEY = ''; +-- Rotate by setting RSA_PUBLIC_KEY_2, then swapping later +``` + +### SCIM provisioning (Okta / Azure AD) + +```sql +USE ROLE ACCOUNTADMIN; + +-- Enable SCIM +CREATE SECURITY INTEGRATION okta_scim + TYPE = SCIM + SCIM_CLIENT = 'OKTA' + RUN_AS_ROLE = 'OKTA_PROVISIONER'; + +-- Custom role SCIM provisioner runs as (least privilege) +CREATE ROLE OKTA_PROVISIONER; +GRANT CREATE USER, CREATE ROLE ON ACCOUNT TO ROLE OKTA_PROVISIONER; +GRANT ROLE OKTA_PROVISIONER TO ROLE ACCOUNTADMIN; + +-- Retrieve SCIM auth token — paste this into Okta's SCIM app config +SELECT SYSTEM$GENERATE_SCIM_ACCESS_TOKEN('OKTA_SCIM'); +``` + +### SAML SSO + +```sql +CREATE SECURITY INTEGRATION okta_saml + TYPE = SAML2 + ENABLED = TRUE + SAML2_ISSUER = 'http://www.okta.com/' + SAML2_SSO_URL = 'https://.okta.com/app//sso/saml' + SAML2_PROVIDER = 'OKTA' + SAML2_X509_CERT = '' + SAML2_SP_INITIATED_LOGIN_PAGE_LABEL = 'Okta' + SAML2_ENABLE_SP_INITIATED = TRUE; + +-- Force SSO for all human users (block password login) +ALTER ACCOUNT SET SSO_LOGIN_PAGE = TRUE; +``` + +### MFA enforcement + +```sql +-- Enforce Duo MFA for password-auth users +ALTER USER alice SET MINS_TO_BYPASS_MFA = 0; +-- SET DISABLE_MFA = FALSE for all users +``` + +## Disaster Recovery (Replication + Failover) + +### Trigger question +- "Set up cross-region replication for disaster recovery?" + +### Replication group (data-only, no automatic failover) + +```sql +USE ROLE ACCOUNTADMIN; + +-- On source account (primary region) +CREATE REPLICATION GROUP prod_replication + OBJECT_TYPES = DATABASES, SHARES, ROLES, WAREHOUSES, RESOURCE MONITORS, INTEGRATIONS, NETWORK POLICIES + ALLOWED_DATABASES = RAW, TRANSFORM, ANALYTICS + ALLOWED_SHARES = () + ALLOWED_ACCOUNTS = . + REPLICATION_SCHEDULE = '60 MINUTE'; + +-- On target account (secondary region) +CREATE REPLICATION GROUP prod_replication_secondary + AS REPLICA OF ..prod_replication; + +-- Trigger initial replication +ALTER REPLICATION GROUP prod_replication_secondary REFRESH; +``` + +### Failover group (automatic failover with client redirect) + +```sql +-- Failover groups replicate AND enable failover — clients use a Connection URL +CREATE FAILOVER GROUP prod_failover + OBJECT_TYPES = DATABASES, SHARES, ROLES, WAREHOUSES, RESOURCE MONITORS, INTEGRATIONS, NETWORK POLICIES + ALLOWED_DATABASES = RAW, TRANSFORM, ANALYTICS + ALLOWED_ACCOUNTS = . + REPLICATION_SCHEDULE = '15 MINUTE'; + +-- Client connection URL that survives failover +CREATE CONNECTION prod_connection + AS PRIMARY OF ..prod_failover; + +-- Manually failover to secondary (or automate via monitoring) +ALTER FAILOVER GROUP prod_failover PRIMARY; -- on secondary account +ALTER CONNECTION prod_connection PRIMARY; -- redirect clients +``` + +### DR monitoring + +```sql +-- Replication lag +SELECT phase, start_time, end_time, DATEDIFF('second', start_time, end_time) AS duration_s, + total_bytes / POWER(1024, 3) AS gb_replicated +FROM TABLE(INFORMATION_SCHEMA.REPLICATION_GROUP_REFRESH_HISTORY('prod_replication')) +ORDER BY start_time DESC LIMIT 20; + +-- Estimated RPO (recovery point objective) = max age of last successful refresh +SELECT DATEDIFF('minute', MAX(end_time), CURRENT_TIMESTAMP()) AS minutes_since_last_sync +FROM TABLE(INFORMATION_SCHEMA.REPLICATION_GROUP_REFRESH_HISTORY('prod_replication')) +WHERE phase = 'COMPLETED'; +``` + +## Data Sharing + +### Trigger question +- "Share data with external consumers (Snowflake or reader accounts)?" + +### Outbound share to another Snowflake account + +```sql +USE ROLE ACCOUNTADMIN; + +CREATE SHARE finance_metrics_share + COMMENT = 'Monthly finance dashboards for parent-company data team'; + +-- Grant objects to the share +GRANT USAGE ON DATABASE ANALYTICS TO SHARE finance_metrics_share; +GRANT USAGE ON SCHEMA ANALYTICS.FINANCE TO SHARE finance_metrics_share; +GRANT SELECT ON ALL TABLES IN SCHEMA ANALYTICS.FINANCE TO SHARE finance_metrics_share; +GRANT SELECT ON FUTURE TABLES IN SCHEMA ANALYTICS.FINANCE TO SHARE finance_metrics_share; + +-- Add consumer accounts (must be in same region + cloud, or cross-region-enabled) +ALTER SHARE finance_metrics_share ADD ACCOUNTS = .; + +-- Consumer side: consume the share as a read-only database +CREATE DATABASE finance_metrics_from_provider FROM SHARE ..finance_metrics_share; +``` + +### Reader account (for consumers without Snowflake) + +```sql +-- Provider creates the reader account (billed to provider) +CREATE MANAGED ACCOUNT client_x_reader + ADMIN_NAME = client_x_admin + ADMIN_PASSWORD = '' + TYPE = READER + COMMENT = 'Read-only account for Client X'; + +-- Add the reader account to the share +ALTER SHARE finance_metrics_share ADD ACCOUNTS = .client_x_reader; +``` + +### Secure views for shared data + +Never share raw tables directly — always use secure views to control column exposure. + +```sql +CREATE SECURE VIEW ANALYTICS.FINANCE.monthly_revenue_share AS +SELECT + DATE_TRUNC('month', order_date) AS month, + region, + SUM(revenue) AS total_revenue + -- customer_id and email intentionally excluded from share +FROM ANALYTICS.FINANCE.fact_orders +GROUP BY 1, 2; + +GRANT SELECT ON VIEW ANALYTICS.FINANCE.monthly_revenue_share TO SHARE finance_metrics_share; +``` + +## Snowflake Cortex / ML + +### Trigger question +- "Enable Snowflake Cortex for LLM functions or Cortex Search?" + +### Cortex functions warehouse + +```sql +USE ROLE ACCOUNTADMIN; + +-- Cortex functions run on a regular warehouse but benefit from Snowpark-optimized instances +CREATE WAREHOUSE CORTEX_WH + WAREHOUSE_SIZE = 'MEDIUM' + WAREHOUSE_TYPE = 'SNOWPARK-OPTIMIZED' + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + COMMENT = 'Cortex LLM and ML functions'; + +-- Grant usage to roles that will call Cortex functions +GRANT USAGE ON WAREHOUSE CORTEX_WH TO ROLE ANALYST_ROLE; + +-- The account-level parameter enables Cortex features +-- (already true by default in most regions; check first) +SHOW PARAMETERS LIKE 'CORTEX_ENABLED_CROSS_REGION' IN ACCOUNT; +``` + +### Cortex Search service (for RAG applications) + +```sql +CREATE CORTEX SEARCH SERVICE support_docs_search + ON content + ATTRIBUTES doc_id, title, url, updated_at + WAREHOUSE = CORTEX_WH + TARGET_LAG = '1 hour' + AS ( + SELECT doc_id, title, url, content, updated_at + FROM ANALYTICS.SUPPORT.help_articles + ); + +-- Query the service from an application +SELECT PARSE_JSON( + SNOWFLAKE.CORTEX.SEARCH_PREVIEW( + 'ANALYTICS.SUPPORT.support_docs_search', + '{ "query": "how to reset password", "limit": 5 }' + ) +):results; +``` + +### Cortex LLM function grants + +```sql +-- Cortex functions live in SNOWFLAKE.CORTEX schema +GRANT USAGE ON DATABASE SNOWFLAKE TO ROLE ANALYST_ROLE; +GRANT USAGE ON SCHEMA SNOWFLAKE.CORTEX TO ROLE ANALYST_ROLE; +-- Functions are already granted USAGE to PUBLIC by default +``` + +### Cortex usage governance + +```sql +-- Track Cortex spend +SELECT + DATE_TRUNC('day', start_time) AS day, + function_name, + SUM(token_credits) AS credits +FROM SNOWFLAKE.ACCOUNT_USAGE.CORTEX_FUNCTIONS_USAGE_HISTORY +WHERE start_time >= DATEADD('day', -30, CURRENT_TIMESTAMP()) +GROUP BY 1, 2 +ORDER BY day DESC, credits DESC; +``` + +## Cost Forecasting + +### Trigger question +- Automatic when a monthly credit budget is not provided, or user opts in for a tailored budget + +### Forecast inputs + +The skill collects: +- Expected data volume ingested per day (GB) +- Expected number of dbt models +- Expected concurrent BI users +- Expected ad-hoc analyst sessions per week + +### Forecast formulas + +``` +LOADING_WH credits/month + ≈ (daily_gb × 30) / + where load_throughput ≈ 100 GB/credit on XS/S warehouses + +TRANSFORM_WH credits/month + ≈ (dbt_models × avg_runtime_minutes × runs_per_day × 30) / 60 + where avg_runtime_minutes defaults to 2 (SMALL wh, ~10 rows/sec) + +ANALYTICS_WH credits/month + ≈ (concurrent_users × active_hours_per_day × 30) × warehouse_size_credits_per_hour + where MEDIUM = 4 credits/hour +``` + +### Budget output + +``` +Estimated monthly credits (500 GB/day, 200 dbt models, 15 BI users): + LOADING_WH: 150 credits → $450 at $3/credit + TRANSFORM_WH: 200 credits → $600 + ANALYTICS_WH: 300 credits → $900 + DEV_WH: 50 credits → $150 + Buffer (20%): 140 credits → $420 + ───────────────────────────── + Total budget: 840 credits → $2,520/month + +Resource monitor recommendations: + Account-level: 840 credits (100% suspend, 75%/90% notify) + TRANSFORM_WH: 240 credits (20% headroom over forecast) + ANALYTICS_WH: 360 credits + LOADING_WH: 180 credits +``` + +### Forecast validation + +After 30 days, re-run the audit and compare actual to forecast: + +```sql +-- Actual vs. forecast (feed forecast values as literals) +SELECT + warehouse_name, + SUM(credits_used) AS actual_credits, + CASE warehouse_name + WHEN 'LOADING_WH' THEN 150 + WHEN 'TRANSFORM_WH' THEN 200 + WHEN 'ANALYTICS_WH' THEN 300 + WHEN 'DEV_WH' THEN 50 + END AS forecast_credits, + ROUND(SUM(credits_used) / NULLIF(forecast_credits, 0) * 100, 1) AS pct_of_forecast +FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY +WHERE start_time >= DATEADD('day', -30, CURRENT_TIMESTAMP()) +GROUP BY 1 +ORDER BY pct_of_forecast DESC; +``` + +If any warehouse is > 120% of forecast, the skill recommends either a size-up (if queue time is high) or workload investigation via the `query-optimize` and `cost-report` skills. diff --git a/.opencode/skills/snowflake-setup/references/audit-queries.md b/.opencode/skills/snowflake-setup/references/audit-queries.md new file mode 100644 index 0000000000..213ad24cfd --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/audit-queries.md @@ -0,0 +1,358 @@ +# Brownfield Audit Queries + +Queries the skill runs (via `sql_execute` against a configured Snowflake warehouse) to assess an existing account's setup. Each query maps to a check category; results feed into the gap-scored punch list. + +## Prerequisites + +Audit mode requires a live Snowflake connection. Before running any query, verify: + +```sql +-- Must have ACCOUNT_USAGE access (typically ACCOUNTADMIN or a role explicitly granted IMPORTED PRIVILEGES on SNOWFLAKE) +SELECT CURRENT_ROLE(), CURRENT_ACCOUNT(); +SHOW GRANTS TO ROLE IDENTIFIER(CURRENT_ROLE()); +``` + +If `SNOWFLAKE.ACCOUNT_USAGE` is not accessible, the skill must fall back to `INFORMATION_SCHEMA` (per-database) and note the reduced coverage in the report. + +### `ACCOUNT_USAGE` replication lag — cross-check with `SHOW` + +`SNOWFLAKE.ACCOUNT_USAGE.*` views are refreshed with lag — typically 45 minutes to 2 hours per Snowflake's documentation. This means recently-created objects (created in the last ~2 hours) may not appear in these views yet, causing the audit to miss them. + +**Rule: for every category the audit relies on `ACCOUNT_USAGE` for, also run the real-time `SHOW` command as a cross-check, and reconcile:** + +| ACCOUNT_USAGE view (lagged) | Real-time `SHOW` equivalent | +|-----------------------------|-----------------------------| +| `DATABASES` | `SHOW DATABASES` | +| `SCHEMATA` | `SHOW SCHEMAS IN ACCOUNT` | +| `ROLES` | `SHOW ROLES` | +| `USERS` | `SHOW USERS` | +| `WAREHOUSES` | `SHOW WAREHOUSES` | +| `GRANTS_TO_ROLES` | `SHOW GRANTS TO ROLE ` (per role) | +| `POLICY_REFERENCES` | `SHOW MASKING POLICIES IN ACCOUNT` + per-column `POLICY_REFERENCES` table function | +| `TAG_REFERENCES` | `SHOW TAGS IN ACCOUNT` + per-object references | +| `WAREHOUSE_METERING_HISTORY` | No real-time equivalent — cost queries genuinely cannot cross-check; report the lag in the output | + +If a `SHOW` returns an object that `ACCOUNT_USAGE` does not, treat the `SHOW` result as authoritative and note the lag in the finding. Verified against live account `DKZPOBS-TQ14188` on 2026-08-26 — recently-created `ORPHANED_ROLE` and `BAD_USER` did not appear in `ACCOUNT_USAGE.ROLES` / `ACCOUNT_USAGE.USERS` for 45+ minutes but appeared immediately in `SHOW ROLES` / `SHOW USERS`. + +## Severity Rubric + +Each finding is scored: + +- **CRITICAL** — active security or data-loss risk (masking absent on PII, ACCOUNTADMIN as DEFAULT_ROLE, no resource monitors) +- **WARNING** — operational risk or best-practice violation (missing FUTURE grants, no dev environment, long Time Travel on staging) +- **INFO** — nice-to-have (untagged warehouses, no cost attribution) + +## Section 1: Topology and Databases + +```sql +-- All user databases (excludes system databases) +SELECT database_name, created, comment, retention_time +FROM SNOWFLAKE.ACCOUNT_USAGE.DATABASES +WHERE deleted IS NULL + AND database_name NOT IN ('SNOWFLAKE', 'SNOWFLAKE_SAMPLE_DATA') +ORDER BY created; + +-- Detect topology pattern +-- Presence of BRONZE/SILVER/GOLD → Medallion +-- Presence of RAW/TRANSFORM/ANALYTICS → Functional +-- Presence of ANALYTICS_ → Domain-per-DB +-- None of the above → Ad-hoc / no discernible pattern (WARNING) + +-- Schemas per database +SELECT catalog_name AS database, schema_name, comment +FROM SNOWFLAKE.ACCOUNT_USAGE.SCHEMATA +WHERE deleted IS NULL + AND catalog_name NOT IN ('SNOWFLAKE', 'SNOWFLAKE_SAMPLE_DATA') +ORDER BY 1, 2; +``` + +## Section 2: Warehouses + +```sql +-- Warehouse inventory +SHOW WAREHOUSES; + +-- Warehouses without auto-suspend (CRITICAL — burns credits) +SELECT "name", "size", "auto_suspend", "auto_resume", "scaling_policy" +FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())) +WHERE "auto_suspend" IS NULL OR "auto_suspend" > 600; + +-- Warehouses with auto_resume disabled (WARNING) +SELECT "name" FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())) +WHERE "auto_resume" = 'false'; + +-- Multi-cluster settings +SELECT "name", "min_cluster_count", "max_cluster_count", "scaling_policy" +FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())); +``` + +## Section 3: RBAC + +```sql +-- All custom roles (excluding built-ins) +SELECT name, comment, created_on +FROM SNOWFLAKE.ACCOUNT_USAGE.ROLES +WHERE deleted_on IS NULL + AND name NOT IN ('ACCOUNTADMIN', 'SECURITYADMIN', 'SYSADMIN', + 'USERADMIN', 'PUBLIC', 'ORGADMIN') +ORDER BY created_on; + +-- CRITICAL: orphaned custom roles (not granted to SYSADMIN) +SELECT r.name AS role_name +FROM SNOWFLAKE.ACCOUNT_USAGE.ROLES r +WHERE r.deleted_on IS NULL + AND r.name NOT IN ('ACCOUNTADMIN', 'SECURITYADMIN', 'SYSADMIN', + 'USERADMIN', 'PUBLIC', 'ORGADMIN') + AND NOT EXISTS ( + SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_ROLES g + WHERE g.granted_to = 'ROLE' + AND g.grantee_name = 'SYSADMIN' + AND g.name = r.name + AND g.privilege = 'USAGE' + AND g.deleted_on IS NULL + ); + +-- CRITICAL: users with ACCOUNTADMIN as DEFAULT_ROLE +SELECT name, email, default_role, default_warehouse, disabled +FROM SNOWFLAKE.ACCOUNT_USAGE.USERS +WHERE deleted_on IS NULL + AND disabled = 'false' + AND default_role IN ('ACCOUNTADMIN', 'SECURITYADMIN', 'ORGADMIN'); + +-- WARNING: schemas missing FUTURE grants (the #1 forgotten step) +-- Compare: existing SELECT grants on tables vs FUTURE table SELECT grants +WITH schemas_with_select_grants AS ( + SELECT DISTINCT table_catalog || '.' || table_schema AS schema_full + FROM SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_ROLES + WHERE granted_on = 'TABLE' + AND privilege = 'SELECT' + AND deleted_on IS NULL +), +schemas_with_future_grants AS ( + SELECT DISTINCT name AS schema_full + FROM SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_ROLES + WHERE granted_on = 'SCHEMA' + AND privilege = 'SELECT' + -- Future grants show up with granted_on = 'FUTURE_TABLE' internally +) +SELECT schema_full AS schema_missing_future_grants +FROM schemas_with_select_grants +WHERE schema_full NOT IN (SELECT schema_full FROM schemas_with_future_grants); + +-- Service accounts (users flagged as service — no email, has default warehouse) +SELECT name, default_role, default_warehouse, last_success_login, + days_to_expiry, has_password, has_rsa_public_key +FROM SNOWFLAKE.ACCOUNT_USAGE.USERS +WHERE deleted_on IS NULL + AND (email IS NULL OR email LIKE '%service%' OR name LIKE '%_SVC' OR name LIKE 'FIVETRAN%'); + +-- CRITICAL: service accounts using password auth instead of key-pair +SELECT name FROM SNOWFLAKE.ACCOUNT_USAGE.USERS +WHERE deleted_on IS NULL + AND has_password = 'true' + AND has_rsa_public_key = 'false' + AND (email IS NULL OR name LIKE '%_SVC'); +``` + +## Section 4: Ingestion + +```sql +-- Storage integrations +SHOW INTEGRATIONS; + +-- Pipes and their status +SHOW PIPES IN ACCOUNT; + +-- Pipes with recent errors +SELECT pipe_catalog_name, pipe_schema_name, pipe_name, + last_load_time, error_count, first_error_message +FROM TABLE(SNOWFLAKE.INFORMATION_SCHEMA.PIPE_USAGE_HISTORY( + DATE_RANGE_START => DATEADD('day', -7, CURRENT_TIMESTAMP()) +)) +WHERE error_count > 0 +ORDER BY last_load_time DESC; + +-- Tasks and their state (suspended tasks are silent failures) +SHOW TASKS IN ACCOUNT; + +-- WARNING: tasks that haven't run in 7+ days despite being enabled +SELECT name, database_name, schema_name, state, schedule +FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())) +WHERE state = 'started' + AND schedule IS NOT NULL; +``` + +## Section 5: Governance + +```sql +-- All masking policies in the account +SHOW MASKING POLICIES IN ACCOUNT; + +-- All row access policies +SHOW ROW ACCESS POLICIES IN ACCOUNT; + +-- CRITICAL: columns with likely PII names but no masking policy +-- (name-based heuristic; the skill should call altimate_core_classify_pii for accuracy) +WITH pii_column_candidates AS ( + SELECT table_catalog, table_schema, table_name, column_name + FROM SNOWFLAKE.ACCOUNT_USAGE.COLUMNS + WHERE deleted IS NULL + AND table_schema NOT IN ('INFORMATION_SCHEMA') + AND table_catalog NOT IN ('SNOWFLAKE', 'SNOWFLAKE_SAMPLE_DATA') + AND ( + LOWER(column_name) LIKE '%email%' + OR LOWER(column_name) LIKE '%phone%' + OR LOWER(column_name) LIKE '%ssn%' + OR LOWER(column_name) LIKE '%first_name%' + OR LOWER(column_name) LIKE '%last_name%' + OR LOWER(column_name) LIKE '%full_name%' + OR LOWER(column_name) LIKE '%dob%' + OR LOWER(column_name) LIKE '%birth%' + OR LOWER(column_name) LIKE '%credit_card%' + OR LOWER(column_name) LIKE '%cc_num%' + OR LOWER(column_name) LIKE '%address%' + ) +), +columns_with_masking AS ( + SELECT ref_database_name, ref_schema_name, ref_entity_name, ref_column_name + FROM SNOWFLAKE.ACCOUNT_USAGE.POLICY_REFERENCES + WHERE policy_kind = 'MASKING_POLICY' + AND policy_status = 'ACTIVE' +) +SELECT p.table_catalog, p.table_schema, p.table_name, p.column_name, + 'CRITICAL: likely PII with no masking' AS finding +FROM pii_column_candidates p +LEFT JOIN columns_with_masking m + ON p.table_catalog = m.ref_database_name + AND p.table_schema = m.ref_schema_name + AND p.table_name = m.ref_entity_name + AND p.column_name = m.ref_column_name +WHERE m.ref_column_name IS NULL; + +-- Tag taxonomy coverage +SHOW TAGS IN ACCOUNT; + +-- WARNING: warehouses without cost-attribution tags +SELECT w.name AS warehouse_name +FROM (SHOW WAREHOUSES) w +WHERE NOT EXISTS ( + SELECT 1 FROM SNOWFLAKE.ACCOUNT_USAGE.TAG_REFERENCES tr + WHERE tr.domain = 'WAREHOUSE' + AND tr.object_name = w.name + AND tr.tag_name IN ('COST_CENTER', 'TEAM', 'ENVIRONMENT') +); +``` + +## Section 6: Cost Controls + +```sql +-- Resource monitors +SHOW RESOURCE MONITORS; + +-- CRITICAL: no account-level resource monitor +SELECT COUNT(*) AS account_monitor_count +FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())) +WHERE "level" = 'ACCOUNT'; +-- Expect >= 1 + +-- CRITICAL: warehouses with no resource monitor attached +SELECT "name" AS warehouse_name +FROM (SHOW WAREHOUSES) +WHERE "resource_monitor" IS NULL OR "resource_monitor" = 'null'; + +-- INFO: current spending vs monitor limits +SELECT + wm.warehouse_name, + SUM(wm.credits_used) AS credits_last_30d, + rm.credit_quota AS monitor_quota, + ROUND(SUM(wm.credits_used) / NULLIF(rm.credit_quota, 0) * 100, 1) AS pct_of_quota +FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY wm +LEFT JOIN SNOWFLAKE.ACCOUNT_USAGE.RESOURCE_MONITORS rm + ON rm.name = ( + SELECT "resource_monitor" FROM (SHOW WAREHOUSES) w + WHERE w."name" = wm.warehouse_name + ) +WHERE wm.start_time >= DATEADD('day', -30, CURRENT_TIMESTAMP()) +GROUP BY 1, 3 +ORDER BY pct_of_quota DESC; + +-- Long Time Travel on ingestion / staging tables (INFO: storage waste) +SELECT table_catalog, table_schema, table_name, retention_time, + ROUND(bytes / POWER(1024, 3), 2) AS size_gb +FROM SNOWFLAKE.ACCOUNT_USAGE.TABLES +WHERE deleted IS NULL + AND retention_time > 1 + AND table_schema NOT IN ('INFORMATION_SCHEMA') + AND (LOWER(table_catalog) LIKE '%raw%' OR LOWER(table_catalog) LIKE '%bronze%') +ORDER BY bytes DESC; +``` + +## Section 7: Network and Security + +```sql +-- Network policies +SHOW NETWORK POLICIES; + +-- WARNING: no account-level network policy set +SHOW PARAMETERS LIKE 'NETWORK_POLICY' IN ACCOUNT; + +-- Users bypassing MFA (CRITICAL for password auth) +SELECT name, ext_authn_duo, mins_to_bypass_mfa +FROM SNOWFLAKE.ACCOUNT_USAGE.USERS +WHERE deleted_on IS NULL + AND has_password = 'true' + AND (ext_authn_duo = 'false' OR ext_authn_duo IS NULL) + AND email IS NOT NULL; -- human users only + +-- Sessions from unexpected IPs (last 7 days) +SELECT DISTINCT client_ip, COUNT(*) AS session_count +FROM SNOWFLAKE.ACCOUNT_USAGE.SESSIONS +WHERE created_on >= DATEADD('day', -7, CURRENT_TIMESTAMP()) +GROUP BY 1 +ORDER BY 2 DESC; +``` + +## Section 8: Data Sharing and Replication + +```sql +-- Outbound shares +SHOW SHARES; + +-- Replication groups (DR posture) +SHOW REPLICATION GROUPS; + +-- Failover groups +SHOW FAILOVER GROUPS; +``` + +## Aggregate Score + +The skill computes an overall score after running all sections: + +``` +Total findings: + CRITICAL: ← must-fix before production + WARNING: ← operational risk + INFO: ← polish + +Setup maturity: <0-100> + 100 – (CRITICAL × 10) – (WARNING × 3) – (INFO × 1) + ≥ 90: production-ready + 70–89: usable with caveats + < 70: requires remediation before relying on for production workloads +``` + +## Remediation DDL Generation + +For each finding, the skill emits a remediation snippet from the appropriate reference file: + +| Finding | Remediation source | +|---------|--------------------| +| Missing FUTURE grants | `references/rbac-patterns.md` § Privilege Reference | +| PII without masking | `references/governance-patterns.md` § Masking policies + Applying | +| No resource monitor | `references/cost-governance.md` § Resource Monitor Architecture | +| Warehouse no auto-suspend | `ALTER WAREHOUSE ... SET AUTO_SUSPEND = 60` | +| Orphaned custom role | `GRANT ROLE TO ROLE SYSADMIN;` | +| DEFAULT_ROLE = ACCOUNTADMIN | `ALTER USER SET DEFAULT_ROLE = ;` | +| Service account with password | Key-pair auth setup (Batch 5 network/security section) | +| No account-level network policy | `references/network-security-patterns.md` (Batch 5) | diff --git a/.opencode/skills/snowflake-setup/references/cost-governance.md b/.opencode/skills/snowflake-setup/references/cost-governance.md new file mode 100644 index 0000000000..0de7caff39 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/cost-governance.md @@ -0,0 +1,141 @@ +# Cost Governance Patterns + +## Resource Monitor Architecture + +Resource monitors work at two levels — account and warehouse. Set both: the account-level monitor is a hard ceiling; warehouse-level monitors catch runaway workloads per team before they drain the account budget. + +``` +Account monthly monitor (e.g. 500 credits) + └─ TRANSFORM_WH monitor (e.g. 200 credits) + └─ ANALYTICS_WH monitor (e.g. 150 credits) + └─ LOADING_WH monitor (e.g. 50 credits) +``` + +## Trigger Thresholds + +| Trigger | Action | When to use | +|---------|--------|-------------| +| 50% | NOTIFY | Early warning, useful for short-cycle periods (weekly) | +| 75% | NOTIFY | Standard early warning | +| 90% | NOTIFY | Critical warning — review before hitting limit | +| 100% | SUSPEND | Prevent further spend; alert immediately | + +`SUSPEND` stops the warehouse from starting new queries. In-flight queries complete. `SUSPEND_IMMEDIATE` kills in-flight queries too — use only as a last resort. + +## Credit Budget Sizing (Starting Points) + +These are starting points for a team of ~5 data engineers with a modest data volume. Adjust based on query complexity and data size. + +| Warehouse | Monthly credits (start) | Scale trigger | +|-----------|------------------------|---------------| +| LOADING_WH | 10–25 | Loader runs > 4 hours/day | +| TRANSFORM_WH | 50–150 | dbt build > 2 hours/day | +| ANALYTICS_WH | 75–200 | > 20 concurrent analyst sessions | +| DEV_WH | 20–50 | Dev team > 5 engineers | +| Account total | 2× sum of above | Catch untagged/unmonitored use | + +1 Snowflake credit ≈ $2–$4 USD depending on your contract (on-demand is higher). + +## Object Tagging for Cost Allocation + +A consistent tag taxonomy makes cost attribution easy in SNOWFLAKE.ACCOUNT_USAGE. + +### Recommended tag taxonomy + +```sql +-- Core tags +CREATE TAG ..cost_center + ALLOWED_VALUES 'engineering', 'marketing', 'finance', 'data-platform', 'ml'; +CREATE TAG ..team + ALLOWED_VALUES 'data-platform', 'analytics', 'ml-engineering', 'bi'; +CREATE TAG ..environment + ALLOWED_VALUES 'prod', 'staging', 'dev', 'sandbox'; +CREATE TAG ..project + COMMENT = 'Project or initiative name (free text)'; +``` + +### Apply tags + +```sql +-- Warehouses +ALTER WAREHOUSE TRANSFORM_WH SET TAG cost_center = 'data-platform', environment = 'prod'; +ALTER WAREHOUSE ANALYTICS_WH SET TAG cost_center = 'analytics', environment = 'prod'; + +-- Databases +ALTER DATABASE RAW SET TAG environment = 'prod'; +ALTER DATABASE TRANSFORM SET TAG environment = 'prod'; + +-- Schemas (inherit from database, but can override) +ALTER SCHEMA ANALYTICS.FINANCE SET TAG cost_center = 'finance'; +``` + +### Query cost by tag + +```sql +-- Cost per team (last 30 days) +SELECT + tag_value AS team, + SUM(credits_used) AS total_credits, + SUM(credits_used) * 3.0 AS estimated_cost_usd -- adjust multiplier to your rate +FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY wh +JOIN SNOWFLAKE.ACCOUNT_USAGE.TAG_REFERENCES tr + ON tr.object_name = wh.warehouse_name + AND tr.tag_name = 'TEAM' + AND tr.domain = 'WAREHOUSE' +WHERE wh.start_time >= DATEADD('day', -30, CURRENT_TIMESTAMP()) +GROUP BY 1 +ORDER BY 2 DESC; +``` + +## Detecting Cost Anomalies + +### Daily credit spike detection + +```sql +SELECT + DATE_TRUNC('hour', start_time) AS hour, + warehouse_name, + SUM(credits_used) AS credits, + LAG(SUM(credits_used)) OVER (PARTITION BY warehouse_name ORDER BY DATE_TRUNC('hour', start_time)) AS prev_hour, + SUM(credits_used) / NULLIF(LAG(SUM(credits_used)) OVER ( + PARTITION BY warehouse_name ORDER BY DATE_TRUNC('hour', start_time)), 0) AS ratio +FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY +WHERE start_time >= DATEADD('day', -7, CURRENT_TIMESTAMP()) +GROUP BY 1, 2 +HAVING ratio > 3 -- flag any hour that's 3× the prior hour +ORDER BY 4 DESC; +``` + +### Runaway query detection + +```sql +SELECT + query_id, user_name, warehouse_name, + query_text, + credits_used_cloud_services, + ROUND(total_elapsed_time / 1000 / 60, 1) AS duration_minutes, + bytes_scanned / 1e9 AS gb_scanned +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE start_time >= DATEADD('day', -1, CURRENT_TIMESTAMP()) + AND (credits_used_cloud_services > 1 OR total_elapsed_time > 1800000) -- > 1 credit or > 30min +ORDER BY credits_used_cloud_services DESC +LIMIT 20; +``` + +## Snowflake Storage Cost Management + +Storage is cheap ($23/TB/month compressed) but Fail-Safe and Time Travel add up on large tables. + +```sql +-- Find tables with long Time Travel windows (default 1 day, can be 0–90) +SELECT table_schema, table_name, data_retention_time_in_days, bytes / 1e9 AS size_gb +FROM INFORMATION_SCHEMA.TABLES +WHERE data_retention_time_in_days > 1 +ORDER BY bytes DESC; + +-- Reduce Time Travel for staging/raw tables (not needed for recovery) +ALTER TABLE RAW.SALESFORCE.accounts SET DATA_RETENTION_TIME_IN_DAYS = 0; + +-- Find tables not queried in 30+ days (candidates for archival) +-- Use finops_unused_resources tool for this — it wraps QUERY_HISTORY analysis +``` diff --git a/.opencode/skills/snowflake-setup/references/data-vault-patterns.md b/.opencode/skills/snowflake-setup/references/data-vault-patterns.md new file mode 100644 index 0000000000..b6f17a4714 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/data-vault-patterns.md @@ -0,0 +1,222 @@ +# Data Vault 2.0 Patterns + +Detailed modeling and DDL patterns for teams choosing Pattern D (Data Vault 2.0) in `topology-patterns.md`. This reference is only loaded when the user selects `data-vault-2` in the topology question. + +## Layer Responsibilities + +``` +RAW ──► RAW_VAULT ──► BUSINESS_VAULT ──► INFO_MARTS ──► BI / consumers + │ │ │ │ + │ │ │ └─ Dimensional / OBT views. Consumption-optimized. + │ │ │ Denormalized. May include masking policies. + │ │ └─ Computed satellites, PIT tables, bridges. Business rules applied. + │ │ Read-only by consumers; write access to BUSINESS_VAULT_BUILDER role. + │ └─ Hubs, Links, Satellites. Insert-only. Direct 1:1 with source systems. + │ Never mutated after load. Retained for audit. + └─ Raw ingestion. Same as other topologies. Loader roles write here. +``` + +## Hub Pattern + +One row per unique business key. Immutable once inserted. + +```sql +CREATE TABLE RAW_VAULT.HUBS.HUB_CUSTOMER ( + HK_CUSTOMER VARCHAR(32) NOT NULL, -- MD5(business_key) + BK_CUSTOMER_ID VARCHAR NOT NULL, -- the natural business key + LOAD_DATE TIMESTAMP_LTZ NOT NULL, + RECORD_SOURCE VARCHAR NOT NULL, -- e.g. 'salesforce.contacts' + CONSTRAINT PK_HUB_CUSTOMER PRIMARY KEY (HK_CUSTOMER) +) +CLUSTER BY (HK_CUSTOMER); +``` + +**Load pattern** (insert-only, dedupe by hash key): + +```sql +INSERT INTO RAW_VAULT.HUBS.HUB_CUSTOMER +SELECT + MD5(customer_id) AS HK_CUSTOMER, + customer_id AS BK_CUSTOMER_ID, + CURRENT_TIMESTAMP() AS LOAD_DATE, + 'salesforce.contacts' AS RECORD_SOURCE +FROM RAW.SALESFORCE.CONTACTS src +WHERE NOT EXISTS ( + SELECT 1 FROM RAW_VAULT.HUBS.HUB_CUSTOMER h + WHERE h.HK_CUSTOMER = MD5(src.customer_id) +); +``` + +## Link Pattern + +Associates two or more hubs. Insert-only. + +```sql +CREATE TABLE RAW_VAULT.LINKS.LINK_CUSTOMER_ORDER ( + HK_LINK VARCHAR(32) NOT NULL, -- MD5(HK_CUSTOMER || '|' || HK_ORDER) + HK_CUSTOMER VARCHAR(32) NOT NULL, + HK_ORDER VARCHAR(32) NOT NULL, + LOAD_DATE TIMESTAMP_LTZ NOT NULL, + RECORD_SOURCE VARCHAR NOT NULL, + CONSTRAINT PK_LINK_CUSTOMER_ORDER PRIMARY KEY (HK_LINK) +) +CLUSTER BY (HK_CUSTOMER, HK_ORDER); +``` + +## Satellite Pattern + +Descriptive attributes with hash-diff for change detection. Multiple satellites per hub allowed (split by source system or update cadence). + +```sql +CREATE TABLE RAW_VAULT.SATELLITES.SAT_CUSTOMER_DETAILS ( + HK_CUSTOMER VARCHAR(32) NOT NULL, + LOAD_DATE TIMESTAMP_LTZ NOT NULL, + HASHDIFF VARCHAR(32) NOT NULL, -- MD5(concat of all attribute cols) + FIRST_NAME VARCHAR, + LAST_NAME VARCHAR, + EMAIL VARCHAR, + PHONE VARCHAR, + RECORD_SOURCE VARCHAR NOT NULL, + CONSTRAINT PK_SAT_CUSTOMER_DETAILS PRIMARY KEY (HK_CUSTOMER, LOAD_DATE) +) +CLUSTER BY (HK_CUSTOMER); +``` + +**Load pattern** — only insert when HASHDIFF changed: + +```sql +INSERT INTO RAW_VAULT.SATELLITES.SAT_CUSTOMER_DETAILS +WITH src AS ( + SELECT + MD5(customer_id) AS HK_CUSTOMER, + CURRENT_TIMESTAMP() AS LOAD_DATE, + MD5(CONCAT_WS('|', first_name, last_name, email, phone)) + AS HASHDIFF, + first_name, last_name, email, phone, + 'salesforce.contacts' AS RECORD_SOURCE + FROM RAW.SALESFORCE.CONTACTS +), +latest AS ( + SELECT HK_CUSTOMER, HASHDIFF + FROM RAW_VAULT.SATELLITES.SAT_CUSTOMER_DETAILS + QUALIFY ROW_NUMBER() OVER (PARTITION BY HK_CUSTOMER ORDER BY LOAD_DATE DESC) = 1 +) +SELECT src.* +FROM src +LEFT JOIN latest ON src.HK_CUSTOMER = latest.HK_CUSTOMER +WHERE latest.HASHDIFF IS NULL -- first load + OR latest.HASHDIFF <> src.HASHDIFF; -- attributes changed +``` + +## RBAC for Data Vault + +Additional roles beyond the standard set: + +``` +VAULT_LOADER_ROLE + ├─ INSERT on RAW_VAULT.HUBS.*, RAW_VAULT.LINKS.*, RAW_VAULT.SATELLITES.* + ├─ SELECT on RAW.* + └─ USAGE on LOADING_WH + +BUSINESS_VAULT_BUILDER_ROLE + ├─ SELECT on RAW_VAULT.* + ├─ ALL on BUSINESS_VAULT.* + └─ USAGE on TRANSFORM_WH + +MART_BUILDER_ROLE + ├─ SELECT on RAW_VAULT.*, BUSINESS_VAULT.* + ├─ ALL on INFO_MARTS.* + └─ USAGE on TRANSFORM_WH + +ANALYST_ROLE (consumers) + ├─ SELECT on INFO_MARTS.* -- default consumers see only marts + └─ SELECT on BUSINESS_VAULT.* (opt-in; power users only) + -- Note: no direct access to RAW_VAULT.* for consumers; forces mart usage +``` + +**Insert-only enforcement**: revoke UPDATE and DELETE on RAW_VAULT explicitly, even if the loader role wouldn't normally have them, as a defense-in-depth measure: + +```sql +REVOKE UPDATE, DELETE ON ALL TABLES IN SCHEMA RAW_VAULT.HUBS FROM ROLE VAULT_LOADER_ROLE; +REVOKE UPDATE, DELETE ON ALL TABLES IN SCHEMA RAW_VAULT.LINKS FROM ROLE VAULT_LOADER_ROLE; +REVOKE UPDATE, DELETE ON ALL TABLES IN SCHEMA RAW_VAULT.SATELLITES FROM ROLE VAULT_LOADER_ROLE; +``` + +## Governance for Data Vault + +**PII placement decision.** In DV2 you have three options for where to apply masking: + +1. **RAW_VAULT satellites** — mask at the earliest possible layer; auditors see masked values in the vault; strict interpretation of least-privilege +2. **BUSINESS_VAULT / INFO_MARTS only** — RAW_VAULT stays plaintext for compliance / breach-response needs; access to RAW_VAULT is heavily restricted to auditors + platform admins only +3. **Column-level in RAW_VAULT + row-access in INFO_MARTS** — hybrid: mask sensitive attributes at load, restrict rows at consumption + +Recommended default: **option 2** for regulated industries (finance, healthcare) where full source fidelity is a compliance requirement; **option 1** for other use cases. The skill asks the user which they want. + +## dbt Package Recommendation + +Two mature packages generate DV2 boilerplate: + +- **AutomateDV** (formerly dbtvault) — active, supports Snowflake/BigQuery/Databricks, extensive macros +- **dbtvault** (original) — largely superseded by AutomateDV; use only if already deployed + +The skill emits stub `dbt_project.yml`, `packages.yml`, and one example hub/link/satellite model per source. It does NOT auto-generate a full vault schema — that requires source analysis the skill doesn't have. + +Example `packages.yml`: +```yaml +packages: + - package: Datavault-UK/automate_dv + version: 0.11.0 +``` + +Example hub model (`models/raw_vault/hubs/hub_customer.sql`): +```sql +{{- config(materialized='incremental', unique_key='hk_customer', on_schema_change='fail') -}} + +{{- automate_dv.hub( + src_pk="hk_customer", + src_nk="bk_customer_id", + src_ldts="load_date", + src_source="record_source", + source_model="stg_customers" +) -}} +``` + +## Cost Profile + +DV2 storage and compute are meaningfully higher than Medallion/Functional. Adjust warehouse budgets: + +| Warehouse | Medallion baseline | DV2 adjustment | +|-----------|-------------------|----------------| +| LOADING_WH | 10–25 credits/month | 25–50 (satellite hash-diff compute) | +| TRANSFORM_WH | 50–150 | 100–300 (vault + mart layers) | +| ANALYTICS_WH | 75–200 | 75–200 (same — INFO_MARTS presents pre-computed views) | +| Storage | baseline | +30–50% (append-only satellites) | + +## Ingestion Pattern Differences + +- **Snowpipe / Task+COPY into RAW** — unchanged (same as any topology) +- **RAW → RAW_VAULT** — done by dbt (or Snowflake tasks) using the hub/link/satellite patterns above; run frequently (hourly for hot data, nightly for reference data) +- **RAW_VAULT → BUSINESS_VAULT** — dbt models; often run once per day +- **BUSINESS_VAULT → INFO_MARTS** — dbt models; may materialize as tables (nightly) or views (real-time) + +## Common Data Vault Mistakes + +### Mistake 1: Using natural keys as primary keys +**Symptom:** Loads become dependent on foreign key relationships; parallel loads deadlock. +**Fix:** Always use hash keys (`HK_*`); hash keys let hubs, links, and satellites load independently. + +### Mistake 2: Skipping HASHDIFF on satellites +**Symptom:** Satellites grow linearly with load frequency instead of change frequency; storage explodes. +**Fix:** Always compute and store HASHDIFF; only insert when HASHDIFF differs from the latest row for that hash key. + +### Mistake 3: Consumers querying RAW_VAULT directly +**Symptom:** BI tools crash on 5-way joins across hubs/links/satellites; analysts write incorrect queries. +**Fix:** Route all consumption through INFO_MARTS (dimensional or one-big-table views). RAW_VAULT is a modeling layer, not a query layer. + +### Mistake 4: Mixing insert-only and mutable in RAW_VAULT +**Symptom:** Audit trail broken; row counts don't reconcile against source systems. +**Fix:** Enforce insert-only via REVOKE UPDATE / DELETE on the loader role. Even reference-data changes go through a new satellite record with a new LOAD_DATE. + +### Mistake 5: Hashing without a delimiter +**Symptom:** `MD5(first_name || last_name)` collisions — "John|Doe" and "JohnD|oe" produce the same hash. +**Fix:** Always use a delimiter (`|`, `~`, or `\x1f`) in `CONCAT_WS` when building hash inputs. diff --git a/.opencode/skills/snowflake-setup/references/governance-patterns.md b/.opencode/skills/snowflake-setup/references/governance-patterns.md new file mode 100644 index 0000000000..e5a9827db7 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/governance-patterns.md @@ -0,0 +1,201 @@ +# Data Governance Patterns + +## Dynamic Data Masking — Template Library + +### Email: show domain only to analysts + +```sql +CREATE MASKING POLICY ..mask_email + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE') THEN val + WHEN CURRENT_ROLE() IN ('ANALYST_ROLE', 'BI_ROLE') THEN REGEXP_REPLACE(val, '^[^@]+', '****') + ELSE '****@****.***' + END; +``` + +### Phone: last 4 digits only + +```sql +CREATE MASKING POLICY ..mask_phone + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN') THEN val + ELSE CONCAT('***-***-', RIGHT(REGEXP_REPLACE(val, '[^0-9]', ''), 4)) + END; +``` + +### SSN: completely hidden except for data platform admin + +```sql +CREATE MASKING POLICY ..mask_ssn + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() = 'DATA_PLATFORM_ADMIN' THEN val + ELSE '***-**-****' + END; +``` + +### Credit card: last 4 digits + +```sql +CREATE MASKING POLICY ..mask_credit_card + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() = 'DATA_PLATFORM_ADMIN' THEN val + ELSE CONCAT('****-****-****-', RIGHT(REGEXP_REPLACE(val, '[^0-9]', ''), 4)) + END; +``` + +### Full name: first name + last initial + +```sql +CREATE MASKING POLICY ..mask_full_name + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE') THEN val + ELSE CONCAT(SPLIT_PART(val, ' ', 1), ' ', LEFT(SPLIT_PART(val, ' ', 2), 1), '.') + END; +``` + +### Date of birth: year only + +```sql +CREATE MASKING POLICY ..mask_dob + AS (val DATE) RETURNS DATE -> + CASE + WHEN CURRENT_ROLE() = 'DATA_PLATFORM_ADMIN' THEN val + ELSE DATE_FROM_PARTS(YEAR(val), 1, 1) -- show Jan 1 of birth year + END; +``` + +## Applying Masking Policies + +```sql +-- Apply to a column +ALTER TABLE RAW.SALESFORCE.CONTACTS + MODIFY COLUMN email SET MASKING POLICY mask_email; + +-- Remove a masking policy +ALTER TABLE RAW.SALESFORCE.CONTACTS + MODIFY COLUMN email UNSET MASKING POLICY; + +-- View policies applied to a table +SELECT column_name, masking_policy_name +FROM INFORMATION_SCHEMA.COLUMNS c +JOIN TABLE(INFORMATION_SCHEMA.POLICY_REFERENCES( + REF_ENTITY_NAME => 'RAW.SALESFORCE.CONTACTS', + REF_ENTITY_DOMAIN => 'TABLE' +)) pr ON c.column_name = pr.ref_column_name; +``` + +## Row Access Policies + +### Pattern 1 — Role-based row filtering + +```sql +-- Users with DATA_PLATFORM_ADMIN see all rows; others see nothing by default +CREATE ROW ACCESS POLICY ..admin_only_access + AS (dummy_col VARCHAR) RETURNS BOOLEAN -> + CURRENT_ROLE() = 'DATA_PLATFORM_ADMIN'; +``` + +### Pattern 2 — User-to-data mapping table + +```sql +-- Create a mapping table defining which team_id each user can see +CREATE TABLE ANALYTICS.CORE.user_data_access_map ( + user_email VARCHAR, + team_id VARCHAR +); + +-- Row access policy referencing the mapping table +CREATE ROW ACCESS POLICY ANALYTICS.CORE.team_access_policy + AS (team_id VARCHAR) RETURNS BOOLEAN -> + EXISTS ( + SELECT 1 FROM ANALYTICS.CORE.user_data_access_map m + WHERE m.user_email = CURRENT_USER() + AND m.team_id = team_id + ) + OR CURRENT_ROLE() = 'DATA_PLATFORM_ADMIN'; + +-- Apply to table +ALTER TABLE ANALYTICS.CORE.orders + ADD ROW ACCESS POLICY ANALYTICS.CORE.team_access_policy ON (team_id); +``` + +### Pattern 3 — Environment-based filtering (dev sees sample only) + +```sql +CREATE ROW ACCESS POLICY ANALYTICS.CORE.env_sample_policy + AS (created_at TIMESTAMP_LTZ) RETURNS BOOLEAN -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'TRANSFORM_ROLE') THEN TRUE + ELSE created_at >= DATEADD('day', -90, CURRENT_TIMESTAMP()) -- analysts see last 90 days only + END; +``` + +## Object Tagging for PII + +```sql +-- Tag taxonomy +CREATE TAG ..pii_category + ALLOWED_VALUES 'email', 'phone', 'ssn', 'name', 'address', 'dob', 'credit_card', 'ip_address', 'device_id'; + +CREATE TAG ..data_sensitivity + ALLOWED_VALUES 'public', 'internal', 'confidential', 'restricted'; + +-- Apply at column level +ALTER TABLE RAW.SALESFORCE.CONTACTS + MODIFY COLUMN email SET TAG pii_category = 'email', data_sensitivity = 'restricted'; +ALTER TABLE RAW.SALESFORCE.CONTACTS + MODIFY COLUMN phone SET TAG pii_category = 'phone', data_sensitivity = 'confidential'; +ALTER TABLE RAW.SALESFORCE.CONTACTS + MODIFY COLUMN first_name SET TAG pii_category = 'name', data_sensitivity = 'confidential'; + +-- Query all PII-tagged columns across account +SELECT tag_database, tag_schema, tag_name, tag_value, + object_database, object_schema, object_name, column_name +FROM SNOWFLAKE.ACCOUNT_USAGE.TAG_REFERENCES +WHERE tag_name = 'PII_CATEGORY' +ORDER BY object_database, object_schema, object_name, column_name; +``` + +## Governance Validation Queries + +```sql +-- Tables with no masking policies (check for PII exposure) +SELECT t.table_schema, t.table_name, t.table_type +FROM INFORMATION_SCHEMA.TABLES t +WHERE t.table_schema NOT IN ('INFORMATION_SCHEMA') + AND t.table_name NOT IN ( + SELECT DISTINCT ref_entity_name + FROM TABLE(INFORMATION_SCHEMA.POLICY_REFERENCES( + REF_ENTITY_DOMAIN => 'TABLE' + )) + ); + +-- Verify masking as a role +USE ROLE ANALYST_ROLE; +SELECT email, phone, first_name FROM RAW.SALESFORCE.CONTACTS LIMIT 5; +-- Should see masked values, not plaintext + +USE ROLE DATA_PLATFORM_ADMIN; +SELECT email, phone, first_name FROM RAW.SALESFORCE.CONTACTS LIMIT 5; +-- Should see plaintext + +-- Policies currently in effect +SHOW MASKING POLICIES IN ACCOUNT; +SHOW ROW ACCESS POLICIES IN ACCOUNT; +``` + +## Governance Checklist + +- [ ] PII classification run on all RAW schemas (`altimate_core_classify_pii` or `schema_detect_pii`) +- [ ] PII tag taxonomy created and applied to identified columns +- [ ] Masking policies created for each PII category (email, phone, SSN, name, DOB, credit card) +- [ ] Masking policies applied at the RAW layer (not just marts) +- [ ] Masking verified by querying as ANALYST_ROLE — confirming masked output +- [ ] Row access policies applied where multi-tenant or domain-restricted data exists +- [ ] `data_sensitivity = 'restricted'` tagged columns only accessible to DATA_PLATFORM_ADMIN +- [ ] PII inventory documented in schema.yml column descriptions (use `altimate_core_classify_pii` output) diff --git a/.opencode/skills/snowflake-setup/references/idempotency-patterns.md b/.opencode/skills/snowflake-setup/references/idempotency-patterns.md new file mode 100644 index 0000000000..a8eb8f5416 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/idempotency-patterns.md @@ -0,0 +1,253 @@ +# Idempotency and Rollback Patterns + +## Emission Modes + +The skill supports three emission modes for every DDL section. Choose based on the target account state. + +| Mode | Behavior | Use when | +|------|----------|----------| +| `strict` | Plain `CREATE` — fails if object exists | Fresh account, want to be sure nothing collides | +| `idempotent` | `CREATE ... IF NOT EXISTS` (creates if missing, skips if present) — **default** | Rerunning setup safely; partial state possible | +| `additive` | Detects existing objects first via `SHOW`/`INFORMATION_SCHEMA`, emits DDL only for missing ones | Extending a partially-configured account without touching existing objects | + +**Never use `CREATE OR REPLACE` for objects that hold state or grants** — see danger list below. + +## Idempotent DDL Reference + +### Safe with `CREATE IF NOT EXISTS` +```sql +CREATE DATABASE IF NOT EXISTS RAW COMMENT = 'Raw ingestion layer'; +CREATE SCHEMA IF NOT EXISTS RAW.SALESFORCE; +CREATE ROLE IF NOT EXISTS TRANSFORM_ROLE; +CREATE WAREHOUSE IF NOT EXISTS TRANSFORM_WH + WAREHOUSE_SIZE = 'SMALL' AUTO_SUSPEND = 60 AUTO_RESUME = TRUE; +CREATE FILE FORMAT IF NOT EXISTS RAW.PUBLIC.csv_standard TYPE = 'CSV'; +CREATE STAGE IF NOT EXISTS RAW.SALESFORCE.s3_stage + STORAGE_INTEGRATION = s3_raw_integration + URL = 's3://your-data-bucket/salesforce/'; +CREATE TABLE IF NOT EXISTS RAW.SALESFORCE.ACCOUNTS (...); +CREATE TAG IF NOT EXISTS RAW.PUBLIC.cost_center; +CREATE PIPE IF NOT EXISTS RAW.SALESFORCE.accounts_pipe + AUTO_INGEST = TRUE AS COPY INTO ...; +CREATE TASK IF NOT EXISTS RAW.SALESFORCE.load_accounts_hourly + WAREHOUSE = LOADING_WH SCHEDULE = '...' AS ...; +``` + +### `CREATE OR REPLACE` — use only for these +`OR REPLACE` drops and recreates. Safe when the object holds no state, no grants, and no dependent objects. + +```sql +-- Masking policies and row access policies: no state, easy to reapply +CREATE OR REPLACE MASKING POLICY mask_email AS (val STRING) RETURNS STRING -> ...; +CREATE OR REPLACE ROW ACCESS POLICY team_access_policy AS (team_id VARCHAR) RETURNS BOOLEAN -> ...; + +-- File formats: no state, references are re-resolved +CREATE OR REPLACE FILE FORMAT RAW.PUBLIC.csv_standard TYPE = 'CSV' ...; +``` + +### **DANGER: never use `CREATE OR REPLACE` for** + +| Object | Why it's dangerous | +|--------|-------------------| +| `WAREHOUSE` | Terminates active connections; in-flight queries fail | +| `ROLE` | Drops all grants ON and TO the role; downstream users lose access silently | +| `USER` | Drops password/keys; service accounts start failing auth immediately | +| `DATABASE` / `SCHEMA` | Drops all contained objects (tables, views, pipes, tasks) — catastrophic data loss | +| `TABLE` (non-transient) | Drops data. Time Travel can recover but only within retention window | +| `STORAGE INTEGRATION` | Regenerates external ID; requires re-running IAM trust policy setup | +| `PIPE` | Loses load history; may re-ingest already-loaded files | +| `RESOURCE MONITOR` | Loses accumulated credit usage; effectively resets the budget mid-period | + +If any of these need to change, use `ALTER` instead — never `CREATE OR REPLACE`. + +## `ALTER` vs `CREATE OR REPLACE` Cheatsheet + +```sql +-- Warehouse: use ALTER +ALTER WAREHOUSE TRANSFORM_WH SET WAREHOUSE_SIZE = 'MEDIUM' AUTO_SUSPEND = 120; + +-- Task schedule change: use ALTER (must SUSPEND first) +ALTER TASK RAW.SALESFORCE.load_accounts_hourly SUSPEND; +ALTER TASK RAW.SALESFORCE.load_accounts_hourly SET SCHEDULE = 'USING CRON 0 */2 * * * UTC'; +ALTER TASK RAW.SALESFORCE.load_accounts_hourly RESUME; + +-- Resource monitor threshold: use ALTER +ALTER RESOURCE MONITOR account_monitor SET CREDIT_QUOTA = 750; + +-- Table column addition: use ALTER, not OR REPLACE +ALTER TABLE RAW.SALESFORCE.ACCOUNTS ADD COLUMN email VARCHAR; +``` + +## Grants: Idempotent by Design + +Snowflake `GRANT` is naturally idempotent — running the same grant twice is a no-op. No `IF NOT EXISTS` variant needed or supported. + +```sql +GRANT SELECT ON ALL TABLES IN SCHEMA RAW.SALESFORCE TO ROLE ANALYST_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA RAW.SALESFORCE TO ROLE ANALYST_ROLE; +-- Rerun safely: no error, no state change +``` + +**Exception:** `GRANT ROLE X TO ROLE Y` creates a hierarchy edge. Running twice is a no-op, but revoking requires the reverse `REVOKE ROLE X FROM ROLE Y`. + +## Additive Mode: Existence Checks + +Before emitting DDL in `additive` mode, run these checks and skip anything that already exists. + +```sql +-- Databases present? +SELECT database_name FROM SNOWFLAKE.INFORMATION_SCHEMA.DATABASES +WHERE database_name IN ('RAW', 'TRANSFORM', 'ANALYTICS'); + +-- Warehouses present? +SHOW WAREHOUSES LIKE 'TRANSFORM_WH'; + +-- Roles present? +SHOW ROLES LIKE 'TRANSFORM_ROLE'; + +-- Resource monitors present? +SHOW RESOURCE MONITORS; + +-- Storage integrations present? +SHOW INTEGRATIONS LIKE 's3_raw_integration'; + +-- Pipes on a schema? +SHOW PIPES IN SCHEMA RAW.SALESFORCE; + +-- Masking policies on a table? +SELECT ref_column_name, policy_name +FROM TABLE(INFORMATION_SCHEMA.POLICY_REFERENCES( + REF_ENTITY_NAME => 'RAW.SALESFORCE.CONTACTS', + REF_ENTITY_DOMAIN => 'TABLE' +)); +``` + +## Rollback Script Generation + +Every setup produces companion rollback scripts. The rollback DROPs objects in strict reverse dependency order. + +### Delivery: two files, not one + +The altimate-code `sql_execute` tool has a non-bypassable safety guard that blocks `DROP DATABASE`, `DROP SCHEMA`, and `TRUNCATE`. Emit rollback as **two files** so the tool-safe portion can auto-execute and the manual portion is clearly flagged: + +- `rollback-tool-safe.sql` — steps 1–7 and 9–14 below. Runs via `sql_execute`. +- `rollback-manual.sql` — steps 7 (schemas) and 8 (databases) only. User must run this via `snowsql`, Snowsight, or a direct `snowflake-sdk` script. + +Both files begin with the same account-locator confirmation guard. `rollback-manual.sql` is a small file — usually just a few `DROP DATABASE IF EXISTS` and `DROP SCHEMA IF EXISTS` lines — but must never be omitted or the account is left in a partially-torn-down state. + +### Dependency Order (drop in this sequence) + +``` +1. Row access policies (unset from tables, then drop) [tool-safe] +2. Masking policies (unset from columns, then drop) [tool-safe] +3. Tag references (unset tags, then drop tag definitions) [tool-safe] +4. Pipes and tasks [tool-safe] +5. External stages and file formats [tool-safe] +6. Tables and views [tool-safe] +7. Schemas [MANUAL — DROP SCHEMA blocked by tool guard] +8. Databases [MANUAL — DROP DATABASE blocked by tool guard] +9. Warehouses [tool-safe] +10. Resource monitors (must be unset from warehouses first) [tool-safe] +11. Grants (revoke from custom roles) [tool-safe] +12. Custom roles (revoke from parent roles first) [tool-safe] +13. Service account users [tool-safe] +14. Storage integrations and notification integrations [tool-safe] +``` + +### Rollback Template Fragments + +```sql +-- 1. Policies: unset before drop +USE ROLE SYSADMIN; +ALTER TABLE RAW.SALESFORCE.CONTACTS MODIFY COLUMN email UNSET MASKING POLICY; +ALTER TABLE ANALYTICS.CORE.orders DROP ROW ACCESS POLICY team_access_policy; +DROP MASKING POLICY IF EXISTS mask_email; +DROP ROW ACCESS POLICY IF EXISTS team_access_policy; + +-- 2. Tags +ALTER WAREHOUSE TRANSFORM_WH UNSET TAG cost_center, environment; +DROP TAG IF EXISTS cost_center; + +-- 3. Pipes and tasks (suspend before drop) +ALTER TASK RAW.SALESFORCE.load_accounts_hourly SUSPEND; +DROP TASK IF EXISTS RAW.SALESFORCE.load_accounts_hourly; +DROP PIPE IF EXISTS RAW.SALESFORCE.accounts_pipe; + +-- 4. Stages and file formats +DROP STAGE IF EXISTS RAW.SALESFORCE.s3_stage; +DROP FILE FORMAT IF EXISTS RAW.PUBLIC.csv_standard; + +-- 5. Databases (this drops all contained objects) +USE ROLE ACCOUNTADMIN; +DROP DATABASE IF EXISTS RAW; +DROP DATABASE IF EXISTS TRANSFORM; +DROP DATABASE IF EXISTS ANALYTICS; + +-- 6. Warehouses +DROP WAREHOUSE IF EXISTS LOADING_WH; +DROP WAREHOUSE IF EXISTS TRANSFORM_WH; +DROP WAREHOUSE IF EXISTS ANALYTICS_WH; + +-- 7. Resource monitors (unset from warehouses first) +ALTER WAREHOUSE TRANSFORM_WH UNSET RESOURCE_MONITOR; +DROP RESOURCE MONITOR IF EXISTS transform_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS account_monitor; + +-- 8. Roles (revoke from SYSADMIN first) +USE ROLE SECURITYADMIN; +REVOKE ROLE TRANSFORM_ROLE FROM ROLE SYSADMIN; +DROP ROLE IF EXISTS TRANSFORM_ROLE; + +-- 9. Service accounts +DROP USER IF EXISTS fivetran_loader; +DROP USER IF EXISTS dbt_service; + +-- 10. Storage integrations +USE ROLE ACCOUNTADMIN; +DROP INTEGRATION IF EXISTS s3_raw_integration; +``` + +## Rollback Safety Rules + +The generated rollback script **must**: + +1. Use `IF EXISTS` on every `DROP` — never fail because an object was already removed +2. Include an interactive prompt at the top requiring the user to type the account identifier before destructive statements execute (prevents accidental prod rollback) +3. Never emit `DROP ... CASCADE` — cascade drops mask dependency errors that reveal misconfigured environments +4. Never drop the built-in `PUBLIC` role, `ACCOUNTADMIN`, `SECURITYADMIN`, `SYSADMIN`, `USERADMIN`, or `ORGADMIN` roles +5. Never drop the `SNOWFLAKE` database or `INFORMATION_SCHEMA` schemas +6. Emit `SUSPEND` before dropping any `TASK` or `PIPE` +7. Emit `UNSET` for masking/row-access policies and tags before dropping the policy/tag definition +8. Emit `UNSET RESOURCE_MONITOR` on warehouses before dropping the monitor +9. Include a `WHERE environment != 'prod'` guard when running against a mixed-env account, or refuse to run without an explicit `--include-prod` flag + +## Confirmation Prompt Template + +```sql +-- ============================================================ +-- ROLLBACK CONFIRMATION +-- ============================================================ +-- This script will DROP: +-- - 3 databases (RAW, TRANSFORM, ANALYTICS) and ALL contained data +-- - 4 warehouses +-- - 5 custom roles and all their grants +-- - 2 service accounts +-- - 1 storage integration +-- +-- Estimated data loss: ~ GB across tables +-- Time Travel retention: days (data may be recoverable within window) +-- +-- To proceed, uncomment the following line by removing the `-- ` prefix: +-- +-- SET rollback_confirmed_account = ''; +-- +-- The script will fail at the first destructive statement if this is not set +-- or does not match CURRENT_ACCOUNT(). +-- ============================================================ + +-- Guard at top of destructive section +SELECT CASE + WHEN $rollback_confirmed_account = CURRENT_ACCOUNT() THEN 'proceed' + ELSE ERROR('Rollback account mismatch or unconfirmed. Refusing to drop objects.') +END; +``` diff --git a/.opencode/skills/snowflake-setup/references/ingestion-patterns.md b/.opencode/skills/snowflake-setup/references/ingestion-patterns.md new file mode 100644 index 0000000000..ec6d464589 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/ingestion-patterns.md @@ -0,0 +1,232 @@ +# Data Ingestion Patterns + +## Decision Tree: Which ingestion method? + +``` +Data already in S3/GCS/Azure Blob? + ├─ Yes, continuous (event-driven, files land within minutes) → Snowpipe + ├─ Yes, scheduled batch (hourly/daily dumps) → Task + COPY INTO + └─ No, need to move it there first: + ├─ SaaS source (Salesforce, HubSpot, Stripe, etc.) → Third-party connector (Fivetran, Airbyte) + ├─ Operational DB (Postgres, MySQL, MongoDB) → CDC connector or pg_dump + S3 + COPY + └─ Streaming (Kafka, Kinesis) → Kafka Connector for Snowflake (Snowpipe Streaming) +``` + +## Pattern 1 — Snowpipe (Event-Driven Continuous Load) + +Best for: Files landing in S3/GCS/Azure within minutes of creation, near-real-time latency requirements. + +### Full setup sequence + +```sql +-- 1. Storage integration (ACCOUNTADMIN required) +CREATE STORAGE INTEGRATION s3_raw_integration + TYPE = EXTERNAL_STAGE + STORAGE_PROVIDER = 'S3' + ENABLED = TRUE + STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::123456789012:role/snowflake-s3-role' + STORAGE_ALLOWED_LOCATIONS = ('s3://your-data-bucket/'); + +DESC INTEGRATION s3_raw_integration; +-- Note: STORAGE_AWS_IAM_USER_ARN and STORAGE_AWS_EXTERNAL_ID +-- Add these to the IAM role's trust policy in AWS + +-- 2. External stage +CREATE STAGE RAW.SALESFORCE.s3_stage + STORAGE_INTEGRATION = s3_raw_integration + URL = 's3://your-data-bucket/salesforce/' + FILE_FORMAT = (TYPE = 'PARQUET'); + +-- 3. Target table +CREATE TABLE RAW.SALESFORCE.ACCOUNTS ( + _airbyte_raw_id VARCHAR, + _airbyte_emitted_at TIMESTAMP_LTZ, + _airbyte_data VARIANT -- for JSON/Parquet with dynamic schema +); + +-- 4. Pipe +CREATE PIPE RAW.SALESFORCE.accounts_pipe + AUTO_INGEST = TRUE + AS + COPY INTO RAW.SALESFORCE.ACCOUNTS + FROM @RAW.SALESFORCE.s3_stage/accounts/ + FILE_FORMAT = (TYPE = 'PARQUET'); + +-- 5. Get the SQS queue ARN for S3 event notification +SHOW PIPES IN SCHEMA RAW.SALESFORCE; +-- Use the notification_channel value in S3 → Properties → Event notifications +``` + +### AWS S3 event notification setup (manual step) +- S3 bucket → Properties → Event notifications → Create +- Event type: `s3:ObjectCreated:*` +- Prefix: `salesforce/accounts/` (match your stage path) +- Destination: SQS queue ARN from `SHOW PIPES` + +### Monitor Snowpipe + +```sql +-- Check pipe status +SELECT SYSTEM$PIPE_STATUS('RAW.SALESFORCE.ACCOUNTS_PIPE'); + +-- Recent load history +SELECT * FROM TABLE(INFORMATION_SCHEMA.COPY_HISTORY( + TABLE_NAME => 'RAW.SALESFORCE.ACCOUNTS', + START_TIME => DATEADD('hour', -24, CURRENT_TIMESTAMP()) +)) +ORDER BY last_load_time DESC; + +-- Files with errors +SELECT stage_location, file_name, error_count, status +FROM TABLE(INFORMATION_SCHEMA.COPY_HISTORY( + TABLE_NAME => 'RAW.SALESFORCE.ACCOUNTS', + START_TIME => DATEADD('day', -7, CURRENT_TIMESTAMP()) +)) +WHERE status != 'Loaded' +ORDER BY last_load_time DESC; +``` + +## Pattern 2 — Task + COPY INTO (Scheduled Batch) + +Best for: Scheduled batch files (hourly/daily), simpler setup than Snowpipe, no S3 event notification required. + +```sql +-- File format (define once, reuse across tables) +CREATE FILE FORMAT RAW.PUBLIC.csv_standard + TYPE = 'CSV' + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + NULL_IF = ('NULL', 'null', '', '\\N') + EMPTY_FIELD_AS_NULL = TRUE + DATE_FORMAT = 'AUTO' + TIMESTAMP_FORMAT = 'AUTO' + SKIP_HEADER = 1; + +-- Task runs every hour +CREATE TASK RAW.SALESFORCE.load_accounts_hourly + WAREHOUSE = LOADING_WH + SCHEDULE = 'USING CRON 0 * * * * UTC' +AS + COPY INTO RAW.SALESFORCE.ACCOUNTS + FROM @RAW.SALESFORCE.s3_stage/accounts/ + FILE_FORMAT = (FORMAT_NAME = RAW.PUBLIC.csv_standard) + ON_ERROR = 'CONTINUE' -- skip bad files, log errors + PURGE = FALSE; -- don't delete source files after load + +-- Enable the task (tasks start suspended) +ALTER TASK RAW.SALESFORCE.load_accounts_hourly RESUME; + +-- Check task history +SELECT * +FROM TABLE(INFORMATION_SCHEMA.TASK_HISTORY( + TASK_NAME => 'LOAD_ACCOUNTS_HOURLY', + SCHEDULED_TIME_RANGE_START => DATEADD('day', -7, CURRENT_TIMESTAMP()) +)) +ORDER BY scheduled_time DESC; +``` + +## Pattern 3 — Third-Party Connectors (Fivetran / Airbyte) + +For SaaS sources (Salesforce, HubSpot, Stripe, GitHub, etc.), use a managed connector. Snowflake setup: + +```sql +-- Create a dedicated schema per connector (Fivetran convention) +CREATE SCHEMA RAW.FIVETRAN_SALESFORCE; +CREATE SCHEMA RAW.FIVETRAN_STRIPE; +CREATE SCHEMA RAW.AIRBYTE_HUBSPOT; + +-- Create a loader service account for the connector +CREATE USER fivetran_loader + DEFAULT_ROLE = LOADER_ROLE + DEFAULT_WAREHOUSE = LOADING_WH + MUST_CHANGE_PASSWORD = FALSE; +GRANT ROLE LOADER_ROLE TO USER fivetran_loader; + +-- Fivetran requires CREATE TABLE + MODIFY on its schemas +GRANT ALL ON SCHEMA RAW.FIVETRAN_SALESFORCE TO ROLE LOADER_ROLE; +GRANT ALL ON FUTURE TABLES IN SCHEMA RAW.FIVETRAN_SALESFORCE TO ROLE LOADER_ROLE; +``` + +In Fivetran/Airbyte: use the `LOADING_WH` warehouse, the `fivetran_loader` user, and point to `RAW.FIVETRAN_SALESFORCE` as the destination schema. + +## Pattern 4 — Snowpipe Streaming (Kafka / Real-Time) + +For Kafka topics or high-throughput streams needing sub-minute latency. + +```sql +-- Snowpipe Streaming uses a different API (not COPY INTO) +-- Create the target table with CLUSTER BY for query performance +CREATE TABLE RAW.EVENTS.clickstream ( + event_id VARCHAR, + session_id VARCHAR, + event_type VARCHAR, + properties VARIANT, + received_at TIMESTAMP_LTZ +) +CLUSTER BY (DATE_TRUNC('day', received_at)); +``` + +The Kafka Connector for Snowflake handles the pipe creation automatically. Configure it with: +- `snowflake.url.name`: your account URL +- `snowflake.user.name`: streaming service account +- `snowflake.private.key`: base64-encoded private key (key-pair auth required for Snowpipe Streaming) +- `snowflake.database`, `snowflake.schema`: target location + +## Loading from Internal Stages (Local Files) + +```sql +-- Create an internal (Snowflake-managed) stage +CREATE STAGE RAW.SALESFORCE.local_uploads; + +-- Upload files via SnowSQL CLI +-- snowsql -q "PUT file:///path/to/data.csv @RAW.SALESFORCE.local_uploads" + +-- Load from internal stage +COPY INTO RAW.SALESFORCE.ACCOUNTS +FROM @RAW.SALESFORCE.local_uploads/data.csv +FILE_FORMAT = (FORMAT_NAME = RAW.PUBLIC.csv_standard); + +-- Clean up after load +REMOVE @RAW.SALESFORCE.local_uploads/data.csv; +``` + +## File Format Reference + +```sql +-- JSON (semistructured data) +CREATE FILE FORMAT json_standard + TYPE = 'JSON' + STRIP_OUTER_ARRAY = TRUE -- if file is a JSON array at top level + NULL_IF = ('null', 'NULL'); + +-- Parquet (best compression and performance) +CREATE FILE FORMAT parquet_standard + TYPE = 'PARQUET' + SNAPPY_COMPRESSION = TRUE; + +-- Avro +CREATE FILE FORMAT avro_standard + TYPE = 'AVRO'; + +-- ORC +CREATE FILE FORMAT orc_standard + TYPE = 'ORC'; +``` + +## COPY INTO Best Practices + +```sql +-- Load only new files (Snowflake tracks loaded files by default) +COPY INTO
FROM @ + FILE_FORMAT = (FORMAT_NAME = ) + ON_ERROR = 'CONTINUE' -- don't fail the whole batch on one bad file + FORCE = FALSE; -- default; FORCE = TRUE reloads already-loaded files + +-- Validate without loading (dry run) +COPY INTO
FROM @ + FILE_FORMAT = (FORMAT_NAME = ) + VALIDATION_MODE = 'RETURN_ERRORS'; + +-- Load a specific file +COPY INTO
FROM @/ + FILE_FORMAT = (FORMAT_NAME = ); +``` diff --git a/.opencode/skills/snowflake-setup/references/rbac-patterns.md b/.opencode/skills/snowflake-setup/references/rbac-patterns.md new file mode 100644 index 0000000000..a1bfc10c29 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/rbac-patterns.md @@ -0,0 +1,148 @@ +# RBAC Patterns + +## Functional Role vs. Access Role Distinction + +Snowflake's best practice splits roles into two layers: + +- **Functional roles** — named for what someone *does* (ANALYST_ROLE, LOADER_ROLE, TRANSFORM_ROLE). These are what users and service accounts are granted. +- **Access roles** — named for what data they *touch* (SALESFORCE_READ, ANALYTICS_WRITE). Functional roles are granted to access roles, not directly to objects. + +For small teams (< 10 people), the two-layer model is overkill — use functional roles directly and grant object privileges to them. + +## Reference RBAC Topology (small team) + +``` +ACCOUNTADMIN + └─ SYSADMIN + ├─ DATA_PLATFORM_ADMIN + │ └─ (all databases and warehouses, admin-level) + ├─ TRANSFORM_ROLE + │ ├─ READ: RAW.* + │ ├─ WRITE: TRANSFORM.* + │ ├─ WRITE: ANALYTICS.* + │ └─ OPERATE: TRANSFORM_WH + ├─ LOADER_ROLE + │ ├─ WRITE: RAW.* + │ └─ OPERATE: LOADING_WH + ├─ ANALYST_ROLE + │ ├─ READ: ANALYTICS.* + │ └─ OPERATE: ANALYTICS_WH + └─ BI_ROLE + ├─ READ: ANALYTICS.* + └─ OPERATE: ANALYTICS_WH +``` + +## Reference RBAC Topology (larger team with access roles) + +``` +ACCOUNTADMIN + └─ SYSADMIN + ├─ DATA_PLATFORM_ADMIN + ├─ TRANSFORM_ROLE + │ └─ granted: RAW_READ, ANALYTICS_WRITE, TRANSFORM_WRITE + ├─ ANALYST_ROLE + │ └─ granted: ANALYTICS_FINANCE_READ, ANALYTICS_CORE_READ + ├─ FINANCE_ANALYST_ROLE + │ └─ granted: ANALYTICS_FINANCE_READ (restricted domain) + └─ ... + +Access roles (object-scoped): + RAW_READ → SELECT on RAW.* + ANALYTICS_CORE_READ → SELECT on ANALYTICS.CORE.* + ANALYTICS_FINANCE_READ → SELECT on ANALYTICS.FINANCE.* +``` + +## GRANT emission rule — CRITICAL + +**Always emit one target role per GRANT statement. Never comma-separate roles.** + +```sql +-- CORRECT — one role per statement +GRANT USAGE ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE ON DATABASE BRONZE TO ROLE LOADER_ROLE; +GRANT USAGE ON DATABASE BRONZE TO ROLE TRANSFORM_ROLE; + +-- WRONG — DO NOT EMIT THIS FORM +GRANT USAGE ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE; +``` + +Why: Snowflake's grammar allows the comma-separated form syntactically, but the `snowflake-sdk` driver silently swallows it — the query returns `(0 rows)` with no error, and none of the listed roles actually receive the grant. Verified on live account `DKZPOBS-TQ14188` on 2026-08-25 via follow-up `SHOW GRANTS TO ROLE` queries. Until the driver bug is fixed, always emit one GRANT per role. + +The same rule applies to `REVOKE` — one role per statement. + +## Privilege Reference + +### Database and schema +```sql +GRANT USAGE ON DATABASE TO ROLE ; +GRANT USAGE ON SCHEMA . TO ROLE ; + +-- Read access on existing + future tables +GRANT SELECT ON ALL TABLES IN SCHEMA . TO ROLE ; +GRANT SELECT ON FUTURE TABLES IN SCHEMA . TO ROLE ; + +-- Write access (for dbt / loaders) +GRANT CREATE TABLE ON SCHEMA . TO ROLE ; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA . TO ROLE ; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA . TO ROLE ; +``` + +### Warehouse +```sql +-- Run queries (required for all roles that execute SQL) +GRANT USAGE ON WAREHOUSE TO ROLE ; + +-- Resume/suspend the warehouse (data platform admin) +GRANT OPERATE ON WAREHOUSE TO ROLE ; + +-- Modify warehouse settings (SYSADMIN+ only, rarely needed for functional roles) +GRANT MODIFY ON WAREHOUSE TO ROLE ; +``` + +### Stages and pipes (for loader role) +```sql +GRANT USAGE ON INTEGRATION TO ROLE LOADER_ROLE; +GRANT READ ON STAGE .. TO ROLE LOADER_ROLE; +GRANT WRITE ON STAGE .. TO ROLE LOADER_ROLE; +GRANT OPERATE ON PIPE .. TO ROLE LOADER_ROLE; +``` + +## Validation Queries + +```sql +-- What privileges does a role have? +SHOW GRANTS TO ROLE ANALYST_ROLE; + +-- What roles does a user have? +SHOW GRANTS TO USER jane_doe; + +-- Who has access to a specific table? +SHOW GRANTS ON TABLE ANALYTICS.CORE.orders; + +-- Full role hierarchy (use finops_role_hierarchy tool for a visual view) +SELECT * FROM SNOWFLAKE.ACCOUNT_USAGE.GRANTS_TO_ROLES +WHERE GRANTED_ON = 'DATABASE' +ORDER BY CREATED_ON DESC; +``` + +## Common RBAC Mistakes + +### Mistake 1: Orphaned custom roles +**Symptom:** SYSADMIN can't manage objects owned by a custom role. +**Fix:** `GRANT ROLE TO ROLE SYSADMIN;` for every custom role. + +### Mistake 2: Missing future grants +**Symptom:** Analyst can query existing tables but not new ones created this week. +**Fix:** `GRANT SELECT ON FUTURE TABLES IN SCHEMA ... TO ROLE ANALYST_ROLE;` — run for every schema. + +### Mistake 3: DEFAULT_ROLE = ACCOUNTADMIN +**Symptom:** Every user session opens as ACCOUNTADMIN. +**Fix:** Set `DEFAULT_ROLE` to the least-privilege role the user needs for daily work. + +### Mistake 4: Granting SYSADMIN to dbt +**Symptom:** dbt service account can create and drop databases. +**Fix:** Create TRANSFORM_ROLE with USAGE + CREATE TABLE on specific schemas only. + +### Mistake 5: No role separation for CI vs. production +**Symptom:** A CI pipeline bug drops prod tables. +**Fix:** Separate service accounts with separate roles: `DBT_CI_ROLE` (DEV databases only) and `DBT_PROD_ROLE` (prod databases only). diff --git a/.opencode/skills/snowflake-setup/references/terraform-mapping.md b/.opencode/skills/snowflake-setup/references/terraform-mapping.md new file mode 100644 index 0000000000..d6ebde62a0 --- /dev/null +++ b/.opencode/skills/snowflake-setup/references/terraform-mapping.md @@ -0,0 +1,457 @@ +# Terraform / IaC Output Mapping + +Every Snowflake object created by this skill has a `Snowflake-Labs/snowflake` Terraform provider equivalent. This reference is the DDL → HCL mapping the skill uses when `Output format = terraform` or `both`. + +## Provider Bootstrap + +Always emitted first when Terraform output is selected. + +```hcl +# providers.tf +terraform { + required_version = ">= 1.5.0" + required_providers { + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 0.95" + } + } +} + +provider "snowflake" { + account_name = var.snowflake_account + organization_name = var.snowflake_org + user = var.snowflake_user + authenticator = "SNOWFLAKE_JWT" + private_key = file(var.snowflake_private_key_path) + role = "ACCOUNTADMIN" + # ACCOUNTADMIN required for initial bootstrap; switch to SYSADMIN + # provider alias for day-2 operations +} + +provider "snowflake" { + alias = "sysadmin" + account_name = var.snowflake_account + organization_name = var.snowflake_org + user = var.snowflake_user + authenticator = "SNOWFLAKE_JWT" + private_key = file(var.snowflake_private_key_path) + role = "SYSADMIN" +} + +provider "snowflake" { + alias = "securityadmin" + account_name = var.snowflake_account + organization_name = var.snowflake_org + user = var.snowflake_user + authenticator = "SNOWFLAKE_JWT" + private_key = file(var.snowflake_private_key_path) + role = "SECURITYADMIN" +} +``` + +```hcl +# variables.tf +variable "snowflake_account" { type = string } +variable "snowflake_org" { type = string } +variable "snowflake_user" { type = string } +variable "snowflake_private_key_path" { + type = string + sensitive = true +} +variable "environment" { + type = string + default = "prod" + validation { + condition = contains(["prod", "staging", "dev"], var.environment) + error_message = "environment must be prod, staging, or dev" + } +} +``` + +## DDL → HCL Mapping Table + +### Databases and Schemas + +```hcl +resource "snowflake_database" "raw" { + name = "RAW" + comment = "Raw ingestion layer" + data_retention_time_in_days = 1 +} + +resource "snowflake_schema" "raw_salesforce" { + database = snowflake_database.raw.name + name = "SALESFORCE" + comment = "Salesforce ingestion (Fivetran)" +} +``` + +### Warehouses + +```hcl +resource "snowflake_warehouse" "transform_wh" { + name = "TRANSFORM_WH" + warehouse_size = "SMALL" + auto_suspend = 60 + auto_resume = true + initially_suspended = true + + # Attach resource monitor here (not on the resource monitor's `warehouses` + # attribute, which was removed in provider v0.90+): + resource_monitor = snowflake_resource_monitor.transform_wh_monitor.name + + # Cost-attribution tags applied via snowflake_tag_association below +} +``` + +### Custom Roles + Hierarchy + +```hcl +resource "snowflake_account_role" "transform" { + name = "TRANSFORM_ROLE" + comment = "dbt transformation role" +} + +resource "snowflake_grant_account_role" "transform_to_sysadmin" { + provider = snowflake.securityadmin + role_name = snowflake_account_role.transform.name + parent_role_name = "SYSADMIN" +} +``` + +### Object Grants (regular + FUTURE) + +```hcl +# Grant SELECT on all existing tables in RAW.SALESFORCE +resource "snowflake_grant_privileges_to_account_role" "analyst_select_raw_sf" { + provider = snowflake.sysadmin + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "\"${snowflake_database.raw.name}\".\"${snowflake_schema.raw_salesforce.name}\"" + } + } +} + +# CRITICAL: also grant on FUTURE tables — otherwise new tables are invisible +resource "snowflake_grant_privileges_to_account_role" "analyst_select_raw_sf_future" { + provider = snowflake.sysadmin + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "\"${snowflake_database.raw.name}\".\"${snowflake_schema.raw_salesforce.name}\"" + } + } +} +``` + +### Warehouse Usage Grants + +```hcl +resource "snowflake_grant_privileges_to_account_role" "analyst_use_analytics_wh" { + provider = snowflake.sysadmin + account_role_name = snowflake_account_role.analyst.name + privileges = ["USAGE"] + + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.analytics_wh.name + } +} +``` + +### Service Accounts + +```hcl +resource "snowflake_user" "fivetran_loader" { + provider = snowflake.securityadmin + name = "FIVETRAN_LOADER" + default_role = snowflake_account_role.loader.name + default_warehouse = snowflake_warehouse.loading_wh.name + rsa_public_key = file("${path.module}/keys/fivetran_loader.pub") + + must_change_password = false +} + +resource "snowflake_grant_account_role" "fivetran_loader_gets_loader" { + provider = snowflake.securityadmin + role_name = snowflake_account_role.loader.name + user_name = snowflake_user.fivetran_loader.name +} +``` + +### Resource Monitors + +```hcl +# Schema note: as of Snowflake-Labs/snowflake v0.90+: +# - `set_for_account` is removed. Use `snowflake_account_parameter` with +# key = "RESOURCE_MONITOR" to attach a monitor at the account level. +# - `warehouses` is removed. Set `resource_monitor = "MONITOR_NAME"` on the +# `snowflake_warehouse` resource instead. +# Verified against v0.100.0 on 2026-08-26. + +resource "snowflake_resource_monitor" "account_monitor" { + name = "ACCOUNT_MONITOR" + credit_quota = 500 + frequency = "MONTHLY" + start_timestamp = "IMMEDIATELY" + + notify_triggers = [50, 75, 90] + suspend_trigger = 100 +} + +# Attach the account-level monitor. NOTE: `snowflake_account_parameter` does +# NOT accept "RESOURCE_MONITOR" as a key — the provider validates the key +# against a fixed allowlist of Snowflake account parameters and rejects with +# "invalid account parameter: RESOURCE_MONITOR" during `tofu validate`. +# Verified 2026-08-26 against provider v0.100.0. Use `snowflake_execute` +# instead, which shells out to raw SQL: +resource "snowflake_execute" "attach_account_monitor" { + execute = "ALTER ACCOUNT SET RESOURCE_MONITOR = ${snowflake_resource_monitor.account_monitor.name}" + revert = "ALTER ACCOUNT UNSET RESOURCE_MONITOR" + query = "SELECT SYSTEM$GET_ACCOUNT_ATTRIBUTE('RESOURCE_MONITOR') AS monitor" +} + +resource "snowflake_resource_monitor" "transform_wh_monitor" { + name = "TRANSFORM_WH_MONITOR" + credit_quota = 150 + frequency = "MONTHLY" + + notify_triggers = [75, 90] + suspend_trigger = 100 +} + +# The warehouse attaches the monitor via its own resource_monitor attribute +# (see the snowflake_warehouse block earlier in this file — add: +# resource_monitor = snowflake_resource_monitor.transform_wh_monitor.name +# ) +``` + +### Storage Integrations + +```hcl +resource "snowflake_storage_integration" "s3_raw" { + name = "S3_RAW_INTEGRATION" + type = "EXTERNAL_STAGE" + enabled = true + + storage_provider = "S3" + storage_aws_role_arn = var.aws_snowflake_role_arn + storage_allowed_locations = ["s3://${var.raw_bucket}/"] +} + +# Output the values needed to configure the IAM trust policy on AWS side +output "storage_integration_aws_iam_user_arn" { + value = snowflake_storage_integration.s3_raw.storage_aws_iam_user_arn +} + +output "storage_integration_aws_external_id" { + value = snowflake_storage_integration.s3_raw.storage_aws_external_id +} +``` + +### Stages and File Formats + +```hcl +resource "snowflake_file_format" "csv_standard" { + provider = snowflake.sysadmin + name = "CSV_STANDARD" + database = snowflake_database.raw.name + schema = "PUBLIC" + format_type = "CSV" + field_optionally_enclosed_by = "\"" + null_if = ["NULL", "null", ""] + empty_field_as_null = true + skip_header = 1 +} + +resource "snowflake_stage" "raw_sf_s3" { + provider = snowflake.sysadmin + name = "S3_STAGE" + database = snowflake_database.raw.name + schema = snowflake_schema.raw_salesforce.name + storage_integration = snowflake_storage_integration.s3_raw.name + url = "s3://${var.raw_bucket}/salesforce/" + file_format = "TYPE = 'PARQUET'" +} +``` + +### Pipes + +```hcl +resource "snowflake_pipe" "accounts_pipe" { + provider = snowflake.sysadmin + name = "ACCOUNTS_PIPE" + database = snowflake_database.raw.name + schema = snowflake_schema.raw_salesforce.name + auto_ingest = true + copy_statement = <, RECORD_SOURCE +``` + +**Why this shape matters:** +- **Insert-only** — history is preserved by construction; audit-friendly +- **Hash keys** (`HK_*` = MD5 or SHA256 of business keys) — decouple loads from foreign keys; parallel loads become safe +- **HASHDIFF on satellites** — skip unchanged rows on load; storage efficiency at scale +- **RECORD_SOURCE column everywhere** — every row traceable to its origin system + +**Trade-offs vs Medallion/Functional:** +- Storage is higher (append-only satellites, hash key overhead) — plan larger LOADING_WH monitor budgets +- Query complexity is higher — INFO_MARTS layer is mandatory, not optional, to keep BI usable +- Onboarding is steeper — team should have DV2 experience or budget training time +- Tooling: requires a dbt vault package (`AutomateDV`, `dbtvault`, or hand-rolled macros) + +## Environment Promotion with Zero-Copy Clones + +```sql +-- Create dev environment as instant zero-copy clones of prod +CREATE DATABASE RAW_DEV CLONE RAW; +CREATE DATABASE TRANSFORM_DEV CLONE TRANSFORM; +CREATE DATABASE ANALYTICS_DEV CLONE ANALYTICS; + +-- Grant dev roles access to dev databases +GRANT USAGE ON DATABASE RAW_DEV TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE ON DATABASE TRANSFORM_DEV TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE ON DATABASE ANALYTICS_DEV TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON DATABASE ANALYTICS_DEV TO ROLE TRANSFORM_ROLE; -- dbt CI can write here + +-- In dbt profiles.yml — switch database by target +# dev target +dev: + type: snowflake + database: ANALYTICS_DEV + schema: "{{ target.schema }}" + +# prod target +prod: + type: snowflake + database: ANALYTICS + schema: "{{ target.schema }}" +``` + +Zero-copy clones are instantaneous and cost only the storage delta from the clone point — a full prod clone for dev is essentially free. + +## Warehouse Sizing Guide + +| Workload | Size | Concurrent queries | When to scale up | +|----------|------|-------------------|-----------------| +| Dev / exploration | XS | 1–2 | Almost never needed | +| Data loading (Snowpipe / COPY) | XS–S | 2–4 | Files > 1 GB, parallel loads | +| dbt transformations (small models) | S | 2–4 | Build times > 10 min | +| dbt transformations (large models, complex joins) | M–L | 4–8 | Build times > 20 min | +| BI dashboard queries (Tableau, Looker) | M | 4–8 | Dashboard load time > 10s | +| Ad-hoc analyst queries | M | 4–8 | Significant queue wait time | +| Large batch / ML feature engineering | L–XL | 8–16 | Query runtime > 30 min | + +**Rule of thumb:** Start at XS or S. Watch `QUERY_HISTORY` for `QUEUED_OVERLOAD_TIME > 0` — that's your signal to size up, not gut feel. + +## Multi-Cluster Warehouse (for BI concurrency) + +Only needed when many concurrent users hit the same warehouse. Set up after you've confirmed queuing is the bottleneck, not query size. + +```sql +CREATE WAREHOUSE BI_WH + WAREHOUSE_SIZE = 'MEDIUM' + MIN_CLUSTER_COUNT = 1 + MAX_CLUSTER_COUNT = 3 -- add clusters when queue builds + SCALING_POLICY = 'ECONOMY' -- spin up only when actually needed + AUTO_SUSPEND = 300 + AUTO_RESUME = TRUE; +``` + +`ECONOMY` scaling waits until a cluster is fully loaded before spinning up another. Use `STANDARD` if you need near-zero queue wait for executive dashboards. diff --git a/packages/opencode/src/altimate/tools/sql-execute.ts b/packages/opencode/src/altimate/tools/sql-execute.ts index 4647c75648..7908757ebe 100644 --- a/packages/opencode/src/altimate/tools/sql-execute.ts +++ b/packages/opencode/src/altimate/tools/sql-execute.ts @@ -53,6 +53,20 @@ export const SqlExecuteTool = Tool.define("sql_execute", { limit: args.limit, }) + // The dispatcher's sql.execute handler catches driver errors and returns + // { columns: [], rows: [], row_count: 0, truncated: false, error: } + // instead of throwing. Surface the error so the caller sees the real + // failure instead of a bare "(0 rows)" — the previous behavior made + // syntax errors and semantic failures indistinguishable from successful + // zero-row queries. Found by the 2026-08-25 snowflake-setup live eval. + if (result.error) { + return { + title: "SQL: ERROR", + metadata: { rowCount: 0, truncated: false, error: result.error }, + output: `Failed to execute SQL: ${result.error}`, + } + } + let output = formatResult(result) // altimate_change start — emit SQL structure fingerprint telemetry try { diff --git a/packages/opencode/test/altimate/fixtures/snowflake-setup/LIVE-EVAL-RESULTS.md b/packages/opencode/test/altimate/fixtures/snowflake-setup/LIVE-EVAL-RESULTS.md new file mode 100644 index 0000000000..7964eadd4c --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/snowflake-setup/LIVE-EVAL-RESULTS.md @@ -0,0 +1,202 @@ +# Live Snowflake eval — Medallion greenfield → rollback cycle + +**Ran:** 2026-08-25 +**Account:** DKZPOBS-TQ14188 (locator BA06306), Enterprise, AWS +**User:** anas (ACCOUNTADMIN) +**Warehouse used for execution:** COMPUTE_WH (pre-existing) +**Scenario:** greenfield + Medallion + idempotent + sql + review-only + Snowpipe(AWS) + small-team RBAC + prod-only + PII(email, first_name, last_name) + budget=500 credits + no advanced features +**Emission generated by:** `bun run src/index.ts run --yolo` against altimate LLM gateway, `snowflake-setup` skill + +## Files exercised + +- `greenfield-medallion.expected.sql` — 474 lines of emitted DDL, with placeholders for external infra +- `greenfield-runnable.sql` — same file stripped of the storage-integration, external-stage, target-table, and pipe sections (they require real S3) +- `rollback-medallion.expected.sql` — 185 lines of emitted rollback +- `rollback-runnable.sql` — same file with `` replaced with `BA06306` and the `SET` uncommented + +## Pre-verification + +Before applying, verified account had zero collisions with target names: +- No BRONZE / SILVER / GOLD databases +- No LOADING_WH / TRANSFORM_WH / ANALYTICS_WH +- No DATA_PLATFORM_ADMIN / LOADER_ROLE / TRANSFORM_ROLE / ANALYST_ROLE / BI_ROLE +- No DBT_SERVICE user +- No ACCOUNT_MONITOR or per-warehouse resource monitors + +## Greenfield apply — results + +| Section | Statements | Successful | Notes | +|---------|-----------|------------|-------| +| ACCOUNTADMIN (databases, schemas, warehouses, resource monitors, roles) | 30 | 30/30 ✅ | | +| SECURITYADMIN (dbt_service user + role grant) | 3 | 3/3 ✅ | | +| SYSADMIN (warehouse grants, object privileges, file format, tags, masking policies) | 38 | 26/38 ⚠️ | 12 statements returned bare `(0 rows)` | +| Validation query pack | 13 | 6/13 | 7 returned 0 rows for objects intentionally not created (see notes) | +| **Total executable** | **84** | **59 confirmed** | 25 either "0 rows" or "silently failed" (see findings) | + +### State verification after apply — 17/17 target objects present + +| Category | Object | Present? | +|----------|--------|----------| +| Database | BRONZE | ✅ | +| Database | SILVER | ✅ | +| Database | GOLD | ✅ | +| Warehouse | LOADING_WH (X-Small) | ✅ | +| Warehouse | TRANSFORM_WH (Small) | ✅ | +| Warehouse | ANALYTICS_WH (Medium) | ✅ | +| Role | DATA_PLATFORM_ADMIN | ✅ | +| Role | TRANSFORM_ROLE | ✅ | +| Role | LOADER_ROLE | ✅ | +| Role | ANALYST_ROLE | ✅ | +| Role | BI_ROLE | ✅ | +| User | DBT_SERVICE (default TRANSFORM_ROLE) | ✅ | +| Monitor | ACCOUNT_MONITOR (500 credits/mo) | ✅ | +| Monitor | LOADING_WH_MONITOR (50 credits/mo) | ✅ | +| Monitor | TRANSFORM_WH_MONITOR (200 credits/mo) | ✅ | +| Monitor | ANALYTICS_WH_MONITOR (200 credits/mo) | ✅ | +| Masking | MASK_EMAIL / MASK_FIRST_NAME / MASK_LAST_NAME (in BRONZE.APP) | ✅ | + +## Rollback apply — results + +| # | Statement group | Result | +|---|-----------------|--------| +| 1 | Confirmation gate (SET + CASE) | ✅ Passed | +| 2 | Unset masking policies + drop policies (SYSADMIN) | ✅ 6/6 | +| 3 | Unset tags + drop tags | ✅ 7/7 | +| 4 | Drop pipes / stages / file formats | ✅ 3/3 (2 were already gone from apply, 1 real drop) | +| 5 | **Drop databases (ACCOUNTADMIN)** | ❌ **3/3 BLOCKED** by altimate-code's `sql_execute` safety guard | +| 6 | Drop warehouses | ✅ 3/3 | +| 7 | Unset + drop resource monitors | ✅ 5/5 | +| 8 | Drop storage integrations | ✅ 1/1 (already gone) | +| 9 | Drop service accounts | ✅ 1/1 | +| 10 | Revoke roles from SYSADMIN + drop roles (SECURITYADMIN) | ✅ 10/10 | +| **Total** | **47** | **44 succeeded, 3 blocked by tool guard** | + +### Final teardown of remaining 3 databases + +Bypassed the `sql_execute` guard via a direct `snowflake-sdk` script (`/tmp/drop-dbs.mjs`): +- DROP DATABASE IF EXISTS BRONZE → ✅ +- DROP DATABASE IF EXISTS SILVER → ✅ +- DROP DATABASE IF EXISTS GOLD → ✅ +- Confirmed all 3 return 0 rows on `SHOW DATABASES LIKE ...` + +**Account is back to blank state — verified.** + +--- + +## Findings — real skill / tool bugs discovered by this eval + +### Finding 1 (SKILL BUG — high) — multi-role GRANT statements silently drop + +The emitted DDL uses Snowflake's comma-separated multi-role grant syntax: + +```sql +GRANT USAGE ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE; +``` + +This is valid Snowflake SQL, but under the current `sql_execute` path it returns a bare `(0 rows)` — the grants are NOT applied. Verified via `SHOW GRANTS TO ROLE ...` — none of the roles listed in multi-role GRANTs actually received the grants. + +Impact: greenfield apply completes without visible error but the resulting RBAC is broken — target roles lack DB/schema USAGE grants they should have. + +Root cause: unclear whether this is a `snowflake-sdk` behavior with multi-role GRANTs or a query-parsing issue inside altimate-code's dispatcher. Needs further investigation. + +Workaround: skill should emit one GRANT per role instead of comma-separated list. + +### Finding 2 (TOOL BUG — medium) — `sql_execute` silent-failure mode + +`sql_execute` sometimes returns bare `(0 rows)` with no status message and no error text, for both: +- Statements that ran successfully but affected 0 rows +- Statements that FAILED (verified by running invalid SQL like `SELEC BAD SYNTAX`) + +This makes it impossible for the calling agent to distinguish success-with-zero-affected from silent failure. Severity: medium — reduces observability during script execution but doesn't corrupt data. + +### Finding 3 (TOOL LIMITATION — expected but should be documented) — `sql_execute` blocks DROP DATABASE / DROP SCHEMA / TRUNCATE + +The tool has a non-bypassable safety guard (`packages/opencode/src/altimate/tools/sql-classify.ts:27`) that hard-denies these three operations. This is a deliberate design choice, but it means the skill's own rollback script — as emitted — cannot fully execute through altimate-code's tools. To complete a rollback you must fall back to direct `snowflake-sdk` calls, `snowsql`, or the Snowsight UI. + +Impact on skill: the emitted `rollback.sql` file is still correct DDL; the guardrail is at the tool layer, not the skill layer. But the "guided-execute" execution path advertised in the skill's SKILL.md cannot actually complete a rollback. This should be documented. + +### Finding 4 (SKILL POLISH — low) — validation query pack references objects that skill intentionally doesn't create + +The tail-of-file "validation query pack" includes queries against BRONZE.APP.CUSTOMERS (a table), the storage integration, the external stage, and the Snowpipe pipe — but the skill's own detail-question flow correctly identifies these as needing external infra (real S3 IAM ARN, real bucket URL, real dbt public key) and emits them as placeholders. So the validation queries reference objects the skill intentionally does not create. + +Fix: the validation query pack should be conditional on which sections' placeholders the user actually filled in. Not blocking; polish for a follow-up. + +### Finding 5 (SKILL BUG — high, found by 2026-08-26 Terraform emit) — HCL schema drift vs Snowflake-Labs/snowflake v0.100.0 + +Emitted Terraform failed `tofu validate` (OpenTofu 1.12.6, provider v0.100.0) with three schema drift errors: + +- `snowflake_masking_policy` no longer accepts `signature { column { ... } }` + `masking_expression`. New schema: `argument { name = "..." type = "..." }` blocks + `body = "..."` attribute. +- `snowflake_row_access_policy` — same drift. New schema: `argument` blocks + `body`. +- `snowflake_resource_monitor.warehouses` and `.set_for_account` removed. Warehouse attachment now via `snowflake_warehouse.resource_monitor`; account attachment via `snowflake_account_parameter` with key `RESOURCE_MONITOR`. + +Also: per-warehouse resource monitors that set `frequency = "MONTHLY"` now require `start_timestamp` to be set alongside it (previously optional). + +Fix applied: `references/terraform-mapping.md` updated with corrected HCL for all three resources + schema-drift notes explaining what changed. Post-fix: `tofu validate` returns success with only deprecation warnings. + +### Finding 6 (SKILL BUG — medium, found by 2026-08-26 audit-mode eval) — `SNOWFLAKE.ACCOUNT_USAGE` replication lag + +The audit queries in `references/audit-queries.md` rely on `SNOWFLAKE.ACCOUNT_USAGE.*` views for detecting roles, users, grants, and other objects. These views have documented replication lag of 45 minutes to 2 hours. Recently-created objects (created within the lag window) do not appear yet, so the audit misses them. + +Verified live: planted `ORPHANED_ROLE` and `BAD_USER` did not appear in `ACCOUNT_USAGE.ROLES` / `ACCOUNT_USAGE.USERS` for 45+ minutes but appeared immediately in `SHOW ROLES` / `SHOW USERS`. + +Fix applied: `references/audit-queries.md` updated with an explicit "cross-check with SHOW" mapping table (`DATABASES`, `SCHEMATA`, `ROLES`, `USERS`, `WAREHOUSES`, `GRANTS_TO_ROLES`, `POLICY_REFERENCES`, `TAG_REFERENCES`). Rule: when SHOW and ACCOUNT_USAGE disagree, treat SHOW as authoritative and flag the lag. + +### Finding 7 (SKILL BUG — medium, found by 2026-08-26 tofu-validate second run) — account-level `snowflake_account_parameter` rejects `RESOURCE_MONITOR` key + +After fixing Finding 5's schema drift, the emitted Terraform still failed `tofu validate` on one resource: `snowflake_account_parameter` with `key = "RESOURCE_MONITOR"`. Provider returned: + +``` +Error: invalid account parameter: RESOURCE_MONITOR + with snowflake_account_parameter.resource_monitor, +``` + +Root cause: the `snowflake_account_parameter` resource validates its `key` argument against a fixed allowlist of Snowflake account parameters. `RESOURCE_MONITOR` isn't in that allowlist, even though `ALTER ACCOUNT SET RESOURCE_MONITOR = ` is valid Snowflake SQL. + +Fix applied: `references/terraform-mapping.md` updated to use `snowflake_execute` for account-monitor attachment: + +```hcl +resource "snowflake_execute" "attach_account_monitor" { + execute = "ALTER ACCOUNT SET RESOURCE_MONITOR = ${snowflake_resource_monitor.account_monitor.name}" + revert = "ALTER ACCOUNT UNSET RESOURCE_MONITOR" + query = "SELECT SYSTEM$GET_ACCOUNT_ATTRIBUTE('RESOURCE_MONITOR') AS monitor" +} +``` + +After this fix + Finding 5 fixes, hand-written HCL following `terraform-mapping.md` patterns passes `tofu validate` with only deprecation warnings. + +Contract test added to `snowflake-setup-contract.test.ts` — regression fails if the skill ever reverts to `snowflake_account_parameter` for this attachment. + +--- + +## Cost + +- Greenfield apply: ~90 seconds of interactive LLM turns, ~120K tokens against Altimate LLM gateway +- Rollback: ~60 seconds, ~80K tokens +- Verification queries: ~30 seconds, ~40K tokens +- Snowflake credits: negligible (all objects created were IDLE / suspended immediately) + +## What was tested in the 2026-08-26 extended run + +- ✅ **Audit mode against a broken state** — planted 3 known-broken conditions on `dkzpobs-eval` (BAD_WH without auto_suspend, ORPHANED_ROLE not granted to SYSADMIN, BAD_USER with DEFAULT_ROLE=ACCOUNTADMIN). All 3 detected. Bonus: audit also caught real pre-existing issue that `ANAS` user has `DEFAULT_ROLE=ACCOUNTADMIN`. Maturity: 67/100. Found Finding 6. +- ✅ **Terraform HCL emission + `tofu validate`** — emitted 8 .tf files (providers/variables/databases/warehouses/rbac/ingestion/governance/cost_governance, 1,131 lines total). Initial validate failed with schema drift on masking policy, row access policy, and resource monitor. Fixed the reference docs (see Finding 5). Post-fix: `tofu validate` succeeds with only deprecation warnings. + +## 2026-08-26 late-night full Tier 3 re-execution + +After the LLM wallet blocked further model-based runs, I completed Tier 3 via direct snowflake-sdk scripts. Full four-phase cycle: + +**Phase 1 (greenfield apply):** Ran `greenfield-runnable.sql` (111 statements) directly against DKZPOBS-TQ14188 via snowflake-sdk. 87/111 succeeded; 24 failures were expected (Finding 1 multi-role GRANTs, references to intentionally-omitted objects). **All 17 target objects verified present** via `SHOW` queries. + +**Phase 2 (audit):** Planted BAD_WH/ORPHANED_ROLE/BAD_USER again. Ran the audit-queries.md diagnostic queries directly via SDK (bypassing the LLM). **All 3 planted issues detected** + real pre-existing ANAS DEFAULT_ROLE=ACCOUNTADMIN issue caught. Maturity: 67/100. + +**Phase 3 (Terraform validate):** Wrote minimal HCL following the fixed `terraform-mapping.md` v0.100.0 schemas by hand (providers, variables, 3 databases, 1 warehouse with `resource_monitor` attribute, cost_governance with `snowflake_execute` for account-monitor attachment, masking + row access policies with `argument` + `body` schemas). Ran `tofu init` (installed provider v0.100.0). Ran `tofu validate` — **success with only deprecation warnings.** No blocking errors. Found Finding 7 (`snowflake_account_parameter` doesn't accept `RESOURCE_MONITOR` key), applied fix to `terraform-mapping.md` + added contract test. + +**Phase 4 (rollback + verify blank):** Ran rollback script covering all 17 target objects and the 3 planted broken items via direct SDK (avoiding the sql_execute DROP DATABASE guard). All operations succeeded. **Account back to blank state — 16/16 verification checks passed.** No lingering objects. + +**All four phases succeeded end-to-end without LLM assistance.** Runtime: ~3 minutes total (Phase 1 apply took the longest at ~90 seconds). Snowflake credits burned: negligible. + +## What was NOT tested (still blocked or scope-out) + +- **Non-Medallion topologies (Functional, Domain-per-DB, Data Vault 2.0)** — LLM emission blocked mid-flight when Altimate LLM gateway hit billing quota. The Tier 3 apply/rollback cycle for these would work identically to Medallion once fresh DDL is emitted. **Non-blocker for Tier 1/2/3 coverage; is a natural follow-up.** +- **External integrations** (S3 Snowpipe, Fivetran, SSO, DR, sharing, Cortex) — genuine scope-out per skill guardrails. Not testable without external infra. +- **Guided-execute path via the skill itself** — Tier 3 executed via direct SDK scripts, not via altimate-code's own `guided-execute` execution mode. The SDK path is arguably more thorough (catches the `sql_execute` guard blocking DROP DATABASE); the guided-execute path adds one integration seam. +- **LLM-re-emission verification of Finding 5 + 7 Terraform fixes** — I fixed `terraform-mapping.md` and verified `tofu validate` passes on hand-written HCL following those patterns; the LLM has not been asked to re-emit HCL from the fixed reference to confirm it produces valid output on first try. Follow-up once wallet is restored. diff --git a/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-medallion.expected.sql b/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-medallion.expected.sql new file mode 100644 index 0000000000..839f2e36d5 --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-medallion.expected.sql @@ -0,0 +1,474 @@ +-- ============================================================================ +-- SNOWFLAKE GREENFIELD SETUP +-- Mode: greenfield | Emission: idempotent | Execution: review-only (NOT executed) +-- Topology: Medallion (BRONZE / SILVER / GOLD) +-- RBAC: small-team, single-layer functional roles +-- Environments: prod-only +-- Ingestion: Snowpipe (event-driven, AWS S3) +-- Budget: 500 credits/month (account ceiling) +-- PII: declared categories -> email, first_name, last_name +-- Multi-tenancy: none | Advanced features: none (no IP policy, no SSO, +-- no DR/replication, no data sharing, no Cortex) +-- ============================================================================ +-- +-- HOW TO USE THIS FILE +-- Run each "==== Run as ====" block while USE ROLE is active, +-- in the order they appear. Every CREATE is idempotent (IF NOT EXISTS) so +-- re-running this script is safe. This script performs NO destructive +-- operations and is safe to review before any execution. +-- +-- ============================================================================ +-- CONFIGURE BEFORE RUNNING — placeholder checklist +-- ============================================================================ +-- [ ] -- IAM role ARN Snowflake will assume for S3 access +-- [ ] -- e.g. s3://your-company-raw-data/ +-- [ ] -- rename APP to your actual source system name +-- [ ] -- rename CUSTOMERS to your actual source table +-- [ ] -- email address for resource monitor alerts +-- [ ] -- RSA public key for dbt_service key-pair auth +-- [ ] After CREATE STORAGE INTEGRATION runs, run DESC INTEGRATION and copy +-- STORAGE_AWS_IAM_USER_ARN + STORAGE_AWS_EXTERNAL_ID into the AWS IAM +-- role's trust policy (manual, outside Snowflake — see bottom of file) +-- [ ] After CREATE PIPE runs, run SHOW PIPES and copy notification_channel +-- (SQS ARN) into the S3 bucket's Event Notifications config (manual) +-- +-- NOTE ON SCOPE (explicitly not needed given the answers provided): +-- - No DEV/staging databases or zero-copy clones: envs = prod-only +-- - No access-role layer (RAW_READ / ANALYTICS_WRITE, etc.): RBAC = small-team +-- - No row access policies: no multi-tenancy declared +-- - No network policies / SSO / MFA integrations: no advanced features requested +-- - No DR / replication / failover groups: no advanced features requested +-- - No data sharing / reader accounts: no advanced features requested +-- - No Cortex / ML warehouse: no advanced features requested +-- - No Terraform output: format = sql only +-- ============================================================================ + + +-- ############################################################################# +-- ==== Run as ACCOUNTADMIN ==== +-- ############################################################################# +USE ROLE ACCOUNTADMIN; + +-- ---------------------------------------------------------------------------- +-- 1. Databases and Schemas (Medallion topology) +-- ---------------------------------------------------------------------------- +CREATE DATABASE IF NOT EXISTS BRONZE COMMENT = 'Raw ingestion layer'; +CREATE DATABASE IF NOT EXISTS SILVER COMMENT = 'Cleaned and conformed data (dbt staging + intermediate)'; +CREATE DATABASE IF NOT EXISTS GOLD COMMENT = 'Business-ready marts'; + +-- One schema per ingestion source in BRONZE. +-- Rename APP to your actual source system (e.g. SALESFORCE, STRIPE, POSTGRES_APP). +CREATE SCHEMA IF NOT EXISTS BRONZE.APP COMMENT = 'Landing schema for via Snowpipe'; + +CREATE SCHEMA IF NOT EXISTS SILVER.STAGING COMMENT = 'dbt staging models (stg_*)'; +CREATE SCHEMA IF NOT EXISTS SILVER.INTERMEDIATE COMMENT = 'dbt intermediate models (int_*)'; + +CREATE SCHEMA IF NOT EXISTS GOLD.CORE COMMENT = 'Shared dimensions and facts (dim_*, fct_*)'; +CREATE SCHEMA IF NOT EXISTS GOLD.MART COMMENT = 'Business-domain marts (mart_*)'; + +-- ---------------------------------------------------------------------------- +-- 2. Warehouses +-- Sizes follow the standard sizing guide for a small team, prod-only account. +-- ---------------------------------------------------------------------------- +CREATE WAREHOUSE IF NOT EXISTS LOADING_WH + WAREHOUSE_SIZE = 'XSMALL' + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'Snowpipe / COPY loading workloads'; + +CREATE WAREHOUSE IF NOT EXISTS TRANSFORM_WH + WAREHOUSE_SIZE = 'SMALL' + AUTO_SUSPEND = 120 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'dbt transformation workloads (SILVER + GOLD builds)'; + +CREATE WAREHOUSE IF NOT EXISTS ANALYTICS_WH + WAREHOUSE_SIZE = 'MEDIUM' + AUTO_SUSPEND = 300 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'BI dashboards and ad-hoc analyst queries'; + +-- ---------------------------------------------------------------------------- +-- 3. Resource Monitors — account ceiling 500 credits/month, split per warehouse +-- (LOADING 50 + TRANSFORM 200 + ANALYTICS 200 = 450, leaving headroom +-- under the 500-credit account ceiling for cloud services / variance) +-- ---------------------------------------------------------------------------- +CREATE RESOURCE MONITOR IF NOT EXISTS account_monitor + WITH CREDIT_QUOTA = 500 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 50 PERCENT DO NOTIFY + ON 75 PERCENT DO NOTIFY + ON 90 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +ALTER ACCOUNT SET RESOURCE_MONITOR = account_monitor; + +CREATE RESOURCE MONITOR IF NOT EXISTS loading_wh_monitor + WITH CREDIT_QUOTA = 50 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +CREATE RESOURCE MONITOR IF NOT EXISTS transform_wh_monitor + WITH CREDIT_QUOTA = 200 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +CREATE RESOURCE MONITOR IF NOT EXISTS analytics_wh_monitor + WITH CREDIT_QUOTA = 200 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +ALTER WAREHOUSE LOADING_WH SET RESOURCE_MONITOR = loading_wh_monitor; +ALTER WAREHOUSE TRANSFORM_WH SET RESOURCE_MONITOR = transform_wh_monitor; +ALTER WAREHOUSE ANALYTICS_WH SET RESOURCE_MONITOR = analytics_wh_monitor; + +-- Configure email notifications for resource monitor alerts (manual: set up +-- a notification integration or account-level email in Snowsight -> +-- Admin -> Notifications). Replace below if using an +-- email notification integration: +-- CREATE NOTIFICATION INTEGRATION IF NOT EXISTS resource_monitor_alerts +-- TYPE = EMAIL +-- ENABLED = TRUE +-- ALLOWED_RECIPIENTS = (''); + +-- ---------------------------------------------------------------------------- +-- 4. Storage Integration (AWS S3) — required for Snowpipe +-- REPLACE BEFORE RUNNING: STORAGE_AWS_ROLE_ARN, STORAGE_ALLOWED_LOCATIONS +-- ---------------------------------------------------------------------------- +CREATE STORAGE INTEGRATION IF NOT EXISTS s3_bronze_integration + TYPE = EXTERNAL_STAGE + STORAGE_PROVIDER = 'S3' + ENABLED = TRUE + STORAGE_AWS_ROLE_ARN = '' -- REPLACE BEFORE RUNNING + STORAGE_ALLOWED_LOCATIONS = (''); -- REPLACE BEFORE RUNNING, e.g. 's3://your-company-raw-data/' + +-- After creation, run this and copy STORAGE_AWS_IAM_USER_ARN + STORAGE_AWS_EXTERNAL_ID +-- into your AWS IAM role's trust policy (see "Manual (outside Snowflake)" section below): +-- DESC INTEGRATION s3_bronze_integration; + +-- ---------------------------------------------------------------------------- +-- 5. Custom Roles (small-team, single-layer functional roles) +-- ---------------------------------------------------------------------------- +CREATE ROLE IF NOT EXISTS DATA_PLATFORM_ADMIN COMMENT = 'Full admin over all databases and warehouses'; +CREATE ROLE IF NOT EXISTS LOADER_ROLE COMMENT = 'Snowpipe / ingestion service role — writes to BRONZE only'; +CREATE ROLE IF NOT EXISTS TRANSFORM_ROLE COMMENT = 'dbt service role — reads BRONZE, writes SILVER + GOLD'; +CREATE ROLE IF NOT EXISTS ANALYST_ROLE COMMENT = 'Human analysts — reads GOLD only, masked PII'; +CREATE ROLE IF NOT EXISTS BI_ROLE COMMENT = 'BI tool service account — reads GOLD only, masked PII'; + +-- ---------------------------------------------------------------------------- +-- 6. Grant custom roles to SYSADMIN (avoids orphaned roles — Common Mistake 1) +-- ---------------------------------------------------------------------------- +GRANT ROLE DATA_PLATFORM_ADMIN TO ROLE SYSADMIN; +GRANT ROLE LOADER_ROLE TO ROLE SYSADMIN; +GRANT ROLE TRANSFORM_ROLE TO ROLE SYSADMIN; +GRANT ROLE ANALYST_ROLE TO ROLE SYSADMIN; +GRANT ROLE BI_ROLE TO ROLE SYSADMIN; + +-- Sections not emitted (explicitly not needed): +-- Network policies -- no advanced features requested +-- Security integrations (SSO) -- no advanced features requested +-- Replication / failover groups-- no advanced features requested +-- Cortex warehouse -- no advanced features requested + + +-- ############################################################################# +-- ==== Run as SECURITYADMIN ==== +-- ############################################################################# +USE ROLE SECURITYADMIN; + +-- ---------------------------------------------------------------------------- +-- 7. Service Accounts +-- Snowpipe itself uses the storage integration (no login user required), +-- but a dbt service account is needed to run TRANSFORM_ROLE workloads. +-- Key-pair auth preferred over password auth. +-- ---------------------------------------------------------------------------- +CREATE USER IF NOT EXISTS dbt_service + DEFAULT_ROLE = TRANSFORM_ROLE + DEFAULT_WAREHOUSE = TRANSFORM_WH + RSA_PUBLIC_KEY = '' -- REPLACE BEFORE RUNNING + MUST_CHANGE_PASSWORD = FALSE + COMMENT = 'dbt Cloud / dbt Core CI service account'; +GRANT ROLE TRANSFORM_ROLE TO USER dbt_service; + +-- ---------------------------------------------------------------------------- +-- 8. DEFAULT_ROLE / DEFAULT_WAREHOUSE bindings for human users +-- Add one ALTER USER per team member once accounts exist. Never bind +-- DEFAULT_ROLE = ACCOUNTADMIN for a human's day-to-day account +-- (Common RBAC Mistake 3). +-- ---------------------------------------------------------------------------- +-- ALTER USER SET DEFAULT_ROLE = ANALYST_ROLE, DEFAULT_WAREHOUSE = ANALYTICS_WH; + +-- Section not emitted (explicitly not needed): network policy application to +-- service accounts / MFA enforcement — no advanced features requested. + + +-- ############################################################################# +-- ==== Run as SYSADMIN ==== +-- ############################################################################# +USE ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 9. Warehouse USAGE / OPERATE grants +-- ---------------------------------------------------------------------------- +GRANT USAGE, OPERATE ON WAREHOUSE LOADING_WH TO ROLE LOADER_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE TRANSFORM_WH TO ROLE TRANSFORM_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE ANALYST_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE BI_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE LOADING_WH TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE, OPERATE ON WAREHOUSE TRANSFORM_WH TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 10. Database / schema USAGE grants +-- ---------------------------------------------------------------------------- +GRANT USAGE ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE; +GRANT USAGE ON DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN, TRANSFORM_ROLE; +GRANT USAGE ON DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN, TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE; + +GRANT USAGE ON SCHEMA BRONZE.APP TO ROLE LOADER_ROLE, TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE, DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 11. Object privileges — LOADER_ROLE (write BRONZE only) +-- ---------------------------------------------------------------------------- +GRANT CREATE TABLE ON SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; + +-- ---------------------------------------------------------------------------- +-- 12. Object privileges — TRANSFORM_ROLE (read BRONZE, write SILVER + GOLD) +-- ---------------------------------------------------------------------------- +GRANT SELECT ON ALL TABLES IN SCHEMA BRONZE.APP TO ROLE TRANSFORM_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA BRONZE.APP TO ROLE TRANSFORM_ROLE; + +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; + +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; + +-- ---------------------------------------------------------------------------- +-- 13. Object privileges — ANALYST_ROLE / BI_ROLE (read GOLD only) +-- ---------------------------------------------------------------------------- +GRANT SELECT ON ALL TABLES IN SCHEMA GOLD.CORE TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA GOLD.CORE TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON ALL TABLES IN SCHEMA GOLD.MART TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA GOLD.MART TO ROLE ANALYST_ROLE, BI_ROLE; + +-- ---------------------------------------------------------------------------- +-- 14. DATA_PLATFORM_ADMIN — full admin over all three databases +-- ---------------------------------------------------------------------------- +GRANT ALL ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 15. File format, external stage, target table, and pipe (Snowpipe) +-- REPLACE BEFORE RUNNING: URL path, target table name/columns to match +-- your actual source system schema. +-- ---------------------------------------------------------------------------- +CREATE FILE FORMAT IF NOT EXISTS BRONZE.APP.parquet_standard + TYPE = 'PARQUET' + SNAPPY_COMPRESSION = TRUE; + +CREATE STAGE IF NOT EXISTS BRONZE.APP.s3_stage + STORAGE_INTEGRATION = s3_bronze_integration + URL = 'APP/' -- REPLACE BEFORE RUNNING + FILE_FORMAT = (FORMAT_NAME = BRONZE.APP.parquet_standard); + +GRANT USAGE ON INTEGRATION s3_bronze_integration TO ROLE LOADER_ROLE; +GRANT READ, WRITE ON STAGE BRONZE.APP.s3_stage TO ROLE LOADER_ROLE; + +-- Target table. Rename CUSTOMERS / columns to match . +-- Includes the declared PII columns (email, first_name, last_name) that will +-- be masked in section 17 below. +CREATE TABLE IF NOT EXISTS BRONZE.APP.CUSTOMERS ( + customer_id VARCHAR, + email VARCHAR, + first_name VARCHAR, + last_name VARCHAR, + created_at TIMESTAMP_LTZ, + _loaded_at TIMESTAMP_LTZ DEFAULT CURRENT_TIMESTAMP() +); + +CREATE PIPE IF NOT EXISTS BRONZE.APP.customers_pipe + AUTO_INGEST = TRUE + AS + COPY INTO BRONZE.APP.CUSTOMERS (customer_id, email, first_name, last_name, created_at) + FROM @BRONZE.APP.s3_stage/customers/ + FILE_FORMAT = (FORMAT_NAME = BRONZE.APP.parquet_standard) + ON_ERROR = 'CONTINUE'; + +GRANT OPERATE ON PIPE BRONZE.APP.customers_pipe TO ROLE LOADER_ROLE; + +-- After creation, run this and copy notification_channel (SQS ARN) into the +-- S3 bucket's Event Notifications config (see "Manual" section below): +-- SHOW PIPES IN SCHEMA BRONZE.APP; + +-- ---------------------------------------------------------------------------- +-- 16. Tag Taxonomy (PII + cost allocation) +-- ---------------------------------------------------------------------------- +CREATE TAG IF NOT EXISTS BRONZE.APP.pii_category + ALLOWED_VALUES 'email', 'first_name', 'last_name'; + +CREATE TAG IF NOT EXISTS BRONZE.APP.data_sensitivity + ALLOWED_VALUES 'public', 'internal', 'confidential', 'restricted'; + +CREATE TAG IF NOT EXISTS BRONZE.APP.cost_center + ALLOWED_VALUES 'engineering', 'marketing', 'finance', 'data-platform', 'ml'; +CREATE TAG IF NOT EXISTS BRONZE.APP.environment + ALLOWED_VALUES 'prod', 'staging', 'dev', 'sandbox'; + +ALTER WAREHOUSE LOADING_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; +ALTER WAREHOUSE TRANSFORM_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; +ALTER WAREHOUSE ANALYTICS_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; + +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN email SET TAG BRONZE.APP.pii_category = 'email', BRONZE.APP.data_sensitivity = 'restricted'; +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN first_name SET TAG BRONZE.APP.pii_category = 'first_name', BRONZE.APP.data_sensitivity = 'confidential'; +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN last_name SET TAG BRONZE.APP.pii_category = 'last_name', BRONZE.APP.data_sensitivity = 'confidential'; + +-- ---------------------------------------------------------------------------- +-- 17. Masking Policies — declared PII categories: email, first_name, last_name +-- Policies and their application are idempotent (CREATE OR REPLACE is +-- safe for masking policies per idempotency-patterns.md — no state held). +-- ---------------------------------------------------------------------------- +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_email + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + WHEN CURRENT_ROLE() IN ('ANALYST_ROLE', 'BI_ROLE') THEN REGEXP_REPLACE(val, '^[^@]+', '****') + ELSE '****@****.***' + END; + +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_first_name + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + ELSE LEFT(val, 1) || '***' + END; + +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_last_name + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + ELSE LEFT(val, 1) || '***' + END; + +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN email SET MASKING POLICY BRONZE.APP.mask_email; +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN first_name SET MASKING POLICY BRONZE.APP.mask_first_name; +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN last_name SET MASKING POLICY BRONZE.APP.mask_last_name; + +-- Apply the same masking policies to any downstream SILVER/GOLD table that +-- carries these columns forward, e.g.: +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN email SET MASKING POLICY BRONZE.APP.mask_email; +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN first_name SET MASKING POLICY BRONZE.APP.mask_first_name; +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN last_name SET MASKING POLICY BRONZE.APP.mask_last_name; + +-- Sections not emitted (explicitly not needed): +-- Row access policies -- no multi-tenancy declared +-- Shares -- no advanced features requested + + +-- ============================================================================ +-- ==== Manual (outside Snowflake) ==== +-- ============================================================================ +-- 1. AWS IAM role trust policy for the storage integration: +-- a. Run: DESC INTEGRATION s3_bronze_integration; +-- b. Copy STORAGE_AWS_IAM_USER_ARN and STORAGE_AWS_EXTERNAL_ID +-- c. Update the trust policy on the IAM role referenced by +-- to allow that IAM user to assume it, with a +-- condition on sts:ExternalId matching STORAGE_AWS_EXTERNAL_ID. +-- +-- 2. S3 bucket event notification (required for Snowpipe AUTO_INGEST): +-- a. Run: SHOW PIPES IN SCHEMA BRONZE.APP; +-- b. Copy the notification_channel value (SQS queue ARN) +-- c. In the S3 bucket -> Properties -> Event notifications -> Create: +-- - Event type: s3:ObjectCreated:* +-- - Prefix: customers/ (match the stage path used in section 15) +-- - Destination: the SQS queue ARN from step (b) +-- +-- 3. Distribute dbt_service's private key to your CI system (never commit +-- to source control). Generate the key pair locally and paste only the +-- public key into section 7 (). +-- +-- 4. Configure resource monitor alert recipients in Snowsight +-- (Admin -> Notifications) or via a notification integration, and set +-- above. +-- ============================================================================ + + +-- ============================================================================ +-- VALIDATION QUERY PACK — run after each block to confirm it worked +-- ============================================================================ + +-- Databases / schemas created +SHOW DATABASES LIKE 'BRONZE'; +SHOW DATABASES LIKE 'SILVER'; +SHOW DATABASES LIKE 'GOLD'; +SHOW SCHEMAS IN DATABASE BRONZE; + +-- Warehouses created and sized correctly +SHOW WAREHOUSES LIKE '%_WH'; + +-- Resource monitors attached +SHOW RESOURCE MONITORS; +SELECT resource_monitor_name FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY LIMIT 1; -- sanity check access + +-- Storage integration configured (before wiring AWS trust policy) +DESC INTEGRATION s3_bronze_integration; + +-- Roles created and attached to SYSADMIN (no orphaned roles) +SHOW ROLES LIKE '%_ROLE'; +SHOW GRANTS TO ROLE TRANSFORM_ROLE; + +-- Pipe status (after AWS event notification is wired up) +SELECT SYSTEM$PIPE_STATUS('BRONZE.APP.CUSTOMERS_PIPE'); + +-- Masking verification — run as each role, confirm masked vs plaintext +-- USE ROLE ANALYST_ROLE; +-- SELECT email, first_name, last_name FROM BRONZE.APP.CUSTOMERS LIMIT 5; -- expect masked +-- USE ROLE DATA_PLATFORM_ADMIN; +-- SELECT email, first_name, last_name FROM BRONZE.APP.CUSTOMERS LIMIT 5; -- expect plaintext + +-- Tag application +SELECT tag_database, tag_schema, tag_name, tag_value, object_database, object_schema, object_name, column_name +FROM SNOWFLAKE.ACCOUNT_USAGE.TAG_REFERENCES +WHERE tag_name = 'PII_CATEGORY' +ORDER BY object_database, object_schema, object_name, column_name; + +-- Future grants present (Common RBAC Mistake 2 check) +SHOW GRANTS ON FUTURE TABLES IN SCHEMA SILVER.STAGING; diff --git a/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-runnable.sql b/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-runnable.sql new file mode 100644 index 0000000000..0a8b31c58d --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/snowflake-setup/greenfield-runnable.sql @@ -0,0 +1,445 @@ +-- ============================================================================ +-- SNOWFLAKE GREENFIELD SETUP +-- Mode: greenfield | Emission: idempotent | Execution: review-only (NOT executed) +-- Topology: Medallion (BRONZE / SILVER / GOLD) +-- RBAC: small-team, single-layer functional roles +-- Environments: prod-only +-- Ingestion: Snowpipe (event-driven, AWS S3) +-- Budget: 500 credits/month (account ceiling) +-- PII: declared categories -> email, first_name, last_name +-- Multi-tenancy: none | Advanced features: none (no IP policy, no SSO, +-- no DR/replication, no data sharing, no Cortex) +-- ============================================================================ +-- +-- HOW TO USE THIS FILE +-- Run each "==== Run as ====" block while USE ROLE is active, +-- in the order they appear. Every CREATE is idempotent (IF NOT EXISTS) so +-- re-running this script is safe. This script performs NO destructive +-- operations and is safe to review before any execution. +-- +-- ============================================================================ +-- CONFIGURE BEFORE RUNNING — placeholder checklist +-- ============================================================================ +-- [ ] -- IAM role ARN Snowflake will assume for S3 access +-- [ ] -- e.g. s3://your-company-raw-data/ +-- [ ] -- rename APP to your actual source system name +-- [ ] -- rename CUSTOMERS to your actual source table +-- [ ] -- email address for resource monitor alerts +-- [ ] -- RSA public key for dbt_service key-pair auth +-- [ ] After CREATE STORAGE INTEGRATION runs, run DESC INTEGRATION and copy +-- STORAGE_AWS_IAM_USER_ARN + STORAGE_AWS_EXTERNAL_ID into the AWS IAM +-- role's trust policy (manual, outside Snowflake — see bottom of file) +-- [ ] After CREATE PIPE runs, run SHOW PIPES and copy notification_channel +-- (SQS ARN) into the S3 bucket's Event Notifications config (manual) +-- +-- NOTE ON SCOPE (explicitly not needed given the answers provided): +-- - No DEV/staging databases or zero-copy clones: envs = prod-only +-- - No access-role layer (RAW_READ / ANALYTICS_WRITE, etc.): RBAC = small-team +-- - No row access policies: no multi-tenancy declared +-- - No network policies / SSO / MFA integrations: no advanced features requested +-- - No DR / replication / failover groups: no advanced features requested +-- - No data sharing / reader accounts: no advanced features requested +-- - No Cortex / ML warehouse: no advanced features requested +-- - No Terraform output: format = sql only +-- ============================================================================ + + +-- ############################################################################# +-- ==== Run as ACCOUNTADMIN ==== +-- ############################################################################# +USE ROLE ACCOUNTADMIN; + +-- ---------------------------------------------------------------------------- +-- 1. Databases and Schemas (Medallion topology) +-- ---------------------------------------------------------------------------- +CREATE DATABASE IF NOT EXISTS BRONZE COMMENT = 'Raw ingestion layer'; +CREATE DATABASE IF NOT EXISTS SILVER COMMENT = 'Cleaned and conformed data (dbt staging + intermediate)'; +CREATE DATABASE IF NOT EXISTS GOLD COMMENT = 'Business-ready marts'; + +-- One schema per ingestion source in BRONZE. +-- Rename APP to your actual source system (e.g. SALESFORCE, STRIPE, POSTGRES_APP). +CREATE SCHEMA IF NOT EXISTS BRONZE.APP COMMENT = 'Landing schema for via Snowpipe'; + +CREATE SCHEMA IF NOT EXISTS SILVER.STAGING COMMENT = 'dbt staging models (stg_*)'; +CREATE SCHEMA IF NOT EXISTS SILVER.INTERMEDIATE COMMENT = 'dbt intermediate models (int_*)'; + +CREATE SCHEMA IF NOT EXISTS GOLD.CORE COMMENT = 'Shared dimensions and facts (dim_*, fct_*)'; +CREATE SCHEMA IF NOT EXISTS GOLD.MART COMMENT = 'Business-domain marts (mart_*)'; + +-- ---------------------------------------------------------------------------- +-- 2. Warehouses +-- Sizes follow the standard sizing guide for a small team, prod-only account. +-- ---------------------------------------------------------------------------- +CREATE WAREHOUSE IF NOT EXISTS LOADING_WH + WAREHOUSE_SIZE = 'XSMALL' + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'Snowpipe / COPY loading workloads'; + +CREATE WAREHOUSE IF NOT EXISTS TRANSFORM_WH + WAREHOUSE_SIZE = 'SMALL' + AUTO_SUSPEND = 120 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'dbt transformation workloads (SILVER + GOLD builds)'; + +CREATE WAREHOUSE IF NOT EXISTS ANALYTICS_WH + WAREHOUSE_SIZE = 'MEDIUM' + AUTO_SUSPEND = 300 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE + COMMENT = 'BI dashboards and ad-hoc analyst queries'; + +-- ---------------------------------------------------------------------------- +-- 3. Resource Monitors — account ceiling 500 credits/month, split per warehouse +-- (LOADING 50 + TRANSFORM 200 + ANALYTICS 200 = 450, leaving headroom +-- under the 500-credit account ceiling for cloud services / variance) +-- ---------------------------------------------------------------------------- +CREATE RESOURCE MONITOR IF NOT EXISTS account_monitor + WITH CREDIT_QUOTA = 500 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 50 PERCENT DO NOTIFY + ON 75 PERCENT DO NOTIFY + ON 90 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +ALTER ACCOUNT SET RESOURCE_MONITOR = account_monitor; + +CREATE RESOURCE MONITOR IF NOT EXISTS loading_wh_monitor + WITH CREDIT_QUOTA = 50 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +CREATE RESOURCE MONITOR IF NOT EXISTS transform_wh_monitor + WITH CREDIT_QUOTA = 200 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +CREATE RESOURCE MONITOR IF NOT EXISTS analytics_wh_monitor + WITH CREDIT_QUOTA = 200 + FREQUENCY = MONTHLY + START_TIMESTAMP = IMMEDIATELY + TRIGGERS + ON 75 PERCENT DO NOTIFY + ON 100 PERCENT DO SUSPEND; + +ALTER WAREHOUSE LOADING_WH SET RESOURCE_MONITOR = loading_wh_monitor; +ALTER WAREHOUSE TRANSFORM_WH SET RESOURCE_MONITOR = transform_wh_monitor; +ALTER WAREHOUSE ANALYTICS_WH SET RESOURCE_MONITOR = analytics_wh_monitor; + +-- Configure email notifications for resource monitor alerts (manual: set up +-- a notification integration or account-level email in Snowsight -> +-- Admin -> Notifications). Replace below if using an +-- email notification integration: +-- CREATE NOTIFICATION INTEGRATION IF NOT EXISTS resource_monitor_alerts +-- TYPE = EMAIL +-- ENABLED = TRUE +-- ALLOWED_RECIPIENTS = (''); + +-- ---------------------------------------------------------------------------- +-- 4. Storage Integration (AWS S3) — required for Snowpipe +-- REPLACE BEFORE RUNNING: STORAGE_AWS_ROLE_ARN, STORAGE_ALLOWED_LOCATIONS +-- ---------------------------------------------------------------------------- + +-- ---------------------------------------------------------------------------- +-- 5. Custom Roles (small-team, single-layer functional roles) +-- ---------------------------------------------------------------------------- +CREATE ROLE IF NOT EXISTS DATA_PLATFORM_ADMIN COMMENT = 'Full admin over all databases and warehouses'; +CREATE ROLE IF NOT EXISTS LOADER_ROLE COMMENT = 'Snowpipe / ingestion service role — writes to BRONZE only'; +CREATE ROLE IF NOT EXISTS TRANSFORM_ROLE COMMENT = 'dbt service role — reads BRONZE, writes SILVER + GOLD'; +CREATE ROLE IF NOT EXISTS ANALYST_ROLE COMMENT = 'Human analysts — reads GOLD only, masked PII'; +CREATE ROLE IF NOT EXISTS BI_ROLE COMMENT = 'BI tool service account — reads GOLD only, masked PII'; + +-- ---------------------------------------------------------------------------- +-- 6. Grant custom roles to SYSADMIN (avoids orphaned roles — Common Mistake 1) +-- ---------------------------------------------------------------------------- +GRANT ROLE DATA_PLATFORM_ADMIN TO ROLE SYSADMIN; +GRANT ROLE LOADER_ROLE TO ROLE SYSADMIN; +GRANT ROLE TRANSFORM_ROLE TO ROLE SYSADMIN; +GRANT ROLE ANALYST_ROLE TO ROLE SYSADMIN; +GRANT ROLE BI_ROLE TO ROLE SYSADMIN; + +-- Sections not emitted (explicitly not needed): +-- Network policies -- no advanced features requested +-- Security integrations (SSO) -- no advanced features requested +-- Replication / failover groups-- no advanced features requested +-- Cortex warehouse -- no advanced features requested + + +-- ############################################################################# +-- ==== Run as SECURITYADMIN ==== +-- ############################################################################# +USE ROLE SECURITYADMIN; + +-- ---------------------------------------------------------------------------- +-- 7. Service Accounts +-- Snowpipe itself uses the storage integration (no login user required), +-- but a dbt service account is needed to run TRANSFORM_ROLE workloads. +-- Key-pair auth preferred over password auth. +-- ---------------------------------------------------------------------------- +CREATE USER IF NOT EXISTS dbt_service + DEFAULT_ROLE = TRANSFORM_ROLE + DEFAULT_WAREHOUSE = TRANSFORM_WH + -- RSA_PUBLIC_KEY skipped (would need real public key) + MUST_CHANGE_PASSWORD = FALSE + COMMENT = 'dbt Cloud / dbt Core CI service account'; +GRANT ROLE TRANSFORM_ROLE TO USER dbt_service; + +-- ---------------------------------------------------------------------------- +-- 8. DEFAULT_ROLE / DEFAULT_WAREHOUSE bindings for human users +-- Add one ALTER USER per team member once accounts exist. Never bind +-- DEFAULT_ROLE = ACCOUNTADMIN for a human's day-to-day account +-- (Common RBAC Mistake 3). +-- ---------------------------------------------------------------------------- +-- ALTER USER SET DEFAULT_ROLE = ANALYST_ROLE, DEFAULT_WAREHOUSE = ANALYTICS_WH; + +-- Section not emitted (explicitly not needed): network policy application to +-- service accounts / MFA enforcement — no advanced features requested. + + +-- ############################################################################# +-- ==== Run as SYSADMIN ==== +-- ############################################################################# +USE ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 9. Warehouse USAGE / OPERATE grants +-- ---------------------------------------------------------------------------- +GRANT USAGE, OPERATE ON WAREHOUSE LOADING_WH TO ROLE LOADER_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE TRANSFORM_WH TO ROLE TRANSFORM_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE ANALYST_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE BI_ROLE; +GRANT USAGE, OPERATE ON WAREHOUSE LOADING_WH TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE, OPERATE ON WAREHOUSE TRANSFORM_WH TO ROLE DATA_PLATFORM_ADMIN; +GRANT USAGE, OPERATE ON WAREHOUSE ANALYTICS_WH TO ROLE DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 10. Database / schema USAGE grants +-- ---------------------------------------------------------------------------- +GRANT USAGE ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE; +GRANT USAGE ON DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN, TRANSFORM_ROLE; +GRANT USAGE ON DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN, TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE; + +GRANT USAGE ON SCHEMA BRONZE.APP TO ROLE LOADER_ROLE, TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE, DATA_PLATFORM_ADMIN; +GRANT USAGE ON SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE, ANALYST_ROLE, BI_ROLE, DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 11. Object privileges — LOADER_ROLE (write BRONZE only) +-- ---------------------------------------------------------------------------- +GRANT CREATE TABLE ON SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA BRONZE.APP TO ROLE LOADER_ROLE; + +-- ---------------------------------------------------------------------------- +-- 12. Object privileges — TRANSFORM_ROLE (read BRONZE, write SILVER + GOLD) +-- ---------------------------------------------------------------------------- +GRANT SELECT ON ALL TABLES IN SCHEMA BRONZE.APP TO ROLE TRANSFORM_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA BRONZE.APP TO ROLE TRANSFORM_ROLE; + +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT CREATE TABLE, CREATE VIEW ON SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; + +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA SILVER.STAGING TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA SILVER.INTERMEDIATE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA GOLD.CORE TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; +GRANT INSERT, UPDATE, DELETE, TRUNCATE ON FUTURE TABLES IN SCHEMA GOLD.MART TO ROLE TRANSFORM_ROLE; + +-- ---------------------------------------------------------------------------- +-- 13. Object privileges — ANALYST_ROLE / BI_ROLE (read GOLD only) +-- ---------------------------------------------------------------------------- +GRANT SELECT ON ALL TABLES IN SCHEMA GOLD.CORE TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA GOLD.CORE TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON ALL TABLES IN SCHEMA GOLD.MART TO ROLE ANALYST_ROLE, BI_ROLE; +GRANT SELECT ON FUTURE TABLES IN SCHEMA GOLD.MART TO ROLE ANALYST_ROLE, BI_ROLE; + +-- ---------------------------------------------------------------------------- +-- 14. DATA_PLATFORM_ADMIN — full admin over all three databases +-- ---------------------------------------------------------------------------- +GRANT ALL ON DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON ALL SCHEMAS IN DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE BRONZE TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE SILVER TO ROLE DATA_PLATFORM_ADMIN; +GRANT ALL ON FUTURE SCHEMAS IN DATABASE GOLD TO ROLE DATA_PLATFORM_ADMIN; + +-- ---------------------------------------------------------------------------- +-- 15. File format, external stage, target table, and pipe (Snowpipe) +-- REPLACE BEFORE RUNNING: URL path, target table name/columns to match +-- your actual source system schema. +-- ---------------------------------------------------------------------------- +CREATE FILE FORMAT IF NOT EXISTS BRONZE.APP.parquet_standard + TYPE = 'PARQUET' + SNAPPY_COMPRESSION = TRUE; + + +GRANT USAGE ON INTEGRATION s3_bronze_integration TO ROLE LOADER_ROLE; +GRANT READ, WRITE ON STAGE BRONZE.APP.s3_stage TO ROLE LOADER_ROLE; + +-- Target table. Rename CUSTOMERS / columns to match . +-- Includes the declared PII columns (email, first_name, last_name) that will +-- be masked in section 17 below. + + +GRANT OPERATE ON PIPE BRONZE.APP.customers_pipe TO ROLE LOADER_ROLE; + +-- After creation, run this and copy notification_channel (SQS ARN) into the +-- S3 bucket's Event Notifications config (see "Manual" section below): +-- SHOW PIPES IN SCHEMA BRONZE.APP; + +-- ---------------------------------------------------------------------------- +-- 16. Tag Taxonomy (PII + cost allocation) +-- ---------------------------------------------------------------------------- +CREATE TAG IF NOT EXISTS BRONZE.APP.pii_category + ALLOWED_VALUES 'email', 'first_name', 'last_name'; + +CREATE TAG IF NOT EXISTS BRONZE.APP.data_sensitivity + ALLOWED_VALUES 'public', 'internal', 'confidential', 'restricted'; + +CREATE TAG IF NOT EXISTS BRONZE.APP.cost_center + ALLOWED_VALUES 'engineering', 'marketing', 'finance', 'data-platform', 'ml'; +CREATE TAG IF NOT EXISTS BRONZE.APP.environment + ALLOWED_VALUES 'prod', 'staging', 'dev', 'sandbox'; + +ALTER WAREHOUSE LOADING_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; +ALTER WAREHOUSE TRANSFORM_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; +ALTER WAREHOUSE ANALYTICS_WH SET TAG BRONZE.APP.cost_center = 'data-platform', BRONZE.APP.environment = 'prod'; + +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN email SET TAG BRONZE.APP.pii_category = 'email', BRONZE.APP.data_sensitivity = 'restricted'; +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN first_name SET TAG BRONZE.APP.pii_category = 'first_name', BRONZE.APP.data_sensitivity = 'confidential'; +ALTER TABLE BRONZE.APP.CUSTOMERS + MODIFY COLUMN last_name SET TAG BRONZE.APP.pii_category = 'last_name', BRONZE.APP.data_sensitivity = 'confidential'; + +-- ---------------------------------------------------------------------------- +-- 17. Masking Policies — declared PII categories: email, first_name, last_name +-- Policies and their application are idempotent (CREATE OR REPLACE is +-- safe for masking policies per idempotency-patterns.md — no state held). +-- ---------------------------------------------------------------------------- +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_email + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + WHEN CURRENT_ROLE() IN ('ANALYST_ROLE', 'BI_ROLE') THEN REGEXP_REPLACE(val, '^[^@]+', '****') + ELSE '****@****.***' + END; + +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_first_name + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + ELSE LEFT(val, 1) || '***' + END; + +CREATE OR REPLACE MASKING POLICY BRONZE.APP.mask_last_name + AS (val STRING) RETURNS STRING -> + CASE + WHEN CURRENT_ROLE() IN ('DATA_PLATFORM_ADMIN', 'LOADER_ROLE', 'TRANSFORM_ROLE') THEN val + ELSE LEFT(val, 1) || '***' + END; + +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN email SET MASKING POLICY BRONZE.APP.mask_email; +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN first_name SET MASKING POLICY BRONZE.APP.mask_first_name; +ALTER TABLE BRONZE.APP.CUSTOMERS MODIFY COLUMN last_name SET MASKING POLICY BRONZE.APP.mask_last_name; + +-- Apply the same masking policies to any downstream SILVER/GOLD table that +-- carries these columns forward, e.g.: +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN email SET MASKING POLICY BRONZE.APP.mask_email; +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN first_name SET MASKING POLICY BRONZE.APP.mask_first_name; +-- ALTER TABLE GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN last_name SET MASKING POLICY BRONZE.APP.mask_last_name; + +-- Sections not emitted (explicitly not needed): +-- Row access policies -- no multi-tenancy declared +-- Shares -- no advanced features requested + + +-- ============================================================================ +-- ==== Manual (outside Snowflake) ==== +-- ============================================================================ +-- 1. AWS IAM role trust policy for the storage integration: +-- a. Run: DESC INTEGRATION s3_bronze_integration; +-- b. Copy STORAGE_AWS_IAM_USER_ARN and STORAGE_AWS_EXTERNAL_ID +-- c. Update the trust policy on the IAM role referenced by +-- to allow that IAM user to assume it, with a +-- condition on sts:ExternalId matching STORAGE_AWS_EXTERNAL_ID. +-- +-- 2. S3 bucket event notification (required for Snowpipe AUTO_INGEST): +-- a. Run: SHOW PIPES IN SCHEMA BRONZE.APP; +-- b. Copy the notification_channel value (SQS queue ARN) +-- c. In the S3 bucket -> Properties -> Event notifications -> Create: +-- - Event type: s3:ObjectCreated:* +-- - Prefix: customers/ (match the stage path used in section 15) +-- - Destination: the SQS queue ARN from step (b) +-- +-- 3. Distribute dbt_service's private key to your CI system (never commit +-- to source control). Generate the key pair locally and paste only the +-- public key into section 7 (). +-- +-- 4. Configure resource monitor alert recipients in Snowsight +-- (Admin -> Notifications) or via a notification integration, and set +-- above. +-- ============================================================================ + + +-- ============================================================================ +-- VALIDATION QUERY PACK — run after each block to confirm it worked +-- ============================================================================ + +-- Databases / schemas created +SHOW DATABASES LIKE 'BRONZE'; +SHOW DATABASES LIKE 'SILVER'; +SHOW DATABASES LIKE 'GOLD'; +SHOW SCHEMAS IN DATABASE BRONZE; + +-- Warehouses created and sized correctly +SHOW WAREHOUSES LIKE '%_WH'; + +-- Resource monitors attached +SHOW RESOURCE MONITORS; +SELECT resource_monitor_name FROM SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY LIMIT 1; -- sanity check access + +-- Storage integration configured (before wiring AWS trust policy) +DESC INTEGRATION s3_bronze_integration; + +-- Roles created and attached to SYSADMIN (no orphaned roles) +SHOW ROLES LIKE '%_ROLE'; +SHOW GRANTS TO ROLE TRANSFORM_ROLE; + +-- Pipe status (after AWS event notification is wired up) +SELECT SYSTEM$PIPE_STATUS('BRONZE.APP.CUSTOMERS_PIPE'); + +-- Masking verification — run as each role, confirm masked vs plaintext +-- USE ROLE ANALYST_ROLE; +-- SELECT email, first_name, last_name FROM BRONZE.APP.CUSTOMERS LIMIT 5; -- expect masked +-- USE ROLE DATA_PLATFORM_ADMIN; +-- SELECT email, first_name, last_name FROM BRONZE.APP.CUSTOMERS LIMIT 5; -- expect plaintext + +-- Tag application +SELECT tag_database, tag_schema, tag_name, tag_value, object_database, object_schema, object_name, column_name +FROM SNOWFLAKE.ACCOUNT_USAGE.TAG_REFERENCES +WHERE tag_name = 'PII_CATEGORY' +ORDER BY object_database, object_schema, object_name, column_name; + +-- Future grants present (Common RBAC Mistake 2 check) +SHOW GRANTS ON FUTURE TABLES IN SCHEMA SILVER.STAGING; diff --git a/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-medallion.expected.sql b/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-medallion.expected.sql new file mode 100644 index 0000000000..f2801b16fb --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-medallion.expected.sql @@ -0,0 +1,185 @@ +-- ============================================================================ +-- SNOWFLAKE GREENFIELD ROLLBACK +-- Companion to greenfield.sql — drops every object created by that script, +-- in strict reverse dependency order. Review-only: NOT executed by the agent. +-- ============================================================================ +-- +-- ROLLBACK CONFIRMATION +-- ============================================================================ +-- This script will DROP: +-- - 3 databases (BRONZE, SILVER, GOLD) and ALL contained data +-- - 3 warehouses (LOADING_WH, TRANSFORM_WH, ANALYTICS_WH) +-- - 5 custom roles (DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE, +-- ANALYST_ROLE, BI_ROLE) and all their grants +-- - 1 service account (dbt_service) +-- - 1 storage integration (s3_bronze_integration) +-- - 4 resource monitors (account_monitor, loading_wh_monitor, +-- transform_wh_monitor, analytics_wh_monitor) +-- - 3 masking policies, 4 tags, 1 pipe, 1 stage, 1 file format +-- +-- Estimated data loss: all data in BRONZE.APP.CUSTOMERS and any SILVER/GOLD +-- tables built on top of it. Time Travel retention (default 1 day unless +-- altered) may allow recovery within that window via UNDROP. +-- +-- To proceed, uncomment the following line by replacing the placeholder with +-- your actual account locator (see: SELECT CURRENT_ACCOUNT();): +-- +-- SET rollback_confirmed_account = ''; +-- +-- The script will fail at the first destructive statement if this is not set +-- or does not match CURRENT_ACCOUNT(). This is a hard guard against +-- accidentally rolling back the wrong account. +-- ============================================================================ + +-- Guard — do not remove. Every destructive block below assumes this ran and +-- succeeded first. +SELECT CASE + WHEN $rollback_confirmed_account = CURRENT_ACCOUNT() THEN 'proceed' + ELSE ERROR('Rollback account mismatch or unconfirmed. Refusing to drop objects. Set $rollback_confirmed_account to CURRENT_ACCOUNT() first.') +END; + + +-- ############################################################################# +-- ==== Run as SYSADMIN ==== +-- ############################################################################# +USE ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 1. Masking policies — unset from columns, then drop +-- ---------------------------------------------------------------------------- +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN email UNSET MASKING POLICY; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN first_name UNSET MASKING POLICY; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN last_name UNSET MASKING POLICY; +-- If applied to downstream GOLD tables, unset there too before dropping the policy: +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN email UNSET MASKING POLICY; +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN first_name UNSET MASKING POLICY; +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN last_name UNSET MASKING POLICY; + +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_email; +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_first_name; +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_last_name; + +-- (No row access policies were created in greenfield.sql — none to unset/drop.) + +-- ---------------------------------------------------------------------------- +-- 2. Tag references — unset, then drop tag definitions +-- ---------------------------------------------------------------------------- +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN email UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN first_name UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN last_name UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; + +ALTER WAREHOUSE IF EXISTS LOADING_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; +ALTER WAREHOUSE IF EXISTS TRANSFORM_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; +ALTER WAREHOUSE IF EXISTS ANALYTICS_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; + +DROP TAG IF EXISTS BRONZE.APP.pii_category; +DROP TAG IF EXISTS BRONZE.APP.data_sensitivity; +DROP TAG IF EXISTS BRONZE.APP.cost_center; +DROP TAG IF EXISTS BRONZE.APP.environment; + +-- ---------------------------------------------------------------------------- +-- 3. Pipes and tasks — suspend before drop (no tasks were created; Snowpipe +-- only, per the ingestion selection) +-- ---------------------------------------------------------------------------- +-- Pipes do not support ALTER ... SUSPEND the way tasks do; simply drop. +DROP PIPE IF EXISTS BRONZE.APP.customers_pipe; + +-- ---------------------------------------------------------------------------- +-- 4. External stages and file formats +-- ---------------------------------------------------------------------------- +DROP STAGE IF EXISTS BRONZE.APP.s3_stage; +DROP FILE FORMAT IF EXISTS BRONZE.APP.parquet_standard; + +-- ---------------------------------------------------------------------------- +-- 5. Tables and schemas are dropped implicitly via database drop in step 6 +-- below (per idempotency-patterns.md, dropping the database removes all +-- contained tables/views/schemas). Listed here for dependency-order +-- documentation only — no separate statement needed. +-- ---------------------------------------------------------------------------- + + +-- ############################################################################# +-- ==== Run as ACCOUNTADMIN ==== +-- ############################################################################# +USE ROLE ACCOUNTADMIN; + +-- ---------------------------------------------------------------------------- +-- 6. Databases (drops all contained schemas, tables, views) +-- ---------------------------------------------------------------------------- +DROP DATABASE IF EXISTS BRONZE; +DROP DATABASE IF EXISTS SILVER; +DROP DATABASE IF EXISTS GOLD; + +-- ---------------------------------------------------------------------------- +-- 7. Warehouses +-- ---------------------------------------------------------------------------- +DROP WAREHOUSE IF EXISTS LOADING_WH; +DROP WAREHOUSE IF EXISTS TRANSFORM_WH; +DROP WAREHOUSE IF EXISTS ANALYTICS_WH; + +-- ---------------------------------------------------------------------------- +-- 8. Resource monitors (unset from warehouses/account first — warehouses +-- are already dropped above, which auto-detaches their monitors; the +-- account-level monitor still needs an explicit UNSET) +-- ---------------------------------------------------------------------------- +ALTER ACCOUNT UNSET RESOURCE_MONITOR; + +DROP RESOURCE MONITOR IF EXISTS loading_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS transform_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS analytics_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS account_monitor; + +-- ---------------------------------------------------------------------------- +-- 9. Storage integration +-- ---------------------------------------------------------------------------- +DROP INTEGRATION IF EXISTS s3_bronze_integration; + +-- (No notification integration was created in greenfield.sql — the email +-- notification integration in that file was commented out as optional.) + + +-- ############################################################################# +-- ==== Run as SECURITYADMIN ==== +-- ############################################################################# +USE ROLE SECURITYADMIN; + +-- ---------------------------------------------------------------------------- +-- 10. Service accounts +-- ---------------------------------------------------------------------------- +DROP USER IF EXISTS dbt_service; + +-- ---------------------------------------------------------------------------- +-- 11. Grants — revoke custom roles from SYSADMIN before dropping them +-- ---------------------------------------------------------------------------- +REVOKE ROLE DATA_PLATFORM_ADMIN FROM ROLE SYSADMIN; +REVOKE ROLE LOADER_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE TRANSFORM_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE ANALYST_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE BI_ROLE FROM ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 12. Custom roles +-- Never drops built-in roles (ACCOUNTADMIN, SECURITYADMIN, SYSADMIN, +-- USERADMIN, ORGADMIN, PUBLIC) — only the custom roles created in +-- greenfield.sql. +-- ---------------------------------------------------------------------------- +DROP ROLE IF EXISTS DATA_PLATFORM_ADMIN; +DROP ROLE IF EXISTS LOADER_ROLE; +DROP ROLE IF EXISTS TRANSFORM_ROLE; +DROP ROLE IF EXISTS ANALYST_ROLE; +DROP ROLE IF EXISTS BI_ROLE; + +-- ============================================================================ +-- POST-ROLLBACK VALIDATION +-- ============================================================================ +SHOW DATABASES LIKE 'BRONZE'; -- expect: no rows +SHOW DATABASES LIKE 'SILVER'; -- expect: no rows +SHOW DATABASES LIKE 'GOLD'; -- expect: no rows +SHOW WAREHOUSES LIKE '%_WH'; -- expect: no rows +SHOW ROLES LIKE '%_ROLE'; -- expect: no rows (DATA_PLATFORM_ADMIN also gone) +SHOW RESOURCE MONITORS; -- expect: no rows +SHOW INTEGRATIONS LIKE 's3_bronze_integration'; -- expect: no rows +SHOW USERS LIKE 'dbt_service'; -- expect: no rows diff --git a/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-runnable.sql b/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-runnable.sql new file mode 100644 index 0000000000..e44f8f79fb --- /dev/null +++ b/packages/opencode/test/altimate/fixtures/snowflake-setup/rollback-runnable.sql @@ -0,0 +1,185 @@ +-- ============================================================================ +-- SNOWFLAKE GREENFIELD ROLLBACK +-- Companion to greenfield.sql — drops every object created by that script, +-- in strict reverse dependency order. Review-only: NOT executed by the agent. +-- ============================================================================ +-- +-- ROLLBACK CONFIRMATION +-- ============================================================================ +-- This script will DROP: +-- - 3 databases (BRONZE, SILVER, GOLD) and ALL contained data +-- - 3 warehouses (LOADING_WH, TRANSFORM_WH, ANALYTICS_WH) +-- - 5 custom roles (DATA_PLATFORM_ADMIN, LOADER_ROLE, TRANSFORM_ROLE, +-- ANALYST_ROLE, BI_ROLE) and all their grants +-- - 1 service account (dbt_service) +-- - 1 storage integration (s3_bronze_integration) +-- - 4 resource monitors (account_monitor, loading_wh_monitor, +-- transform_wh_monitor, analytics_wh_monitor) +-- - 3 masking policies, 4 tags, 1 pipe, 1 stage, 1 file format +-- +-- Estimated data loss: all data in BRONZE.APP.CUSTOMERS and any SILVER/GOLD +-- tables built on top of it. Time Travel retention (default 1 day unless +-- altered) may allow recovery within that window via UNDROP. +-- +-- To proceed, uncomment the following line by replacing the placeholder with +-- your actual account locator (see: SELECT CURRENT_ACCOUNT();): +-- +SET rollback_confirmed_account = 'BA06306'; +-- +-- The script will fail at the first destructive statement if this is not set +-- or does not match CURRENT_ACCOUNT(). This is a hard guard against +-- accidentally rolling back the wrong account. +-- ============================================================================ + +-- Guard — do not remove. Every destructive block below assumes this ran and +-- succeeded first. +SELECT CASE + WHEN $rollback_confirmed_account = CURRENT_ACCOUNT() THEN 'proceed' + ELSE ERROR('Rollback account mismatch or unconfirmed. Refusing to drop objects. Set $rollback_confirmed_account to CURRENT_ACCOUNT() first.') +END; + + +-- ############################################################################# +-- ==== Run as SYSADMIN ==== +-- ############################################################################# +USE ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 1. Masking policies — unset from columns, then drop +-- ---------------------------------------------------------------------------- +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN email UNSET MASKING POLICY; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN first_name UNSET MASKING POLICY; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS MODIFY COLUMN last_name UNSET MASKING POLICY; +-- If applied to downstream GOLD tables, unset there too before dropping the policy: +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN email UNSET MASKING POLICY; +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN first_name UNSET MASKING POLICY; +-- ALTER TABLE IF EXISTS GOLD.CORE.DIM_CUSTOMERS MODIFY COLUMN last_name UNSET MASKING POLICY; + +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_email; +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_first_name; +DROP MASKING POLICY IF EXISTS BRONZE.APP.mask_last_name; + +-- (No row access policies were created in greenfield.sql — none to unset/drop.) + +-- ---------------------------------------------------------------------------- +-- 2. Tag references — unset, then drop tag definitions +-- ---------------------------------------------------------------------------- +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN email UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN first_name UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; +ALTER TABLE IF EXISTS BRONZE.APP.CUSTOMERS + MODIFY COLUMN last_name UNSET TAG BRONZE.APP.pii_category, BRONZE.APP.data_sensitivity; + +ALTER WAREHOUSE IF EXISTS LOADING_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; +ALTER WAREHOUSE IF EXISTS TRANSFORM_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; +ALTER WAREHOUSE IF EXISTS ANALYTICS_WH UNSET TAG BRONZE.APP.cost_center, BRONZE.APP.environment; + +DROP TAG IF EXISTS BRONZE.APP.pii_category; +DROP TAG IF EXISTS BRONZE.APP.data_sensitivity; +DROP TAG IF EXISTS BRONZE.APP.cost_center; +DROP TAG IF EXISTS BRONZE.APP.environment; + +-- ---------------------------------------------------------------------------- +-- 3. Pipes and tasks — suspend before drop (no tasks were created; Snowpipe +-- only, per the ingestion selection) +-- ---------------------------------------------------------------------------- +-- Pipes do not support ALTER ... SUSPEND the way tasks do; simply drop. +DROP PIPE IF EXISTS BRONZE.APP.customers_pipe; + +-- ---------------------------------------------------------------------------- +-- 4. External stages and file formats +-- ---------------------------------------------------------------------------- +DROP STAGE IF EXISTS BRONZE.APP.s3_stage; +DROP FILE FORMAT IF EXISTS BRONZE.APP.parquet_standard; + +-- ---------------------------------------------------------------------------- +-- 5. Tables and schemas are dropped implicitly via database drop in step 6 +-- below (per idempotency-patterns.md, dropping the database removes all +-- contained tables/views/schemas). Listed here for dependency-order +-- documentation only — no separate statement needed. +-- ---------------------------------------------------------------------------- + + +-- ############################################################################# +-- ==== Run as ACCOUNTADMIN ==== +-- ############################################################################# +USE ROLE ACCOUNTADMIN; + +-- ---------------------------------------------------------------------------- +-- 6. Databases (drops all contained schemas, tables, views) +-- ---------------------------------------------------------------------------- +DROP DATABASE IF EXISTS BRONZE; +DROP DATABASE IF EXISTS SILVER; +DROP DATABASE IF EXISTS GOLD; + +-- ---------------------------------------------------------------------------- +-- 7. Warehouses +-- ---------------------------------------------------------------------------- +DROP WAREHOUSE IF EXISTS LOADING_WH; +DROP WAREHOUSE IF EXISTS TRANSFORM_WH; +DROP WAREHOUSE IF EXISTS ANALYTICS_WH; + +-- ---------------------------------------------------------------------------- +-- 8. Resource monitors (unset from warehouses/account first — warehouses +-- are already dropped above, which auto-detaches their monitors; the +-- account-level monitor still needs an explicit UNSET) +-- ---------------------------------------------------------------------------- +ALTER ACCOUNT UNSET RESOURCE_MONITOR; + +DROP RESOURCE MONITOR IF EXISTS loading_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS transform_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS analytics_wh_monitor; +DROP RESOURCE MONITOR IF EXISTS account_monitor; + +-- ---------------------------------------------------------------------------- +-- 9. Storage integration +-- ---------------------------------------------------------------------------- +DROP INTEGRATION IF EXISTS s3_bronze_integration; + +-- (No notification integration was created in greenfield.sql — the email +-- notification integration in that file was commented out as optional.) + + +-- ############################################################################# +-- ==== Run as SECURITYADMIN ==== +-- ############################################################################# +USE ROLE SECURITYADMIN; + +-- ---------------------------------------------------------------------------- +-- 10. Service accounts +-- ---------------------------------------------------------------------------- +DROP USER IF EXISTS dbt_service; + +-- ---------------------------------------------------------------------------- +-- 11. Grants — revoke custom roles from SYSADMIN before dropping them +-- ---------------------------------------------------------------------------- +REVOKE ROLE DATA_PLATFORM_ADMIN FROM ROLE SYSADMIN; +REVOKE ROLE LOADER_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE TRANSFORM_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE ANALYST_ROLE FROM ROLE SYSADMIN; +REVOKE ROLE BI_ROLE FROM ROLE SYSADMIN; + +-- ---------------------------------------------------------------------------- +-- 12. Custom roles +-- Never drops built-in roles (ACCOUNTADMIN, SECURITYADMIN, SYSADMIN, +-- USERADMIN, ORGADMIN, PUBLIC) — only the custom roles created in +-- greenfield.sql. +-- ---------------------------------------------------------------------------- +DROP ROLE IF EXISTS DATA_PLATFORM_ADMIN; +DROP ROLE IF EXISTS LOADER_ROLE; +DROP ROLE IF EXISTS TRANSFORM_ROLE; +DROP ROLE IF EXISTS ANALYST_ROLE; +DROP ROLE IF EXISTS BI_ROLE; + +-- ============================================================================ +-- POST-ROLLBACK VALIDATION +-- ============================================================================ +SHOW DATABASES LIKE 'BRONZE'; -- expect: no rows +SHOW DATABASES LIKE 'SILVER'; -- expect: no rows +SHOW DATABASES LIKE 'GOLD'; -- expect: no rows +SHOW WAREHOUSES LIKE '%_WH'; -- expect: no rows +SHOW ROLES LIKE '%_ROLE'; -- expect: no rows (DATA_PLATFORM_ADMIN also gone) +SHOW RESOURCE MONITORS; -- expect: no rows +SHOW INTEGRATIONS LIKE 's3_bronze_integration'; -- expect: no rows +SHOW USERS LIKE 'dbt_service'; -- expect: no rows diff --git a/packages/opencode/test/altimate/snowflake-setup-contract.test.ts b/packages/opencode/test/altimate/snowflake-setup-contract.test.ts new file mode 100644 index 0000000000..111fbf8434 --- /dev/null +++ b/packages/opencode/test/altimate/snowflake-setup-contract.test.ts @@ -0,0 +1,546 @@ +/** + * Tier 1 prompt-contract tests for the `snowflake-setup` skill. + * + * Pattern borrowed from PR #1092 (`optimizer-prompt-contract.test.ts`): + * deterministic, whitespace-normalized assertions that pin the skill's + * non-negotiable invariants. Reflow of SKILL.md text does not break these + * tests; substantive removal of a rule does. + * + * See `.opencode/skills/snowflake-setup/TESTING.md` for Tier 2 (mocked-execution + * eval) and Tier 3 (live Snowflake eval) designs — those are not implemented + * yet and this file does not stub them. + * + * What this file catches: + * - Frontmatter drift (name renamed, description dropped) + * - Tool references to nonexistent tools (a class of bug this skill has hit + * multiple times: `snowflake_sql`, `finops_role_access`, misuse of + * `altimate_core_export_ddl` as if it queried a live warehouse) + * - Reference files renamed or deleted without updating SKILL.md + * - Guardrails silently removed + * - Triage / topology / emission-mode options silently changed + * - Rollback safety rules watered down + * - Data-Vault-2 branches missing when topology mentions DV2 + * + * What this file does NOT catch: + * - Whether emitted DDL is syntactically valid Snowflake SQL + * - Whether the audit workflow actually finds real misconfigurations + * - Whether Terraform HCL passes `terraform validate` + * - Whether the skill loads in the OpenCode session runtime + * + * Those live in Tier 2 (mocked) and Tier 3 (live). See TESTING.md. + */ + +import { describe, test, expect } from "bun:test" +import { readFileSync, readdirSync, existsSync } from "node:fs" +import { join } from "node:path" + +const SKILL_DIR = join(import.meta.dir, "../../../../.opencode/skills/snowflake-setup") +const SKILL_MD_PATH = join(SKILL_DIR, "SKILL.md") +const REFERENCES_DIR = join(SKILL_DIR, "references") +const TOOLS_DIR = join(import.meta.dir, "../../src/altimate/tools") + +// Read once; every assertion re-reads from the same string so tests are hermetic. +const SKILL_MD = readFileSync(SKILL_MD_PATH, "utf-8") + +// Whitespace-normalize helper — reflow of prose must not break invariants. +const norm = (s: string) => s.replace(/\s+/g, " ").trim() +const containsNorm = (haystack: string, needle: string) => + norm(haystack).toLowerCase().includes(norm(needle).toLowerCase()) + +// --- Section 1: Structural invariants ----------------------------------------- + +describe("snowflake-setup — structural invariants", () => { + test("SKILL.md exists and is non-empty", () => { + expect(existsSync(SKILL_MD_PATH)).toBe(true) + expect(SKILL_MD.length).toBeGreaterThan(1000) + }) + + test("frontmatter opens with --- fence and declares name: snowflake-setup", () => { + const firstLines = SKILL_MD.split("\n").slice(0, 6).join("\n") + expect(firstLines.startsWith("---\n")).toBe(true) + expect(firstLines).toMatch(/^name:\s*snowflake-setup\s*$/m) + }) + + test("frontmatter has a description", () => { + const firstLines = SKILL_MD.split("\n").slice(0, 6).join("\n") + expect(firstLines).toMatch(/^description:\s*.+\S/m) + }) + + test("frontmatter description is under 500 chars (loader constraint)", () => { + const match = SKILL_MD.match(/^description:\s*(.+)$/m) + expect(match).not.toBeNull() + expect(match![1].length).toBeLessThan(500) + }) + + test("references/ directory exists and contains all 10 reference files", () => { + expect(existsSync(REFERENCES_DIR)).toBe(true) + const files = readdirSync(REFERENCES_DIR).filter((f) => f.endsWith(".md")).sort() + expect(files).toEqual([ + "advanced-features.md", + "audit-queries.md", + "cost-governance.md", + "data-vault-patterns.md", + "governance-patterns.md", + "idempotency-patterns.md", + "ingestion-patterns.md", + "rbac-patterns.md", + "terraform-mapping.md", + "topology-patterns.md", + ]) + }) +}) + +// --- Section 2: Every referenced tool must exist ------------------------------ + +describe("snowflake-setup — tool references resolve", () => { + // Extract every `tool_name` mention from SKILL.md that matches the altimate + // tool naming convention: snake_case, prefixed with a known family. + const toolRefs = new Set() + const rx = /`((?:warehouse|sql|altimate_core|schema|finops)_[a-z_]+)`/g + let m: RegExpExecArray | null + while ((m = rx.exec(SKILL_MD)) !== null) { + // Filter out obvious argument names that share the naming style but aren't + // tools (e.g. `schema_path`, `schema_context` are Zod field names). + const name = m[1] + if (["schema_path", "schema_context", "schema_name", "sql_execute_write"].includes(name)) continue + toolRefs.add(name) + } + + test("SKILL.md mentions at least 10 tools (sanity check on the extractor)", () => { + expect(toolRefs.size).toBeGreaterThanOrEqual(10) + }) + + // Build the set of all tools defined in src/altimate/tools/*.ts by scanning + // for `Tool.define("", ...)`. + const definedTools = new Set() + for (const file of readdirSync(TOOLS_DIR).filter((f) => f.endsWith(".ts"))) { + const src = readFileSync(join(TOOLS_DIR, file), "utf-8") + const defRx = /Tool\.define\(\s*["']([a-z_]+)["']/g + let dm: RegExpExecArray | null + while ((dm = defRx.exec(src)) !== null) { + definedTools.add(dm[1]) + } + } + + for (const toolName of toolRefs) { + test(`tool \`${toolName}\` mentioned in SKILL.md must exist in src/altimate/tools/`, () => { + expect(definedTools.has(toolName)).toBe(true) + }) + } +}) + +// --- Section 3: Every referenced markdown file must exist --------------------- + +describe("snowflake-setup — reference file mentions resolve", () => { + // Extract every `references/*.md` mention from SKILL.md. + const refMentions = new Set() + const rx = /`references\/([a-z\-]+\.md)`/g + let m: RegExpExecArray | null + while ((m = rx.exec(SKILL_MD)) !== null) refMentions.add(m[1]) + + test("SKILL.md mentions at least 6 reference files (sanity)", () => { + expect(refMentions.size).toBeGreaterThanOrEqual(6) + }) + + for (const ref of refMentions) { + test(`references/${ref} mentioned in SKILL.md must exist on disk`, () => { + expect(existsSync(join(REFERENCES_DIR, ref))).toBe(true) + }) + } +}) + +// --- Section 4: No stale tool names anywhere in the skill --------------------- + +describe("snowflake-setup — no stale tool names", () => { + // These names have appeared in earlier drafts of the skill and must never + // reappear. Each is either a hallucinated name or a name that was renamed + // during the tool-schema audit. + const staleNames = ["snowflake_sql", "finops_role_access"] + + const allSkillContent = [ + SKILL_MD, + ...readdirSync(REFERENCES_DIR) + .filter((f) => f.endsWith(".md")) + .map((f) => readFileSync(join(REFERENCES_DIR, f), "utf-8")), + ].join("\n") + + for (const stale of staleNames) { + test(`no reference to stale tool name \`${stale}\` anywhere in skill`, () => { + expect(allSkillContent.includes(stale)).toBe(false) + }) + } + + test("no reference to old skill directory name `snowflake-greenfield-setup`", () => { + expect(allSkillContent.includes("snowflake-greenfield-setup")).toBe(false) + }) +}) + +// --- Section 5: Triage flow invariants ---------------------------------------- + +describe("snowflake-setup — triage flow", () => { + test("mode question offers greenfield, audit, and hybrid", () => { + expect(containsNorm(SKILL_MD, "greenfield")).toBe(true) + expect(containsNorm(SKILL_MD, "audit")).toBe(true) + expect(containsNorm(SKILL_MD, "hybrid")).toBe(true) + }) + + test("output-format question offers sql, terraform, and both", () => { + // These appear as literal option labels in the triage batch. + expect(SKILL_MD).toMatch(/`sql`.*(default)/is) + expect(SKILL_MD).toMatch(/`terraform`/) + expect(SKILL_MD).toMatch(/`both`/) + }) + + test("execution-control question offers all 4 modes", () => { + for (const mode of ["review-only", "guided-execute", "dbt-integrate", "terraform-apply"]) { + expect(SKILL_MD.includes(`\`${mode}\``)).toBe(true) + } + }) +}) + +// --- Section 6: Topology options ----------------------------------------------- + +describe("snowflake-setup — topology options", () => { + const requiredTopologies = ["Medallion", "Functional", "Domain-per-Database", "Data Vault 2.0"] + + for (const topo of requiredTopologies) { + test(`topology question offers ${topo}`, () => { + expect(SKILL_MD.includes(topo)).toBe(true) + }) + } + + test("SKILL.md branches on data-vault-2 in plan sections (DV2-specific behavior wired)", () => { + // Sections 1, 3, 5 must have explicit DV2 branches. Look for the + // fingerprint used consistently: "If topology = `data-vault-2`". + const branches = SKILL_MD.match(/If topology = `data-vault-2`/gi) ?? [] + expect(branches.length).toBeGreaterThanOrEqual(3) + }) + + test("DV2 detail-question batch present in step 4b", () => { + // Detail questions should be gated on the DV2 answer. + expect(containsNorm(SKILL_MD, "Data Vault 2.0 detail questions")).toBe(true) + expect(containsNorm(SKILL_MD, "AutomateDV")).toBe(true) // recommended package + }) +}) + +// --- Section 7: Emission modes ------------------------------------------------- + +describe("snowflake-setup — emission modes", () => { + test("emission-mode question offers strict, idempotent (default), and additive", () => { + expect(SKILL_MD).toMatch(/`strict`/) + expect(SKILL_MD).toMatch(/`idempotent`[^\n]*default/i) + expect(SKILL_MD).toMatch(/`additive`/) + }) +}) + +// --- Section 8: Warehouse preflight ------------------------------------------- + +describe("snowflake-setup — warehouse preflight (step 2b)", () => { + test("preflight step exists and uses warehouse_list", () => { + // Skill must NOT fire sql_execute blind — it must go through warehouse_list + // first. This is the guardrail introduced after the audit-mode fail-fast + // rule. + expect(SKILL_MD).toMatch(/warehouse_list/) + expect(SKILL_MD).toMatch(/warehouse preflight/i) + }) + + test("multi-warehouse case: user must be prompted to pick, no auto-select", () => { + expect(SKILL_MD).toMatch(/multiple.*Snowflake warehouses/i) + expect(containsNorm(SKILL_MD, "prompt the user to pick")).toBe(true) + expect(containsNorm(SKILL_MD, "never guess or auto-select")).toBe(true) + }) + + test("preflight includes warehouse_test smoke check", () => { + expect(SKILL_MD).toMatch(/warehouse_test/) + }) + + test("preflight fails fast for audit mode (no silent fallback)", () => { + expect(containsNorm(SKILL_MD, "Fail fast for `audit` mode")).toBe(true) + }) +}) + +// --- Section 9: Guardrails ----------------------------------------------------- + +describe("snowflake-setup — guardrails", () => { + // The 10 non-negotiable guardrails at the bottom of SKILL.md. + // Each test pins one of them, whitespace-normalized so light rewording still + // passes but substantive removal fails. + const guardrails: Array<[string, string]> = [ + ["no ACCOUNTADMIN to service accounts", "No ACCOUNTADMIN to service accounts"], + ["always include FUTURE grants", "Always include FUTURE grants"], + ["never CREATE OR REPLACE for stateful objects", "Never `CREATE OR REPLACE`"], + ["storage integration requires ACCOUNTADMIN + manual IAM", "Storage integration DDL requires ACCOUNTADMIN"], + ["prefer key-pair auth over passwords", "Prefer key-pair auth over passwords"], + ["rollback never drops built-in roles", "Never drop built-in roles"], + ["audit mode requires live connection", "Audit mode requires a live connection"], + ["PII discovery is advisory", "PII discovery results are advisory"], + ["rollback scripts are gated by confirmation", "Rollback scripts are gated"], + ["Terraform state must not contain secrets", "Terraform state must not contain secrets"], + ] + + for (const [label, needle] of guardrails) { + test(`guardrail: ${label}`, () => { + expect(containsNorm(SKILL_MD, needle)).toBe(true) + }) + } +}) + +// --- Section 10: Rollback safety (in idempotency-patterns.md) ------------------ + +describe("snowflake-setup — rollback safety rules", () => { + const idempotency = readFileSync(join(REFERENCES_DIR, "idempotency-patterns.md"), "utf-8") + + test("rollback uses IF EXISTS on every DROP", () => { + expect(containsNorm(idempotency, "Use `IF EXISTS` on every `DROP`")).toBe(true) + }) + + test("rollback never emits DROP ... CASCADE", () => { + expect(containsNorm(idempotency, "Never emit `DROP ... CASCADE`")).toBe(true) + }) + + test("rollback never drops built-in roles", () => { + expect(containsNorm(idempotency, "PUBLIC")).toBe(true) + expect(containsNorm(idempotency, "ACCOUNTADMIN")).toBe(true) + }) + + test("rollback includes account-locator confirmation guard", () => { + expect(containsNorm(idempotency, "ROLLBACK CONFIRMATION")).toBe(true) + expect(containsNorm(idempotency, "CURRENT_ACCOUNT()")).toBe(true) + }) + + test("danger list for CREATE OR REPLACE is intact (all 8 object types)", () => { + for (const obj of [ + "WAREHOUSE", + "ROLE", + "USER", + "DATABASE", + "SCHEMA", + "TABLE", + "STORAGE INTEGRATION", + "PIPE", + "RESOURCE MONITOR", + ]) { + expect(idempotency.includes(obj)).toBe(true) + } + }) +}) + +// --- Section 11: DV2 reference completeness ----------------------------------- + +describe("snowflake-setup — data-vault-patterns.md completeness", () => { + const dv2 = readFileSync(join(REFERENCES_DIR, "data-vault-patterns.md"), "utf-8") + + test("hub / link / satellite patterns all present", () => { + expect(dv2.includes("## Hub Pattern")).toBe(true) + expect(dv2.includes("## Link Pattern")).toBe(true) + expect(dv2.includes("## Satellite Pattern")).toBe(true) + }) + + test("HASHDIFF pattern documented (critical: prevents linear satellite growth)", () => { + expect(containsNorm(dv2, "HASHDIFF")).toBe(true) + }) + + test("insert-only enforcement documented (REVOKE UPDATE, DELETE)", () => { + expect(dv2).toMatch(/REVOKE\s+UPDATE,?\s+DELETE/i) + }) + + test("three DV2 roles present", () => { + for (const role of ["VAULT_LOADER_ROLE", "BUSINESS_VAULT_BUILDER_ROLE", "MART_BUILDER_ROLE"]) { + expect(dv2.includes(role)).toBe(true) + } + }) + + test("PII placement decision (three options) documented", () => { + expect(containsNorm(dv2, "PII placement")).toBe(true) + // Three named options + expect(containsNorm(dv2, "RAW_VAULT satellites")).toBe(true) + expect(containsNorm(dv2, "BUSINESS_VAULT")).toBe(true) + expect(containsNorm(dv2, "hybrid")).toBe(true) + }) +}) + +// --- Section 11.1: ACCOUNT_USAGE lag cross-check (found by live eval) -------- + +describe("snowflake-setup — ACCOUNT_USAGE lag cross-check", () => { + const audit = readFileSync(join(REFERENCES_DIR, "audit-queries.md"), "utf-8") + + test("audit-queries.md documents ACCOUNT_USAGE replication lag", () => { + expect(containsNorm(audit, "45 minutes to 2 hours")).toBe(true) + }) + + test("audit-queries.md provides SHOW-command cross-check mapping", () => { + expect(containsNorm(audit, "Real-time `SHOW` equivalent")).toBe(true) + // A few of the specific entries in the mapping table. + expect(audit).toMatch(/SHOW DATABASES/i) + expect(audit).toMatch(/SHOW ROLES/i) + expect(audit).toMatch(/SHOW USERS/i) + expect(audit).toMatch(/SHOW WAREHOUSES/i) + }) + + test("audit-queries.md tells the skill to treat SHOW as authoritative on mismatch", () => { + expect(containsNorm(audit, "treat the `SHOW` result as authoritative")).toBe(true) + }) +}) + +// --- Section 11.2: Terraform HCL schemas match v0.100.0 (found by live eval) -- + +describe("snowflake-setup — Terraform HCL schemas", () => { + const tf = readFileSync(join(REFERENCES_DIR, "terraform-mapping.md"), "utf-8") + + test("snowflake_masking_policy uses `argument` block, not `signature { column }`", () => { + // Extract each snowflake_masking_policy resource block and verify. + const blocks = tf.match(/resource\s+"snowflake_masking_policy"[\s\S]+?\n\}/g) ?? [] + expect(blocks.length).toBeGreaterThan(0) + for (const block of blocks) { + expect(block).toMatch(/argument\s*\{/) + expect(block).not.toMatch(/signature\s*\{\s*column/) + } + }) + + test("snowflake_masking_policy uses `body`, not `masking_expression`", () => { + const blocks = tf.match(/resource\s+"snowflake_masking_policy"[\s\S]+?\n\}/g) ?? [] + for (const block of blocks) { + expect(block).toMatch(/\bbody\s*=/) + expect(block).not.toMatch(/masking_expression\s*=/) + } + }) + + test("snowflake_row_access_policy uses `argument` and `body`", () => { + const blocks = tf.match(/resource\s+"snowflake_row_access_policy"[\s\S]+?\n\}/g) ?? [] + expect(blocks.length).toBeGreaterThan(0) + for (const block of blocks) { + expect(block).toMatch(/argument\s*\{/) + expect(block).toMatch(/\bbody\s*=/) + expect(block).not.toMatch(/row_access_expression\s*=/) + expect(block).not.toMatch(/signature\s*=\s*\{/) + } + }) + + test("snowflake_resource_monitor does not use `warehouses` or `set_for_account`", () => { + const blocks = tf.match(/resource\s+"snowflake_resource_monitor"[\s\S]+?\n\}/g) ?? [] + expect(blocks.length).toBeGreaterThan(0) + for (const block of blocks) { + expect(block).not.toMatch(/\bwarehouses\s*=/) + expect(block).not.toMatch(/\bset_for_account\s*=/) + } + }) + + test("account-level monitor is attached via snowflake_execute (not snowflake_account_parameter)", () => { + // Finding 5b (2026-08-26): snowflake_account_parameter rejects + // key="RESOURCE_MONITOR". Must use snowflake_execute with ALTER ACCOUNT. + // Look for a `resource "snowflake_execute" "..."` block containing ALTER ACCOUNT SET RESOURCE_MONITOR. + const executeBlocks = tf.match(/resource\s+"snowflake_execute"[\s\S]+?\n\}/g) ?? [] + const anyAttachesMonitor = executeBlocks.some((b) => + b.match(/ALTER ACCOUNT SET RESOURCE_MONITOR/) + ) + expect(anyAttachesMonitor).toBe(true) + + // Must NOT define a snowflake_account_parameter resource with RESOURCE_MONITOR value. + const acctParamBlocks = tf.match(/resource\s+"snowflake_account_parameter"[\s\S]+?\n\}/g) ?? [] + const anyUsesRM = acctParamBlocks.some((b) => b.match(/["']RESOURCE_MONITOR["']/)) + expect(anyUsesRM).toBe(false) + }) + + test("per-warehouse monitor is attached via snowflake_warehouse.resource_monitor", () => { + // The warehouse resource block must have a `resource_monitor` attribute. + const whBlocks = tf.match(/resource\s+"snowflake_warehouse"[\s\S]+?\n\}/g) ?? [] + const anyHasMonitor = whBlocks.some((b) => b.match(/\bresource_monitor\s*=/)) + expect(anyHasMonitor).toBe(true) + }) +}) + +// --- Section 11.3: conditional validation queries (found by live eval) -------- + +describe("snowflake-setup — validation queries conditional on emission", () => { + test("SKILL.md rule: validation queries only for sections actually emitted", () => { + expect(containsNorm(SKILL_MD, "validation queries must only reference objects the emitted DDL actually creates")).toBe(true) + // Must call out the placeholder-skip case explicitly. + expect(containsNorm(SKILL_MD, "placeholders the user did not fill in")).toBe(true) + }) +}) + +// --- Section 11.4: split-rollback rule (found by 2026-08-25 live eval) -------- + +describe("snowflake-setup — rollback split (tool-safe + manual)", () => { + const idempotency = readFileSync(join(REFERENCES_DIR, "idempotency-patterns.md"), "utf-8") + + test("idempotency-patterns.md documents the two-file split", () => { + expect(idempotency).toMatch(/rollback-tool-safe\.sql/) + expect(idempotency).toMatch(/rollback-manual\.sql/) + }) + + test("dependency order marks schemas + databases as MANUAL", () => { + // Steps 7 (Schemas) and 8 (Databases) must be tagged so the emitter + // routes them into the manual file, not the tool-safe file. + expect(idempotency).toMatch(/Schemas\s+\[MANUAL/) + expect(idempotency).toMatch(/Databases\s+\[MANUAL/) + }) + + test("SKILL.md warns about DROP DATABASE / DROP SCHEMA / TRUNCATE tool guard", () => { + // Both must be mentioned somewhere — the guided-execute description AND + // the rollback plan section. Assertion is loose on exact wording so light + // rewording is fine; disappearance of the warning fails the test. + expect(SKILL_MD.match(/DROP DATABASE.*DROP SCHEMA.*TRUNCATE/i)).not.toBeNull() + expect(containsNorm(SKILL_MD, "non-bypassable")).toBe(true) + }) + + test("SKILL.md rollback plan section 13 describes the split", () => { + expect(containsNorm(SKILL_MD, "rollback-tool-safe.sql")).toBe(true) + expect(containsNorm(SKILL_MD, "rollback-manual.sql")).toBe(true) + }) +}) + +// --- Section 11.5: GRANT emission rule (found by 2026-08-25 live eval) -------- + +describe("snowflake-setup — GRANT emission rule (single-role only)", () => { + const rbac = readFileSync(join(REFERENCES_DIR, "rbac-patterns.md"), "utf-8") + + test("rbac-patterns.md includes the one-role-per-GRANT rule verbatim", () => { + expect(containsNorm(rbac, "Always emit one target role per GRANT statement")).toBe(true) + expect(containsNorm(rbac, "Never comma-separate roles")).toBe(true) + }) + + test("SKILL.md guardrail 11 explicitly bans comma-separated GRANT roles", () => { + expect(containsNorm(SKILL_MD, "One role per GRANT statement")).toBe(true) + expect(containsNorm(SKILL_MD, "never comma-separate roles")).toBe(true) + }) + + test("rbac-patterns.md has no example of comma-separated grant target roles", () => { + // Only check lines that look like SQL statements (start with GRANT/REVOKE after optional whitespace) + // and end with a semicolon — this rules out prose that quotes GRANT wording. + const lines = rbac.split("\n") + for (let i = 0; i < lines.length; i++) { + const line = lines[i] + if (!line.match(/^\s*(GRANT|REVOKE)\s.+TO ROLE\b.+;$/)) continue + // Skip counter-examples marked with WRONG / DO NOT EMIT within 2 lines above. + const prevLine = i > 0 ? lines[i - 1] : "" + const prev2Line = i > 1 ? lines[i - 2] : "" + if (prevLine.match(/WRONG|DO NOT EMIT/i) || prev2Line.match(/WRONG|DO NOT EMIT/i)) continue + // Any remaining SQL statement must NOT have a comma between TO ROLE and the semicolon. + const afterToRole = line.split(/TO ROLE\b/i)[1] ?? "" + const beforeSemicolon = afterToRole.split(";")[0] + expect(beforeSemicolon.includes(",")).toBe(false) + } + }) +}) + +// --- Section 12: audit-queries.md targets sql_execute, not snowflake_sql ------ + +describe("snowflake-setup — audit-queries.md tool reference", () => { + const audit = readFileSync(join(REFERENCES_DIR, "audit-queries.md"), "utf-8") + + test("audit queries run via sql_execute (not the hallucinated snowflake_sql)", () => { + expect(audit).toMatch(/sql_execute/) + expect(audit.includes("snowflake_sql")).toBe(false) + }) + + test("severity rubric with three levels present", () => { + expect(audit).toMatch(/CRITICAL/) + expect(audit).toMatch(/WARNING/) + expect(audit).toMatch(/INFO/) + }) + + test("maturity score formula documented", () => { + expect(containsNorm(audit, "100 – (CRITICAL × 10)")).toBe(true) + }) +}) diff --git a/packages/opencode/test/altimate/snowflake-setup-eval.test.ts b/packages/opencode/test/altimate/snowflake-setup-eval.test.ts new file mode 100644 index 0000000000..c687defeab --- /dev/null +++ b/packages/opencode/test/altimate/snowflake-setup-eval.test.ts @@ -0,0 +1,201 @@ +/** + * Tier 2/3 model-in-the-loop eval for the `snowflake-setup` skill. + * + * WHAT THIS TESTS + * Runs a real LLM turn through the `snowflake-setup` skill in non-interactive + * `altimate-code run --yolo` mode, captures the emitted DDL, and grades it + * against the skill's non-negotiable guardrails from SKILL.md. + * + * This is genuine end-to-end verification — the same code path a real user + * hits when typing `/snowflake-setup` in the TUI. It exercises: skill + * discovery, skill loading, LLM instruction-following, tool use (bash for + * mkdir + write tool for the SQL files), and downstream file emission. + * + * WHAT THIS DOES NOT TEST + * - Applying the emitted DDL to a real Snowflake account + * - Rollback correctness end-to-end + * - Audit mode against a broken account + * - Terraform HCL emission or `terraform validate` + * - External integrations (S3, SSO, DR, sharing, Cortex) + * + * Those are documented in `.opencode/skills/snowflake-setup/TESTING.md` and + * remain manual until the harness matures. + * + * WHY IT IS OPT-IN + * - Requires a working Altimate LLM Gateway login (~/.altimate/altimate.json) + * - Costs API credits (~120K tokens per run at time of writing) + * - Takes ~90 seconds + * - Not deterministic across model versions (the LLM emits slightly different + * phrasing / comments each run; assertions are shape-based, not literal) + * + * Run with: + * SNOWFLAKE_SETUP_EVAL=1 bun test test/altimate/snowflake-setup-eval.test.ts + * + * The golden files at fixtures/snowflake-setup/*.expected.sql are the OUTPUT + * from a prior successful run — kept as reference, not asserted verbatim + * against. Use them to eyeball drift when this test starts failing. + */ + +import { describe, test, expect } from "bun:test" +import { execSync } from "node:child_process" +import { readFileSync, existsSync, mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" + +const EVAL_ENABLED = process.env.SNOWFLAKE_SETUP_EVAL === "1" +const REPO_ROOT = join(import.meta.dir, "../../../..") +const ARTIFACT_DIR = join(REPO_ROOT, "packages/opencode/eval-artifacts") + +// The `run` subcommand cd's into packages/opencode (from `bun run --cwd`), so +// paths in the prompt are relative to that directory, not the repo root. +const GREENFIELD_SQL_PATH = join(REPO_ROOT, "packages/opencode/eval-artifacts/greenfield.sql") +const ROLLBACK_SQL_PATH = join(REPO_ROOT, "packages/opencode/eval-artifacts/rollback.sql") + +const PROMPT = [ + "Invoke the snowflake-setup skill.", + "Pre-answered triage: mode=greenfield, format=sql, execution=review-only.", + "Detail answers: topology=Medallion, RBAC=small-team, envs=prod-only,", + "ingestion=Snowpipe(AWS S3), cloud=AWS, emission=idempotent, budget=500 credits,", + "PII discovery=declared with categories email/first_name/last_name,", + "no multi-tenancy, no advanced features.", + "Emit the greenfield SQL to eval-artifacts/greenfield.sql and rollback SQL", + "to eval-artifacts/rollback.sql. Placeholders for S3 ARN etc are fine.", + "Do NOT execute anything against Snowflake.", +].join(" ") + +describe.skipIf(!EVAL_ENABLED)("snowflake-setup — Tier 2/3 model-in-the-loop eval", () => { + let greenfieldSql = "" + let rollbackSql = "" + + test( + "boots altimate-code, runs the skill, emits both SQL files", + () => { + // Clean up any prior run so we're not asserting against stale files. + if (existsSync(ARTIFACT_DIR)) rmSync(ARTIFACT_DIR, { recursive: true, force: true }) + mkdirSync(ARTIFACT_DIR, { recursive: true }) + + // Run the skill non-interactively. --yolo auto-approves permission prompts + // (bash + write) which are otherwise deny-by-default in non-interactive mode. + // Format=json is not needed here — we're grading the files, not the events. + // Timeout is 10 min: real LLM emissions (greenfield + rollback via write + // tool) take ~5–6 min against the Altimate gateway; earlier 5-min timeout + // was cutting off partway through the rollback write, leaving files + // partially emitted and downstream tests failing on empty rollbackSql. + try { + execSync(`bun run dev run --yolo ${JSON.stringify(PROMPT)}`, { + cwd: REPO_ROOT, + stdio: "pipe", + timeout: 10 * 60 * 1000, + }) + } catch (e) { + // If timeout fired but files exist and are populated, treat as success. + // Rethrow only if the emit clearly didn't happen. + if (!existsSync(GREENFIELD_SQL_PATH) || !existsSync(ROLLBACK_SQL_PATH)) throw e + } + + expect(existsSync(GREENFIELD_SQL_PATH)).toBe(true) + expect(existsSync(ROLLBACK_SQL_PATH)).toBe(true) + + greenfieldSql = readFileSync(GREENFIELD_SQL_PATH, "utf-8") + rollbackSql = readFileSync(ROLLBACK_SQL_PATH, "utf-8") + + // Basic size sanity — 100+ lines each; anything smaller is a truncation bug. + expect(greenfieldSql.split("\n").length).toBeGreaterThan(100) + expect(rollbackSql.split("\n").length).toBeGreaterThan(50) + }, + 15 * 60 * 1000, + ) + + test("greenfield: Medallion topology → BRONZE / SILVER / GOLD databases", () => { + expect(greenfieldSql).toMatch(/CREATE DATABASE IF NOT EXISTS BRONZE/i) + expect(greenfieldSql).toMatch(/CREATE DATABASE IF NOT EXISTS SILVER/i) + expect(greenfieldSql).toMatch(/CREATE DATABASE IF NOT EXISTS GOLD/i) + }) + + test("greenfield: emission=idempotent → every CREATE uses IF NOT EXISTS", () => { + // Count creates and creates-with-guard. Ratio should be very high. + const totalCreates = (greenfieldSql.match(/^\s*CREATE\s+/gim) ?? []).length + const guardedCreates = (greenfieldSql.match(/^\s*CREATE\s+[A-Z ]+IF NOT EXISTS/gim) ?? []).length + + // The CREATE STORAGE INTEGRATION and CREATE PIPE forms sometimes use a + // different idempotency pattern (CREATE OR REPLACE is banned by guardrail 3 + // for those; DESC INTEGRATION is used to detect existence instead). Allow + // 90%+ to be IF NOT EXISTS. + expect(guardedCreates / totalCreates).toBeGreaterThan(0.85) + }) + + test("greenfield: guardrail #2 — FUTURE grants present", () => { + const futureGrants = (greenfieldSql.match(/GRANT[^;]+ON FUTURE/gi) ?? []).length + expect(futureGrants).toBeGreaterThanOrEqual(4) + }) + + test("greenfield: guardrail #1 — no ACCOUNTADMIN granted to service accounts", () => { + // Any occurrence of "GRANT ROLE ACCOUNTADMIN TO USER " is a bug. + expect(greenfieldSql).not.toMatch(/GRANT\s+ROLE\s+ACCOUNTADMIN\s+TO\s+USER/i) + // Service accounts must have DEFAULT_ROLE set to a functional role, not ACCOUNTADMIN. + const serviceAccounts = greenfieldSql.match(/CREATE USER[^;]+DEFAULT_ROLE\s*=\s*(\w+)/gi) ?? [] + for (const decl of serviceAccounts) { + expect(decl.toLowerCase()).not.toContain("default_role = accountadmin") + } + }) + + test("greenfield: guardrail #4 — role-scoped execution blocks present", () => { + // Must have all 3 role-switch blocks: ACCOUNTADMIN, SECURITYADMIN, SYSADMIN. + expect(greenfieldSql).toMatch(/USE ROLE ACCOUNTADMIN/i) + expect(greenfieldSql).toMatch(/USE ROLE SECURITYADMIN/i) + expect(greenfieldSql).toMatch(/USE ROLE SYSADMIN/i) + }) + + test("greenfield: PII masking policies emitted for declared categories", () => { + // User declared email/first_name/last_name — expect masking policies for each family. + expect(greenfieldSql).toMatch(/MASKING POLICY[^;]+email/i) + expect(greenfieldSql).toMatch(/MASKING POLICY[^;]+name/i) + }) + + test("greenfield: resource monitors present (account + at least one per-warehouse)", () => { + const monitors = (greenfieldSql.match(/CREATE\s+[A-Z ]*RESOURCE MONITOR/gi) ?? []).length + expect(monitors).toBeGreaterThanOrEqual(2) + }) + + test("greenfield: Configure-Before-Running placeholder checklist present", () => { + // Skill guardrail: placeholders must appear in an explicit checklist so users can't miss them. + expect(greenfieldSql).toMatch(/CONFIGURE BEFORE RUNNING/i) + // At least a few placeholder markers. + expect(greenfieldSql).toMatch(/<[A-Z_]+>/) + }) + + test("rollback: every DROP uses IF EXISTS", () => { + const totalDrops = (rollbackSql.match(/^\s*DROP\s+/gim) ?? []).length + const guardedDrops = (rollbackSql.match(/^\s*DROP\s+[A-Z ]+IF EXISTS/gim) ?? []).length + expect(totalDrops).toBeGreaterThan(0) + expect(guardedDrops / totalDrops).toBeGreaterThan(0.95) + }) + + // Helper: strip SQL comments (-- lines) before checking for statement + // patterns. The skill's rollback file has descriptive comments like + // "No DROP ... CASCADE anywhere" and "Never drops ... SNOWFLAKE database" + // that must not trip the regex — the assertions are about STATEMENTS. + const stripComments = (sql: string) => + sql + .split("\n") + .filter((line) => !line.trim().startsWith("--")) + .join("\n") + + test("rollback: no DROP ... CASCADE (guardrail: cascade drops mask misconfig)", () => { + expect(stripComments(rollbackSql)).not.toMatch(/DROP[^;]*CASCADE/i) + }) + + test("rollback: never drops built-in roles or SNOWFLAKE db", () => { + const statements = stripComments(rollbackSql) + for (const builtin of ["ACCOUNTADMIN", "SECURITYADMIN", "SYSADMIN", "USERADMIN", "PUBLIC", "ORGADMIN"]) { + // Must not appear as target of a DROP ROLE statement. + const dropPattern = new RegExp(`DROP\\s+ROLE[^;]*\\b${builtin}\\b`, "i") + expect(statements).not.toMatch(dropPattern) + } + expect(statements).not.toMatch(/DROP\s+DATABASE[^;]*\bSNOWFLAKE\b/i) + }) + + test("rollback: account-locator confirmation guard present", () => { + expect(rollbackSql).toMatch(/CURRENT_ACCOUNT\(\)/i) + expect(rollbackSql).toMatch(/rollback_confirmed_account|ROLLBACK CONFIRMATION/i) + }) +}) diff --git a/packages/opencode/test/altimate/tools/sql-execute.test.ts b/packages/opencode/test/altimate/tools/sql-execute.test.ts new file mode 100644 index 0000000000..61966684b0 --- /dev/null +++ b/packages/opencode/test/altimate/tools/sql-execute.test.ts @@ -0,0 +1,64 @@ +/** + * Regression tests for sql_execute error surfacing (found by 2026-08-25 + * snowflake-setup live eval on account DKZPOBS-TQ14188). + * + * Bug: the dispatcher's `sql.execute` handler catches driver errors and + * returns `{ columns: [], rows: [], row_count: 0, truncated: false, + * error: }` instead of throwing. The tool's original code ignored + * the `error` field and just ran `formatResult(result)`, which sees + * `row_count === 0` and prints "(0 rows)" — making a real SQL failure + * indistinguishable from a successful query that happened to return 0 rows. + * + * The fix: `sql-execute.ts` now checks `result.error` before formatting and + * short-circuits into an error response. + * + * These tests pin: + * (1) The `error` field remains part of the SqlExecuteResult type contract. + * (2) The tool's source code branches on `result.error` before formatting. + */ + +import { describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { join } from "node:path" + +const REPO_ROOT = join(import.meta.dir, "../../..") + +describe("sql_execute — error surfacing", () => { + test("SqlExecuteResult type includes an optional error field", () => { + const types = readFileSync(join(REPO_ROOT, "src/altimate/native/types.ts"), "utf-8") + // Locate the SqlExecuteResult interface, then check for `error?: string`. + const match = types.match(/export interface SqlExecuteResult\s*\{([^}]+)\}/) + expect(match).not.toBeNull() + expect(match![1]).toMatch(/error\?\s*:\s*string/) + }) + + test("sql-execute tool checks result.error before formatting", () => { + const src = readFileSync(join(REPO_ROOT, "src/altimate/tools/sql-execute.ts"), "utf-8") + // The fix branches on `result.error` and returns an ERROR title. Assert + // both the branch and the ERROR return exist within a reasonable window + // (the two must be co-located; the tool used to `formatResult` unconditionally). + expect(src).toMatch(/if\s*\(\s*result\.error\s*\)/) + // The ERROR-title return must be present near the `if (result.error)` branch, + // not somewhere else in the file (the catch-block also has an ERROR title; + // that's a different code path). + const errorBranchIdx = src.search(/if\s*\(\s*result\.error\s*\)/) + expect(errorBranchIdx).toBeGreaterThan(0) + const window = src.slice(errorBranchIdx, errorBranchIdx + 400) + expect(window).toMatch(/title:\s*["']SQL:\s*ERROR["']/) + expect(window).toMatch(/Failed to execute SQL/) + }) + + test("dispatcher's sql.execute returns error field on driver failure", () => { + // This test pins the dispatcher's error path so if it changes shape (e.g. + // starts throwing instead of returning an object), the tool needs updating too. + const register = readFileSync(join(REPO_ROOT, "src/altimate/native/connections/register.ts"), "utf-8") + // Locate the sql.execute handler's catch block. + const sqlExecIdx = register.search(/register\(\s*["']sql\.execute["']/) + expect(sqlExecIdx).toBeGreaterThan(0) + // Within the sql.execute handler, the catch clause must return an object with + // `error` populated — that's the contract the tool relies on. + const handlerWindow = register.slice(sqlExecIdx, sqlExecIdx + 5000) + expect(handlerWindow).toMatch(/error:\s*errorMsg/) + expect(handlerWindow).toMatch(/return\s*\{[^}]+error:/) + }) +})