From affa35c3e0f5504cf4d85588f73e91a48bc21d50 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 18 Aug 2026 20:35:10 -0700 Subject: [PATCH 1/2] fix(ingestion): set code location env vars at a scope that reaches the container Dropping --global scoped variables to `OcotilloAPI` -- the project name -- rather than to `ocotillo-automated-ingestion`, the name in dagster_cloud.yaml. Dagster+ accepts the unknown location without complaint, so INGESTION_GCP_CREDENTIALS_JSON appeared correctly set in the UI while the container never received it, and the asset kept failing with DefaultCredentialsError as though no key had been set. `code_location_name` in [tool.dg.project] looks like the fix and is ignored for this command, so that avenue is closed and the attempt is not left in the file. Back to --global, which is broader than ideal: this deployment also hosts aqueduct_dagster_defs_definitions and die-orchestration, which can read these values. That is a real cost, recorded in the script along with the alternative -- scoping in the Dagster+ UI against the correct location name -- rather than traded away silently. A credential the container cannot read is worth less than one scoped more broadly than we would like. Co-Authored-By: Claude Opus 5 --- .../scripts/set_code_location_env.sh | 27 +++++++++++++------ pyproject.toml | 1 + 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/automated_ingestion/scripts/set_code_location_env.sh b/automated_ingestion/scripts/set_code_location_env.sh index e4e872d8c..49015dc78 100755 --- a/automated_ingestion/scripts/set_code_location_env.sh +++ b/automated_ingestion/scripts/set_code_location_env.sh @@ -14,10 +14,9 @@ # read -rs "DIVERHUB_PASSWORD?Diver-HUB password: "; echo # export DIVERHUB_USERNAME DIVERHUB_PASSWORD # -# Variables are scoped to this code location, not the deployment. If an earlier -# run set them with --global, delete those deployment-level entries in the -# Dagster+ UI afterwards -- otherwise both exist and which one wins is not -# obvious from either place. +# Variables are set at deployment scope. See the comment on set_var for why +# location scoping through dg does not work, and what to do instead if these +# values must not be visible to the other code locations in this deployment. # # Usage: # ./automated_ingestion/scripts/set_code_location_env.sh storage @@ -34,10 +33,22 @@ set -euo pipefail DG="uv run --with dagster-dg-cli dg" PHASE="${1:-}" -# No --global: that sets the variable at deployment level, where every other -# code location in this deployment can read it. This deployment is shared, so -# the vendor and database credentials stay scoped to this location. -set_var() { echo " $1"; $DG plus create env "$@" -y >/dev/null; } +# --global sets the variable at deployment level. That is broader than ideal -- +# this deployment also hosts aqueduct_dagster_defs_definitions and +# die-orchestration, which can then read these values -- but it is the scope +# that actually reaches the container. +# +# Location scoping through dg does not work here: dg names the location from the +# project (`OcotilloAPI`), not from `location_name` in dagster_cloud.yaml +# (`ocotillo-automated-ingestion`), and `code_location_name` in [tool.dg.project] +# is ignored for this command. Dagster+ accepts the unknown name without +# complaint, so the variable shows as set in the UI and is absent in the +# container -- which costs an afternoon to work out from a +# DefaultCredentialsError. +# +# To scope properly, set the variable in the Dagster+ UI against +# `ocotillo-automated-ingestion` instead. +set_var() { echo " $1"; $DG plus create env "$@" --global -y >/dev/null; } case "$PHASE" in storage) diff --git a/pyproject.toml b/pyproject.toml index b33d0e623..0fc90ce14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,6 +141,7 @@ directory_type = "project" [tool.dg.project] root_module = "automated_ingestion" + # Bare `--cov` measures every imported module, which pulls the whole virtualenv # into the report. Scope it to first-party code instead. Keep this a single # source root -- listing each package separately makes coverage treat every From a178f90132cacabbc09ccd4cb77055defc48d839 Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 18 Aug 2026 20:36:36 -0700 Subject: [PATCH 2/2] fix(ingestion): reject a bare Cloud SQL instance name CLOUD_SQL_INSTANCE_NAME was set to `dataservices`, which every layer accepts until the connector tries to use it and raises a ValueError from inside the driver, well below anything this project wrote. The connector wants the full connection name, PROJECT:REGION:INSTANCE. The script now checks the shape before setting it, and points at the gcloud command that prints the right value. Co-Authored-By: Claude Opus 5 --- .../scripts/set_code_location_env.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/automated_ingestion/scripts/set_code_location_env.sh b/automated_ingestion/scripts/set_code_location_env.sh index 49015dc78..c6cd1de70 100755 --- a/automated_ingestion/scripts/set_code_location_env.sh +++ b/automated_ingestion/scripts/set_code_location_env.sh @@ -83,8 +83,22 @@ vendor) set_var DIVERHUB_PASSWORD --from-local-env ;; database) - : "${CLOUD_SQL_INSTANCE_NAME:?export it first}" + : "${CLOUD_SQL_INSTANCE_NAME:?export it first, as PROJECT:REGION:INSTANCE}" : "${CLOUD_SQL_DATABASE:?export it first}" + + # The connector wants the full connection name, not the instance name. A bare + # name is accepted by everything up to the point of connecting and then fails + # with a ValueError from deep inside the driver, several layers below anything + # this project wrote. Catch it here instead. + # gcloud sql instances list --format='value(name,connectionName)' + case "$CLOUD_SQL_INSTANCE_NAME" in + *:*:*) ;; + *) + echo "error: CLOUD_SQL_INSTANCE_NAME must be PROJECT:REGION:INSTANCE," >&2 + echo " got '${CLOUD_SQL_INSTANCE_NAME}'." >&2 + exit 65 + ;; + esac echo "Cloud SQL connection:" set_var DB_DRIVER cloudsql set_var CLOUD_SQL_IP_TYPE public