From 4a0da4248c9075e765a219b1efb66ec48366ffe1 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 19 Aug 2026 13:47:46 -0700 Subject: [PATCH 1/4] ci(dagster): deploy code location as PEX instead of an image Both Dagster+ CD workflows built and pushed a container image to ECR on every run, which dominated deploy time even when only ingestion source changed. Serverless deploys can instead publish two PEX files -- deps.pex and source.pex -- and reuse a cached deps.pex when the resolved requirements are unchanged, so a source-only change uploads ~9MB rather than rebuilding a full image. Each workflow now runs actions/utils/prerun, which reports pex-deploy or docker-deploy from ENABLE_FAST_DEPLOYS. The PEX path is the default; the old image build survives verbatim in a second job reached by setting that variable to 'false', as the escape hatch for anything PEX cannot express (a dependency with no Linux wheel that also fails to build from source, a system package needing apt). Every dependency currently resolves to a linux/cp313 wheel, so neither case applies today. Two details are load-bearing: `--no-hashes` on the PEX export, and only there. The PEX builder unions requirements.txt with [project].dependencies from pyproject.toml, and those pins carry no hashes -- a hashed requirements.txt would put pip in --require-hashes mode, where every unhashed line is a hard error. Both sources resolve from the same uv.lock, so the duplicate pins agree. The Docker path keeps hashes; it feeds the file straight to pip install -r with nothing unhashed mixed in. python_version 3.13, against an action default of 3.8. The PEX files are resolved for one interpreter, and requires-python is >= 3.13. On the branch workflow, prerun also handles the closed-PR teardown: it runs `ci branch-deployment` to mark the deployment closed and reports skip, so the docker action's own closed-PR branch is no longer reached. dagster_cloud_post_install.sh is unchanged apart from its header. Only the Docker path still runs it; the source-pex builder performs the same `uv pip install --no-deps .` of this repository itself, which is what keeps db/ and domain/ importable from the process that executes a step. Co-Authored-By: Claude Opus 5 --- .github/workflows/CD_dagster_branch.yml | 152 +++++++++++++++++++++--- .github/workflows/CD_dagster_prod.yml | 138 +++++++++++++++++++-- dagster_cloud_post_install.sh | 8 +- 3 files changed, 272 insertions(+), 26 deletions(-) diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml index 69394860a..7c0928788 100644 --- a/.github/workflows/CD_dagster_branch.yml +++ b/.github/workflows/CD_dagster_branch.yml @@ -3,6 +3,21 @@ # # Path-filtered: most PRs in this repository touch only the API and should not # create a Dagster+ deployment at all. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. The deps.pex cache +# is keyed per repository, not per deployment, so a PR that changes no +# dependency reuses the one the last prod deploy published. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-branch-docker-deploy` job. Keep both +# paths working, and keep this file's setting in step with CD_dagster_prod.yml -- +# the two share the deps.pex cache, and a PR built the other way just misses it. name: CD (Dagster+ branch deployment) on: @@ -32,6 +47,22 @@ concurrency: group: dagster-branch-deploy-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + jobs: dagster-branch-deploy: runs-on: ubuntu-latest @@ -39,32 +70,125 @@ jobs: # untrusted fork would run our code against our infrastructure regardless. if: github.event.pull_request.head.repo.full_name == github.repository - # The action's notify steps post build status as a PR comment and read the - # token from the workflow environment -- `env.GITHUB_TOKEN`, not the - # `secrets` context. Without this the run dies on an empty-token assertion - # before it ever reaches Dagster+, which reads as an auth failure but is - # not one. + # The notify steps post build status as a PR comment and read the token from + # the workflow environment -- `env.GITHUB_TOKEN`, not the `secrets` context. + # Without this the run dies on an empty-token assertion before it ever + # reaches Dagster+, which reads as an auth failure but is not one. env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} + steps: - - name: Check out source repository - uses: actions/checkout@v7.0.1 + # Two jobs in one step. On a `closed` event it runs `ci + # branch-deployment`, which marks the branch deployment closed so stale + # deployments do not accumulate, and reports `skip` -- both deploy paths + # then do nothing. Otherwise it reads ENABLE_FAST_DEPLOYS and reports + # `pex-deploy` or `docker-deploy`. + # + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.18 # parse_workspace performs its own `actions/checkout`, which cleans the - # working tree. It has to run *before* requirements.txt is generated, or - # the generated file is deleted before the deploy step can use it. + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' id: parse uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 with: - dagster_cloud_file: dagster_cloud.yaml + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them, then creates or + # updates the branch deployment for this PR -- the action derives the + # deployment name from the pull_request event, so there is no `deployment` + # input to set here. + # + # On an ubuntu-24.04 runner deps.pex is built inside a python:3.13-slim + # container so the wheels match the serverless base image; source.pex is + # always built on the runner. That container only spins up when the + # dependency hash changes. + - name: Deploy to Dagster+ branch deployment + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.18 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-branch-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-branch-deploy + if: needs.dagster-branch-deploy.outputs.build_info + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-branch-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} - name: Install uv in container uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. - name: Generate requirements.txt run: | uv export \ @@ -74,17 +198,17 @@ jobs: --group ingestion \ --output-file requirements.txt - # Runs on `closed` too: the action tears the branch deployment down when - # the PR is merged or abandoned, so stale deployments do not accumulate. + # checkout_repo is false because requirements.txt is generated above and + # a second checkout would discard it. - name: Deploy to Dagster+ branch deployment uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v1.13.18 with: organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + location: ${{ toJson(matrix.location) }} checkout_repo: false # The action defaults to python:3.8-slim, which cannot install a # lockfile resolved for requires-python >= 3.13 -- pip reports the # pins as having no matching distribution rather than as a version # conflict, which reads like a broken requirements file. - base_image: python:3.13-slim + base_image: python:${{ env.PYTHON_VERSION }}-slim diff --git a/.github/workflows/CD_dagster_prod.yml b/.github/workflows/CD_dagster_prod.yml index 5c55ffce3..aa5bfe34d 100644 --- a/.github/workflows/CD_dagster_prod.yml +++ b/.github/workflows/CD_dagster_prod.yml @@ -16,6 +16,20 @@ # filter includes pyproject.toml and uv.lock because the location's dependency # set is exported from them, so a lockfile bump changes the built image even # when no ingestion source file does. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-prod-docker-deploy` job. That job is +# the escape hatch for anything the PEX path cannot express -- a dependency with +# no Linux wheel that also fails to build from source, or a system package that +# has to be installed with apt. Keep both paths working. name: CD (Dagster+ prod) on: @@ -44,6 +58,22 @@ concurrency: group: dagster-prod-deploy cancel-in-progress: false +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + jobs: dagster-prod-deploy: runs-on: ubuntu-latest @@ -54,29 +84,113 @@ jobs: # it would put an approval gate on routine merges once `production` requires # reviewers, which is a gate on the wrong thing: this publishes code, not # data. + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} steps: - - name: Check out source repository - uses: actions/checkout@v7.0.1 + # Reads ENABLE_FAST_DEPLOYS and emits `pex-deploy` or `docker-deploy`. + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.18 # parse_workspace performs its own `actions/checkout`, which cleans the - # working tree. It has to run *before* requirements.txt is generated, or - # the generated file is deleted before the deploy step can use it. + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' id: parse uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 with: - dagster_cloud_file: dagster_cloud.yaml + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them. deps.pex is keyed by + # a hash of the resolved requirements and cached per repository, so a run + # that changes only ingestion source skips the dependency build entirely. + # + # On an ubuntu-24.04 runner the action builds deps.pex inside a + # python:3.13-slim container so the wheels match the serverless base + # image; source.pex is always built on the runner. That container only + # spins up when the dependency hash changes. + - name: Deploy to Dagster+ prod + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.18 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + deployment: prod + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-prod-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-prod-deploy + if: needs.dagster-prod-deploy.outputs.build_info + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-prod-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} - name: Install uv in container uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" - # Dagster+ builds from a requirements.txt, which the repo does not keep - # under version control. `--group ingestion` adds dagster and dlt on top - # of the runtime dependencies; the runtime ones are needed too, because - # the loader imports `db/` and `domain/`. + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. - name: Generate requirements.txt run: | uv export \ @@ -93,10 +207,12 @@ jobs: with: organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + location: ${{ toJson(matrix.location) }} checkout_repo: false # The action defaults to python:3.8-slim, which cannot install a # lockfile resolved for requires-python >= 3.13 -- pip reports the # pins as having no matching distribution rather than as a version # conflict, which reads like a broken requirements file. - base_image: python:3.13-slim + base_image: python:${{ env.PYTHON_VERSION }}-slim + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dagster_cloud_post_install.sh b/dagster_cloud_post_install.sh index c093ae1c8..09cbc6e9b 100755 --- a/dagster_cloud_post_install.sh +++ b/dagster_cloud_post_install.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash -# Runs inside the Dagster+ image build, after the repository has been copied to +# Runs inside the Dagster+ *image* build, after the repository has been copied to # /opt/dagster/app and the pinned requirements installed. # +# Only the Docker fallback path reaches this script. The default PEX path builds +# no image: `dagster_cloud_cli`'s source-pex builder runs its own +# `uv pip install --target ... --no-deps .` over this repository, which is the +# same install by a different route. Both CD_dagster_*.yml workflows document +# the switch between the two. +# # Installs this repository as a package so `db`, `domain`, `services`, `core`, # and `schemas` resolve from site-packages. Without it they are importable only # while /opt/dagster/app happens to be on sys.path -- true for the process that From b7a6fb3e80092f05f212564c71049b4f5507fec3 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 19 Aug 2026 14:14:39 -0700 Subject: [PATCH 2/4] ci(dagster): add a dispatchable heartbeat smoke test A successful deploy is weaker evidence than a successful run. The agent loading the code location proves the loader process can import the package; it says nothing about the process that executes a step, which is a different process with a different sys.path. That gap is why assets/heartbeat.py exists, and it matters more now that the code location ships as PEX files rather than an image, because the two package the repository by different routes. Nothing could launch that asset from CI. dagster-cloud-action's launch_job identifies what to run by job name and exposes no asset selection, so an asset reachable only through the implicit __ASSET_JOB is unreachable. So wrap it in a named job, ingestion_heartbeat_check, and add a workflow_dispatch workflow that launches it against a deployment given as an input -- a branch deployment id or prod -- with wait: true so the workflow result is the materialization result rather than just "a run was launched". Dispatch-only on purpose. It costs a Dagster+ run, and the interesting time to spend one is after a deploy that changed how the code location is packaged, not on every push. No retry policy on the job: a retry would mask exactly the failure it exists to surface, since an import that works at load time and fails at execution does so deterministically. Co-Authored-By: Claude Opus 5 --- .github/workflows/smoke_dagster_location.yml | 51 ++++++++++++++++++++ automated_ingestion/defs/definitions.py | 3 +- automated_ingestion/defs/jobs/heartbeat.py | 49 +++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/smoke_dagster_location.yml create mode 100644 automated_ingestion/defs/jobs/heartbeat.py diff --git a/.github/workflows/smoke_dagster_location.yml b/.github/workflows/smoke_dagster_location.yml new file mode 100644 index 000000000..c3d40e37e --- /dev/null +++ b/.github/workflows/smoke_dagster_location.yml @@ -0,0 +1,51 @@ +# Materializes the heartbeat asset on a Dagster+ deployment, on demand. +# +# This is the run-time half of the deploy check. CD_dagster_*.yml prove the +# agent can load the code location; the loader is not the process that executes +# a step, and the two do not necessarily agree about sys.path, so loading +# cleanly does not prove a step can import `db` or `domain`. Materializing +# `ingestion_heartbeat` does. It touches no database, no network and no GCS, so a +# failure here is a packaging or deployment problem and nothing else. +# +# Dispatch-only. It costs a Dagster+ run, and the interesting time to spend one +# is after a deploy that changed how the code location is packaged -- not on +# every push. +# +# `deployment` takes a branch deployment id (the hex name in the Dagster+ URL, +# which CD_dagster_branch.yml prints as "Deploying to branch deployment: ...") +# or `prod`. Run it against the ref whose deployment you are testing: the job +# must exist in the deployed code location, not just on the branch. +name: Smoke test (Dagster+ code location) + +on: + workflow_dispatch: + inputs: + deployment: + description: "Dagster+ deployment: a branch deployment id, or 'prod'" + required: true + default: "prod" + +permissions: + contents: read + +concurrency: + group: dagster-smoke-${{ github.event.inputs.deployment }} + cancel-in-progress: false + +jobs: + heartbeat: + runs-on: ubuntu-latest + steps: + # wait: true makes the action poll the run and fail the step if the run + # fails, so the workflow result is the materialization result rather than + # just "a run was launched". + - name: Materialize ingestion_heartbeat + uses: dagster-io/dagster-cloud-action/actions/launch_job@v1.13.18 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location_name: ocotillo-automated-ingestion + deployment: ${{ github.event.inputs.deployment }} + job_name: ingestion_heartbeat_check + wait: "true" + interval: "10" diff --git a/automated_ingestion/defs/definitions.py b/automated_ingestion/defs/definitions.py index 3d4bd6c72..1bf2f7947 100644 --- a/automated_ingestion/defs/definitions.py +++ b/automated_ingestion/defs/definitions.py @@ -24,6 +24,7 @@ from dagster import Definitions from automated_ingestion.defs.assets import all_assets +from automated_ingestion.defs.jobs.heartbeat import heartbeat_job from automated_ingestion.defs.jobs.san_acacia import ( san_acacia_job, san_acacia_weekly_schedule, @@ -32,7 +33,7 @@ defs = Definitions( assets=all_assets(), - jobs=[san_acacia_job], + jobs=[heartbeat_job, san_acacia_job], schedules=[san_acacia_weekly_schedule], resources={"database": OcotilloDatabase()}, ) diff --git a/automated_ingestion/defs/jobs/heartbeat.py b/automated_ingestion/defs/jobs/heartbeat.py new file mode 100644 index 000000000..d95f80b66 --- /dev/null +++ b/automated_ingestion/defs/jobs/heartbeat.py @@ -0,0 +1,49 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +A named job wrapping the heartbeat asset, so a deploy can be smoke-tested +without the Dagster+ UI. + +The asset alone is not enough for that: `dagster-cloud-action`'s `launch_job` +identifies what to run by job name and exposes no asset selection, so an asset +reachable only through the implicit `__ASSET_JOB` cannot be launched from CI. +`.github/workflows/smoke_dagster_location.yml` runs this one. + +Worth having because a successful deploy is weaker evidence than a successful +run. The agent loading the code location proves the *loader* process can import +the package; it says nothing about the process that executes a step, which is a +different process with a different sys.path. See the note in +`assets/heartbeat.py` -- that gap is the whole reason the asset exists, and this +job is how CI closes it. +""" + +from dagster import AssetSelection, define_asset_job + +from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat + +heartbeat_job = define_asset_job( + name="ingestion_heartbeat_check", + selection=AssetSelection.assets(ingestion_heartbeat), + description=( + "Materialize the heartbeat asset only. Touches no database, no network, " + "and no GCS, so a failure is a packaging or deployment problem." + ), + # No retry policy. A retry would mask exactly the failure this job exists to + # surface: an import that works at load time and fails at execution does so + # deterministically. +) + +# ============= EOF ============================================= From 77195a89d775ffafedcf209cbb415cace10e28c6 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 19 Aug 2026 14:18:30 -0700 Subject: [PATCH 3/4] ci(dagster): materialize the heartbeat after a branch deploy The dispatch-only smoke test cannot cover the case it was written for. GitHub offers workflow_dispatch only for workflows present on the default branch, so a PR that changes how the code location is packaged -- exactly when the check is worth running -- cannot run it. So run it from the deploy workflow, as the last step of the PEX path. That also fixes the ordering for free: a separate workflow would race the deploy and could launch the job before the agent has synced the new code location, or before the job exists in it at all. The deployment name has to be resolved rather than assumed. It is derived from the branch and build_deploy_python_executable does not report it, so ask for it with the same `ci branch-deployment` call the deploy makes internally; that call is idempotent and returns the existing deployment for the PR. smoke_dagster_location.yml stays, with its purpose narrowed to the cases the deploy workflow does not reach: prod, and branch deployments that predate this step. Co-Authored-By: Claude Opus 5 --- .github/workflows/CD_dagster_branch.yml | 37 ++++++++++++++++++++ .github/workflows/smoke_dagster_location.yml | 11 ++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml index 7c0928788..6d7a61d28 100644 --- a/.github/workflows/CD_dagster_branch.yml +++ b/.github/workflows/CD_dagster_branch.yml @@ -160,6 +160,43 @@ jobs: build_output_dir: "$GITHUB_WORKSPACE/build" python_version: ${{ env.PYTHON_VERSION }} + # Materializing the heartbeat is the run-time half of the check. The steps + # above prove the agent can load the code location; the loader is not the + # process that executes a step, and the two do not necessarily agree about + # sys.path, so loading cleanly does not prove a step can import `db` or + # `domain`. This does. The asset touches no database, no network and no + # GCS, so a failure here is a packaging problem and nothing else -- which + # is worth one Dagster+ run on a PR that already changed how the code + # location is packaged. + # + # The deployment name has to be asked for rather than assumed: it is + # derived from the branch, and the deploy action does not report it. This + # is the same `ci branch-deployment` call the deploy makes internally, and + # it is idempotent -- it returns the existing deployment for this PR. + - name: Resolve branch deployment name + if: steps.prerun.outputs.result == 'pex-deploy' + id: branch_deployment + run: | + name=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud ci branch-deployment project-repo) + echo "name=$name" >> "$GITHUB_OUTPUT" + echo "Branch deployment: $name" + + # wait: true makes the action poll the run and fail this step if the run + # fails, so a green check means the asset materialized rather than just + # that a run was launched. + - name: Materialize ingestion_heartbeat + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/launch_job@v1.13.18 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location_name: ocotillo-automated-ingestion + deployment: ${{ steps.branch_deployment.outputs.name }} + job_name: ingestion_heartbeat_check + wait: "true" + interval: "10" + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This # is the pre-PEX workflow unchanged, including the post-install hook in # dagster_cloud_post_install.sh that the PEX path replaces with its own diff --git a/.github/workflows/smoke_dagster_location.yml b/.github/workflows/smoke_dagster_location.yml index c3d40e37e..f5d59a533 100644 --- a/.github/workflows/smoke_dagster_location.yml +++ b/.github/workflows/smoke_dagster_location.yml @@ -7,9 +7,14 @@ # `ingestion_heartbeat` does. It touches no database, no network and no GCS, so a # failure here is a packaging or deployment problem and nothing else. # -# Dispatch-only. It costs a Dagster+ run, and the interesting time to spend one -# is after a deploy that changed how the code location is packaged -- not on -# every push. +# Dispatch-only, and mainly for `prod`: CD_dagster_branch.yml already runs this +# same job against a branch deployment as the last step of every PEX deploy, so +# on a PR the check happens without anyone asking. This workflow is the way to +# ask for it anywhere else -- after a prod deploy, or against a branch +# deployment that was deployed before this check existed. +# +# Note a GitHub constraint: `workflow_dispatch` is only offered for workflows +# present on the default branch, so this is not runnable from a feature branch. # # `deployment` takes a branch deployment id (the hex name in the Dagster+ URL, # which CD_dagster_branch.yml prints as "Deploying to branch deployment: ...") From f95ad29ddee91318b7076f4271fc1e1f66fd9a29 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 19 Aug 2026 14:24:45 -0700 Subject: [PATCH 4/4] fix(ci): assert the heartbeat run succeeded The previous commit claimed a green check meant the asset materialized. It did not. `launch_job`'s run.sh captures the CLI output in a command substitution and never checks the exit code, deciding success by whether it can regex a run id out of the text. Underneath, `dagster-cloud job launch --wait` reports a failed run with `ui.error(...)` -- and `ui.error` only returns an exception rather than raising it, so the CLI exits 0 as well. The action documents "fail if the run fails"; at neither layer can it. A failed materialization produced a passing step. Call the CLI directly and require the "finished successfully" line. Matching on output is not lovely, but it is the only signal either layer emits, and an assertion that can fail is worth more than one that reads better. Co-Authored-By: Claude Opus 5 --- .github/workflows/CD_dagster_branch.yml | 43 ++++++++++++++------ .github/workflows/smoke_dagster_location.yml | 42 +++++++++++++------ 2 files changed, 61 insertions(+), 24 deletions(-) diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml index 6d7a61d28..5b349474e 100644 --- a/.github/workflows/CD_dagster_branch.yml +++ b/.github/workflows/CD_dagster_branch.yml @@ -182,20 +182,39 @@ jobs: echo "name=$name" >> "$GITHUB_OUTPUT" echo "Branch deployment: $name" - # wait: true makes the action poll the run and fail this step if the run - # fails, so a green check means the asset materialized rather than just - # that a run was launched. + # Deliberately not the vendor's `launch_job` action, and the success + # assertion is deliberately our own. + # + # `launch_job` documents `wait: true` as "the action will wait for the run + # to finish and fail if the run fails". It does wait, but it cannot fail: + # its run.sh captures the CLI output in a command substitution and never + # checks the exit code, deciding success by whether it can regex a run id + # out of the text. Underneath, `dagster-cloud job launch --wait` reports a + # failed run with `ui.error(...)` -- and `ui.error` only *returns* an + # exception rather than raising it, so the CLI exits 0 too. A failed + # materialization would have produced a green check on both counts. + # + # So call the CLI directly and require the success line. Matching on output + # is not lovely, but it is the only signal either layer actually emits. - name: Materialize ingestion_heartbeat if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/launch_job@v1.13.18 - with: - organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location_name: ocotillo-automated-ingestion - deployment: ${{ steps.branch_deployment.outputs.name }} - job_name: ingestion_heartbeat_check - wait: "true" - interval: "10" + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ steps.branch_deployment.outputs.name }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This # is the pre-PEX workflow unchanged, including the post-install hook in diff --git a/.github/workflows/smoke_dagster_location.yml b/.github/workflows/smoke_dagster_location.yml index f5d59a533..d38ab9ad8 100644 --- a/.github/workflows/smoke_dagster_location.yml +++ b/.github/workflows/smoke_dagster_location.yml @@ -37,20 +37,38 @@ concurrency: group: dagster-smoke-${{ github.event.inputs.deployment }} cancel-in-progress: false +env: + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + jobs: heartbeat: runs-on: ubuntu-latest steps: - # wait: true makes the action poll the run and fail the step if the run - # fails, so the workflow result is the materialization result rather than - # just "a run was launched". - - name: Materialize ingestion_heartbeat - uses: dagster-io/dagster-cloud-action/actions/launch_job@v1.13.18 + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 with: - organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location_name: ocotillo-automated-ingestion - deployment: ${{ github.event.inputs.deployment }} - job_name: ingestion_heartbeat_check - wait: "true" - interval: "10" + version: "latest" + + # The vendor's `launch_job` action is not used here, for the reason spelled + # out in CD_dagster_branch.yml: neither it nor `dagster-cloud job launch` + # exits nonzero on a failed run, so `wait: true` waits without gating. + # Requiring the success line is what makes this workflow's result mean + # something. + - name: Materialize ingestion_heartbeat + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ github.event.inputs.deployment }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac