diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml index 69394860a..5b349474e 100644 --- a/.github/workflows/CD_dagster_branch.yml +++ b/.github/workflows/CD_dagster_branch.yml @@ -3,6 +3,21 @@ # # Path-filtered: most PRs in this repository touch only the API and should not # create a Dagster+ deployment at all. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. The deps.pex cache +# is keyed per repository, not per deployment, so a PR that changes no +# dependency reuses the one the last prod deploy published. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-branch-docker-deploy` job. Keep both +# paths working, and keep this file's setting in step with CD_dagster_prod.yml -- +# the two share the deps.pex cache, and a PR built the other way just misses it. name: CD (Dagster+ branch deployment) on: @@ -32,6 +47,22 @@ concurrency: group: dagster-branch-deploy-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + jobs: dagster-branch-deploy: runs-on: ubuntu-latest @@ -39,32 +70,181 @@ jobs: # untrusted fork would run our code against our infrastructure regardless. if: github.event.pull_request.head.repo.full_name == github.repository - # The action's notify steps post build status as a PR comment and read the - # token from the workflow environment -- `env.GITHUB_TOKEN`, not the - # `secrets` context. Without this the run dies on an empty-token assertion - # before it ever reaches Dagster+, which reads as an auth failure but is - # not one. + # The notify steps post build status as a PR comment and read the token from + # the workflow environment -- `env.GITHUB_TOKEN`, not the `secrets` context. + # Without this the run dies on an empty-token assertion before it ever + # reaches Dagster+, which reads as an auth failure but is not one. env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} + steps: - - name: Check out source repository - uses: actions/checkout@v7.0.1 + # Two jobs in one step. On a `closed` event it runs `ci + # branch-deployment`, which marks the branch deployment closed so stale + # deployments do not accumulate, and reports `skip` -- both deploy paths + # then do nothing. Otherwise it reads ENABLE_FAST_DEPLOYS and reports + # `pex-deploy` or `docker-deploy`. + # + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.18 # parse_workspace performs its own `actions/checkout`, which cleans the - # working tree. It has to run *before* requirements.txt is generated, or - # the generated file is deleted before the deploy step can use it. + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' id: parse uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 with: - dagster_cloud_file: dagster_cloud.yaml + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them, then creates or + # updates the branch deployment for this PR -- the action derives the + # deployment name from the pull_request event, so there is no `deployment` + # input to set here. + # + # On an ubuntu-24.04 runner deps.pex is built inside a python:3.13-slim + # container so the wheels match the serverless base image; source.pex is + # always built on the runner. That container only spins up when the + # dependency hash changes. + - name: Deploy to Dagster+ branch deployment + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.18 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + + # Materializing the heartbeat is the run-time half of the check. The steps + # above prove the agent can load the code location; the loader is not the + # process that executes a step, and the two do not necessarily agree about + # sys.path, so loading cleanly does not prove a step can import `db` or + # `domain`. This does. The asset touches no database, no network and no + # GCS, so a failure here is a packaging problem and nothing else -- which + # is worth one Dagster+ run on a PR that already changed how the code + # location is packaged. + # + # The deployment name has to be asked for rather than assumed: it is + # derived from the branch, and the deploy action does not report it. This + # is the same `ci branch-deployment` call the deploy makes internally, and + # it is idempotent -- it returns the existing deployment for this PR. + - name: Resolve branch deployment name + if: steps.prerun.outputs.result == 'pex-deploy' + id: branch_deployment + run: | + name=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud ci branch-deployment project-repo) + echo "name=$name" >> "$GITHUB_OUTPUT" + echo "Branch deployment: $name" + + # Deliberately not the vendor's `launch_job` action, and the success + # assertion is deliberately our own. + # + # `launch_job` documents `wait: true` as "the action will wait for the run + # to finish and fail if the run fails". It does wait, but it cannot fail: + # its run.sh captures the CLI output in a command substitution and never + # checks the exit code, deciding success by whether it can regex a run id + # out of the text. Underneath, `dagster-cloud job launch --wait` reports a + # failed run with `ui.error(...)` -- and `ui.error` only *returns* an + # exception rather than raising it, so the CLI exits 0 too. A failed + # materialization would have produced a green check on both counts. + # + # So call the CLI directly and require the success line. Matching on output + # is not lovely, but it is the only signal either layer actually emits. + - name: Materialize ingestion_heartbeat + if: steps.prerun.outputs.result == 'pex-deploy' + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ steps.branch_deployment.outputs.name }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-branch-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-branch-deploy + if: needs.dagster-branch-deploy.outputs.build_info + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-branch-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.head_ref }} - name: Install uv in container uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. - name: Generate requirements.txt run: | uv export \ @@ -74,17 +254,17 @@ jobs: --group ingestion \ --output-file requirements.txt - # Runs on `closed` too: the action tears the branch deployment down when - # the PR is merged or abandoned, so stale deployments do not accumulate. + # checkout_repo is false because requirements.txt is generated above and + # a second checkout would discard it. - name: Deploy to Dagster+ branch deployment uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v1.13.18 with: organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + location: ${{ toJson(matrix.location) }} checkout_repo: false # The action defaults to python:3.8-slim, which cannot install a # lockfile resolved for requires-python >= 3.13 -- pip reports the # pins as having no matching distribution rather than as a version # conflict, which reads like a broken requirements file. - base_image: python:3.13-slim + base_image: python:${{ env.PYTHON_VERSION }}-slim diff --git a/.github/workflows/CD_dagster_prod.yml b/.github/workflows/CD_dagster_prod.yml index 5c55ffce3..aa5bfe34d 100644 --- a/.github/workflows/CD_dagster_prod.yml +++ b/.github/workflows/CD_dagster_prod.yml @@ -16,6 +16,20 @@ # filter includes pyproject.toml and uv.lock because the location's dependency # set is exported from them, so a lockfile bump changes the built image even # when no ingestion source file does. +# +# ## Two deploy paths +# +# The default path builds no container image at all: it packages the +# dependencies and the source into two PEX files and uploads them, so a source +# change reuses the previously published deps.pex instead of rebuilding and +# pushing several hundred megabytes to ECR. See +# https://dagster.io/blog/fast-deploys-with-pex-and-docker. +# +# Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back +# to the Docker image build in the `dagster-prod-docker-deploy` job. That job is +# the escape hatch for anything the PEX path cannot express -- a dependency with +# no Linux wheel that also fails to build from source, or a system package that +# has to be installed with apt. Keep both paths working. name: CD (Dagster+ prod) on: @@ -44,6 +58,22 @@ concurrency: group: dagster-prod-deploy cancel-in-progress: false +env: + # The PEX path targets `$DAGSTER_CLOUD_URL/`, so this is the + # organization URL with no deployment path appended. The Docker path takes the + # organization id as an action input instead and ignores this. + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` + # and so decides which of the two jobs below runs. + ENABLE_FAST_DEPLOYS: "true" + # The PEX files are resolved for this interpreter only, and requires-python is + # >= 3.13. Anything lower makes the deps resolve fail with "no matching + # distribution", which reads like a broken requirements file rather than a + # version mismatch. + PYTHON_VERSION: "3.13" + DAGSTER_CLOUD_FILE: dagster_cloud.yaml + jobs: dagster-prod-deploy: runs-on: ubuntu-latest @@ -54,29 +84,113 @@ jobs: # it would put an approval gate on routine merges once `production` requires # reviewers, which is a gate on the wrong thing: this publishes code, not # data. + outputs: + # Set only on the Docker path. The fallback job keys off it being empty. + build_info: ${{ steps.parse.outputs.build_info }} steps: - - name: Check out source repository - uses: actions/checkout@v7.0.1 + # Reads ENABLE_FAST_DEPLOYS and emits `pex-deploy` or `docker-deploy`. + # Its own checkout goes to prerun_checkout_dir, so it does not disturb the + # working tree either path sets up below. + - name: Prerun checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.18 # parse_workspace performs its own `actions/checkout`, which cleans the - # working tree. It has to run *before* requirements.txt is generated, or - # the generated file is deleted before the deploy step can use it. + # working tree -- fine here because the Docker path runs in a separate job + # that checks out again. Only its `build_info` output crosses the boundary. - name: Parse dagster_cloud.yaml + if: steps.prerun.outputs.result == 'docker-deploy' id: parse uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 with: - dagster_cloud_file: dagster_cloud.yaml + dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} + + # Checked out under a subdirectory because build_deploy_python_executable + # takes an absolute path to the location file and does not check out + # anything itself, so nothing later can clobber the generated + # requirements.txt. + - name: Check out source repository + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} + path: project-repo + + - name: Install uv + if: steps.prerun.outputs.result == 'pex-deploy' + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # `--group ingestion` adds dagster and dlt on top of the runtime + # dependencies; the runtime ones are needed too, because the loader + # imports `db/` and `domain/`. + # + # `--no-hashes` is required on this path and only on this path. The PEX + # builder unions requirements.txt with `[project].dependencies` from + # pyproject.toml, and those pins carry no hashes -- a hashed + # requirements.txt would put pip in --require-hashes mode, where every + # unhashed line is an error. Both sources resolve from the same uv.lock, + # so the duplicate pins agree and drop out. + - name: Generate requirements.txt + if: steps.prerun.outputs.result == 'pex-deploy' + working-directory: project-repo + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --no-hashes \ + --group ingestion \ + --output-file requirements.txt + + # Builds deps.pex and source.pex and publishes them. deps.pex is keyed by + # a hash of the resolved requirements and cached per repository, so a run + # that changes only ingestion source skips the dependency build entirely. + # + # On an ubuntu-24.04 runner the action builds deps.pex inside a + # python:3.13-slim container so the wheels match the serverless base + # image; source.pex is always built on the runner. That container only + # spins up when the dependency hash changes. + - name: Deploy to Dagster+ prod + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.18 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: ${{ env.PYTHON_VERSION }} + deployment: prod + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This + # is the pre-PEX workflow unchanged, including the post-install hook in + # dagster_cloud_post_install.sh that the PEX path replaces with its own + # `uv pip install --no-deps .` of the repository. + dagster-prod-docker-deploy: + runs-on: ubuntu-latest + needs: dagster-prod-deploy + if: needs.dagster-prod-deploy.outputs.build_info + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster-prod-deploy.outputs.build_info) }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + with: + ref: ${{ github.sha }} - name: Install uv in container uses: astral-sh/setup-uv@v10.0.1 with: version: "latest" - # Dagster+ builds from a requirements.txt, which the repo does not keep - # under version control. `--group ingestion` adds dagster and dlt on top - # of the runtime dependencies; the runtime ones are needed too, because - # the loader imports `db/` and `domain/`. + # Dagster+ builds from a requirements.txt. Hashes are kept here: this path + # feeds the file straight to `pip install -r`, with nothing unhashed mixed + # in. - name: Generate requirements.txt run: | uv export \ @@ -93,10 +207,12 @@ jobs: with: organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + location: ${{ toJson(matrix.location) }} checkout_repo: false # The action defaults to python:3.8-slim, which cannot install a # lockfile resolved for requires-python >= 3.13 -- pip reports the # pins as having no matching distribution rather than as a version # conflict, which reads like a broken requirements file. - base_image: python:3.13-slim + base_image: python:${{ env.PYTHON_VERSION }}-slim + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/smoke_dagster_location.yml b/.github/workflows/smoke_dagster_location.yml new file mode 100644 index 000000000..d38ab9ad8 --- /dev/null +++ b/.github/workflows/smoke_dagster_location.yml @@ -0,0 +1,74 @@ +# Materializes the heartbeat asset on a Dagster+ deployment, on demand. +# +# This is the run-time half of the deploy check. CD_dagster_*.yml prove the +# agent can load the code location; the loader is not the process that executes +# a step, and the two do not necessarily agree about sys.path, so loading +# cleanly does not prove a step can import `db` or `domain`. Materializing +# `ingestion_heartbeat` does. It touches no database, no network and no GCS, so a +# failure here is a packaging or deployment problem and nothing else. +# +# Dispatch-only, and mainly for `prod`: CD_dagster_branch.yml already runs this +# same job against a branch deployment as the last step of every PEX deploy, so +# on a PR the check happens without anyone asking. This workflow is the way to +# ask for it anywhere else -- after a prod deploy, or against a branch +# deployment that was deployed before this check existed. +# +# Note a GitHub constraint: `workflow_dispatch` is only offered for workflows +# present on the default branch, so this is not runnable from a feature branch. +# +# `deployment` takes a branch deployment id (the hex name in the Dagster+ URL, +# which CD_dagster_branch.yml prints as "Deploying to branch deployment: ...") +# or `prod`. Run it against the ref whose deployment you are testing: the job +# must exist in the deployed code location, not just on the branch. +name: Smoke test (Dagster+ code location) + +on: + workflow_dispatch: + inputs: + deployment: + description: "Dagster+ deployment: a branch deployment id, or 'prod'" + required: true + default: "prod" + +permissions: + contents: read + +concurrency: + group: dagster-smoke-${{ github.event.inputs.deployment }} + cancel-in-progress: false + +env: + DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud + +jobs: + heartbeat: + runs-on: ubuntu-latest + steps: + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # The vendor's `launch_job` action is not used here, for the reason spelled + # out in CD_dagster_branch.yml: neither it nor `dagster-cloud job launch` + # exits nonzero on a failed run, so `wait: true` waits without gating. + # Requiring the success line is what makes this workflow's result mean + # something. + - name: Materialize ingestion_heartbeat + env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + DEPLOYMENT: ${{ github.event.inputs.deployment }} + run: | + set -uo pipefail + out=$(uvx --from "dagster-cloud-cli==1.13.18" \ + dagster-cloud job launch \ + --url "$DAGSTER_CLOUD_URL" \ + --deployment "$DEPLOYMENT" \ + --location ocotillo-automated-ingestion \ + --job ingestion_heartbeat_check \ + --wait --interval 10 2>&1 | tee /dev/stderr) + case "$out" in + *"finished successfully"*) ;; + *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" + exit 1 ;; + esac diff --git a/automated_ingestion/defs/definitions.py b/automated_ingestion/defs/definitions.py index 3d4bd6c72..1bf2f7947 100644 --- a/automated_ingestion/defs/definitions.py +++ b/automated_ingestion/defs/definitions.py @@ -24,6 +24,7 @@ from dagster import Definitions from automated_ingestion.defs.assets import all_assets +from automated_ingestion.defs.jobs.heartbeat import heartbeat_job from automated_ingestion.defs.jobs.san_acacia import ( san_acacia_job, san_acacia_weekly_schedule, @@ -32,7 +33,7 @@ defs = Definitions( assets=all_assets(), - jobs=[san_acacia_job], + jobs=[heartbeat_job, san_acacia_job], schedules=[san_acacia_weekly_schedule], resources={"database": OcotilloDatabase()}, ) diff --git a/automated_ingestion/defs/jobs/heartbeat.py b/automated_ingestion/defs/jobs/heartbeat.py new file mode 100644 index 000000000..d95f80b66 --- /dev/null +++ b/automated_ingestion/defs/jobs/heartbeat.py @@ -0,0 +1,49 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +A named job wrapping the heartbeat asset, so a deploy can be smoke-tested +without the Dagster+ UI. + +The asset alone is not enough for that: `dagster-cloud-action`'s `launch_job` +identifies what to run by job name and exposes no asset selection, so an asset +reachable only through the implicit `__ASSET_JOB` cannot be launched from CI. +`.github/workflows/smoke_dagster_location.yml` runs this one. + +Worth having because a successful deploy is weaker evidence than a successful +run. The agent loading the code location proves the *loader* process can import +the package; it says nothing about the process that executes a step, which is a +different process with a different sys.path. See the note in +`assets/heartbeat.py` -- that gap is the whole reason the asset exists, and this +job is how CI closes it. +""" + +from dagster import AssetSelection, define_asset_job + +from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat + +heartbeat_job = define_asset_job( + name="ingestion_heartbeat_check", + selection=AssetSelection.assets(ingestion_heartbeat), + description=( + "Materialize the heartbeat asset only. Touches no database, no network, " + "and no GCS, so a failure is a packaging or deployment problem." + ), + # No retry policy. A retry would mask exactly the failure this job exists to + # surface: an import that works at load time and fails at execution does so + # deterministically. +) + +# ============= EOF ============================================= diff --git a/dagster_cloud_post_install.sh b/dagster_cloud_post_install.sh index c093ae1c8..09cbc6e9b 100755 --- a/dagster_cloud_post_install.sh +++ b/dagster_cloud_post_install.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash -# Runs inside the Dagster+ image build, after the repository has been copied to +# Runs inside the Dagster+ *image* build, after the repository has been copied to # /opt/dagster/app and the pinned requirements installed. # +# Only the Docker fallback path reaches this script. The default PEX path builds +# no image: `dagster_cloud_cli`'s source-pex builder runs its own +# `uv pip install --target ... --no-deps .` over this repository, which is the +# same install by a different route. Both CD_dagster_*.yml workflows document +# the switch between the two. +# # Installs this repository as a package so `db`, `domain`, `services`, `core`, # and `schemas` resolve from site-packages. Without it they are importable only # while /opt/dagster/app happens to be on sys.path -- true for the process that