feat(transducer): publish and range-delete for corrected hydrographs #71
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Creates a Dagster+ branch deployment for a pull request, so ingestion changes | |
| # can be materialized against an isolated deployment before they reach prod. | |
| # | |
| # Path-filtered: most PRs in this repository touch only the API and should not | |
| # create a Dagster+ deployment at all. | |
| # | |
| # ## Two deploy paths | |
| # | |
| # The default path builds no container image at all: it packages the | |
| # dependencies and the source into two PEX files and uploads them, so a source | |
| # change reuses the previously published deps.pex instead of rebuilding and | |
| # pushing several hundred megabytes to ECR. See | |
| # https://dagster.io/blog/fast-deploys-with-pex-and-docker. The deps.pex cache | |
| # is keyed per repository, not per deployment, so a PR that changes no | |
| # dependency reuses the one the last prod deploy published. | |
| # | |
| # Setting ENABLE_FAST_DEPLOYS to 'false' below switches the whole workflow back | |
| # to the Docker image build in the `dagster-branch-docker-deploy` job. Keep both | |
| # paths working, and keep this file's setting in step with CD_dagster_prod.yml -- | |
| # the two share the deps.pex cache, and a PR built the other way just misses it. | |
| name: CD (Dagster+ branch deployment) | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, closed] | |
| paths: | |
| - "automated_ingestion/**" | |
| # The code location imports db/ models and domain/ rules in-process, | |
| # so a change to either alters what this image runs even when no | |
| # ingestion file moves. Without these, a domain fix merged to | |
| # production would leave the pipeline running the old rule against | |
| # the live database. The cost is that ordinary API changes to these | |
| # directories also trigger a build; a stale code location is worse. | |
| - "db/**" | |
| - "domain/**" | |
| - "dagster_cloud.yaml" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| - ".github/workflows/CD_dagster_branch.yml" | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| # One deployment per PR; a force-push supersedes the run it interrupts. | |
| concurrency: | |
| group: dagster-branch-deploy-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| env: | |
| # The PEX path targets `$DAGSTER_CLOUD_URL/<branch deployment>`, so this is the | |
| # organization URL with no deployment path appended. The Docker path takes the | |
| # organization id as an action input instead and ignores this. | |
| DAGSTER_CLOUD_URL: https://${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }}.dagster.cloud | |
| DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| # Read by actions/utils/prerun, which reports `pex-deploy` or `docker-deploy` | |
| # and so decides which of the two jobs below runs. | |
| ENABLE_FAST_DEPLOYS: "true" | |
| # The PEX files are resolved for this interpreter only, and requires-python is | |
| # >= 3.13. Anything lower makes the deps resolve fail with "no matching | |
| # distribution", which reads like a broken requirements file rather than a | |
| # version mismatch. | |
| PYTHON_VERSION: "3.13" | |
| DAGSTER_CLOUD_FILE: dagster_cloud.yaml | |
| jobs: | |
| dagster-branch-deploy: | |
| runs-on: ubuntu-latest | |
| # Forks cannot read the Dagster+ secrets, and a branch deployment from an | |
| # untrusted fork would run our code against our infrastructure regardless. | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| # The notify steps post build status as a PR comment and read the token from | |
| # the workflow environment -- `env.GITHUB_TOKEN`, not the `secrets` context. | |
| # Without this the run dies on an empty-token assertion before it ever | |
| # reaches Dagster+, which reads as an auth failure but is not one. | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| outputs: | |
| # Set only on the Docker path. The fallback job keys off it being empty. | |
| build_info: ${{ steps.parse.outputs.build_info }} | |
| steps: | |
| # Two jobs in one step. On a `closed` event it runs `ci | |
| # branch-deployment`, which marks the branch deployment closed so stale | |
| # deployments do not accumulate, and reports `skip` -- both deploy paths | |
| # then do nothing. Otherwise it reads ENABLE_FAST_DEPLOYS and reports | |
| # `pex-deploy` or `docker-deploy`. | |
| # | |
| # Its own checkout goes to prerun_checkout_dir, so it does not disturb the | |
| # working tree either path sets up below. | |
| - name: Prerun checks | |
| id: prerun | |
| uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v1.13.18 | |
| # parse_workspace performs its own `actions/checkout`, which cleans the | |
| # working tree -- fine here because the Docker path runs in a separate job | |
| # that checks out again. Only its `build_info` output crosses the boundary. | |
| - name: Parse dagster_cloud.yaml | |
| if: steps.prerun.outputs.result == 'docker-deploy' | |
| id: parse | |
| uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 | |
| with: | |
| dagster_cloud_file: ${{ env.DAGSTER_CLOUD_FILE }} | |
| # Checked out under a subdirectory because build_deploy_python_executable | |
| # takes an absolute path to the location file and does not check out | |
| # anything itself, so nothing later can clobber the generated | |
| # requirements.txt. | |
| - name: Check out source repository | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| uses: actions/checkout@v7.0.1 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| path: project-repo | |
| - name: Install uv | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "latest" | |
| # `--group ingestion` adds dagster and dlt on top of the runtime | |
| # dependencies; the runtime ones are needed too, because the loader | |
| # imports `db/` and `domain/`. | |
| # | |
| # `--no-hashes` is required on this path and only on this path. The PEX | |
| # builder unions requirements.txt with `[project].dependencies` from | |
| # pyproject.toml, and those pins carry no hashes -- a hashed | |
| # requirements.txt would put pip in --require-hashes mode, where every | |
| # unhashed line is an error. Both sources resolve from the same uv.lock, | |
| # so the duplicate pins agree and drop out. | |
| - name: Generate requirements.txt | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| working-directory: project-repo | |
| run: | | |
| uv export \ | |
| --format requirements-txt \ | |
| --no-emit-project \ | |
| --no-dev \ | |
| --no-hashes \ | |
| --group ingestion \ | |
| --output-file requirements.txt | |
| # Builds deps.pex and source.pex and publishes them, then creates or | |
| # updates the branch deployment for this PR -- the action derives the | |
| # deployment name from the pull_request event, so there is no `deployment` | |
| # input to set here. | |
| # | |
| # On an ubuntu-24.04 runner deps.pex is built inside a python:3.13-slim | |
| # container so the wheels match the serverless base image; source.pex is | |
| # always built on the runner. That container only spins up when the | |
| # dependency hash changes. | |
| - name: Deploy to Dagster+ branch deployment | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v1.13.18 | |
| with: | |
| dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/${{ env.DAGSTER_CLOUD_FILE }}" | |
| build_output_dir: "$GITHUB_WORKSPACE/build" | |
| python_version: ${{ env.PYTHON_VERSION }} | |
| # Materializing the heartbeat is the run-time half of the check. The steps | |
| # above prove the agent can load the code location; the loader is not the | |
| # process that executes a step, and the two do not necessarily agree about | |
| # sys.path, so loading cleanly does not prove a step can import `db` or | |
| # `domain`. This does. The asset touches no database, no network and no | |
| # GCS, so a failure here is a packaging problem and nothing else -- which | |
| # is worth one Dagster+ run on a PR that already changed how the code | |
| # location is packaged. | |
| # | |
| # The deployment name has to be asked for rather than assumed: it is | |
| # derived from the branch, and the deploy action does not report it. This | |
| # is the same `ci branch-deployment` call the deploy makes internally, and | |
| # it is idempotent -- it returns the existing deployment for this PR. | |
| - name: Resolve branch deployment name | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| id: branch_deployment | |
| run: | | |
| name=$(uvx --from "dagster-cloud-cli==1.13.18" \ | |
| dagster-cloud ci branch-deployment project-repo) | |
| echo "name=$name" >> "$GITHUB_OUTPUT" | |
| echo "Branch deployment: $name" | |
| # Deliberately not the vendor's `launch_job` action, and the success | |
| # assertion is deliberately our own. | |
| # | |
| # `launch_job` documents `wait: true` as "the action will wait for the run | |
| # to finish and fail if the run fails". It does wait, but it cannot fail: | |
| # its run.sh captures the CLI output in a command substitution and never | |
| # checks the exit code, deciding success by whether it can regex a run id | |
| # out of the text. Underneath, `dagster-cloud job launch --wait` reports a | |
| # failed run with `ui.error(...)` -- and `ui.error` only *returns* an | |
| # exception rather than raising it, so the CLI exits 0 too. A failed | |
| # materialization would have produced a green check on both counts. | |
| # | |
| # So call the CLI directly and require the success line. Matching on output | |
| # is not lovely, but it is the only signal either layer actually emits. | |
| - name: Materialize ingestion_heartbeat | |
| if: steps.prerun.outputs.result == 'pex-deploy' | |
| env: | |
| DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| DEPLOYMENT: ${{ steps.branch_deployment.outputs.name }} | |
| run: | | |
| set -uo pipefail | |
| out=$(uvx --from "dagster-cloud-cli==1.13.18" \ | |
| dagster-cloud job launch \ | |
| --url "$DAGSTER_CLOUD_URL" \ | |
| --deployment "$DEPLOYMENT" \ | |
| --location ocotillo-automated-ingestion \ | |
| --job ingestion_heartbeat_check \ | |
| --wait --interval 10 2>&1 | tee /dev/stderr) | |
| case "$out" in | |
| *"finished successfully"*) ;; | |
| *) echo "::error title=Heartbeat failed::ingestion_heartbeat did not finish successfully on $DEPLOYMENT" | |
| exit 1 ;; | |
| esac | |
| # Fallback path, reached only when ENABLE_FAST_DEPLOYS is 'false' above. This | |
| # is the pre-PEX workflow unchanged, including the post-install hook in | |
| # dagster_cloud_post_install.sh that the PEX path replaces with its own | |
| # `uv pip install --no-deps .` of the repository. | |
| dagster-branch-docker-deploy: | |
| runs-on: ubuntu-latest | |
| needs: dagster-branch-deploy | |
| if: needs.dagster-branch-deploy.outputs.build_info | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| location: ${{ fromJSON(needs.dagster-branch-deploy.outputs.build_info) }} | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v7.0.1 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| - name: Install uv in container | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "latest" | |
| # Dagster+ builds from a requirements.txt. Hashes are kept here: this path | |
| # feeds the file straight to `pip install -r`, with nothing unhashed mixed | |
| # in. | |
| - name: Generate requirements.txt | |
| run: | | |
| uv export \ | |
| --format requirements-txt \ | |
| --no-emit-project \ | |
| --no-dev \ | |
| --group ingestion \ | |
| --output-file requirements.txt | |
| # checkout_repo is false because requirements.txt is generated above and | |
| # a second checkout would discard it. | |
| - name: Deploy to Dagster+ branch deployment | |
| uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v1.13.18 | |
| with: | |
| organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} | |
| dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| location: ${{ toJson(matrix.location) }} | |
| checkout_repo: false | |
| # The action defaults to python:3.8-slim, which cannot install a | |
| # lockfile resolved for requires-python >= 3.13 -- pip reports the | |
| # pins as having no matching distribution rather than as a version | |
| # conflict, which reads like a broken requirements file. | |
| base_image: python:${{ env.PYTHON_VERSION }}-slim |