Merge pull request #860 from DataIntegrationGroup/feat/san-acacia-wee… #23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Deploys the `ocotillo-automated-ingestion` code location to the Dagster+ prod | |
| # deployment. | |
| # | |
| # "prod" here names the Dagster+ deployment, not the API's `production` branch. | |
| # It tracks `staging`, which is this repository's integration branch and the | |
| # only place `automated_ingestion/` currently exists -- `production` is still on | |
| # 1.2.2 and would never fire. This mirrors how `ocotillo-api-staging` follows | |
| # staging, and keeps the code location current with the work. | |
| # | |
| # Move this to `production` in the same change that first points the pipeline at | |
| # the production database. Until then a deploy here publishes code, not data: | |
| # the location has no database or vendor credentials, so the worst it can do is | |
| # fail to materialize. | |
| # | |
| # Path-filtered so an ordinary API change does not spend a Dagster+ build. The | |
| # filter includes pyproject.toml and uv.lock because the location's dependency | |
| # set is exported from them, so a lockfile bump changes the built image even | |
| # when no ingestion source file does. | |
| name: CD (Dagster+ prod) | |
| on: | |
| push: | |
| branches: [staging] | |
| paths: | |
| - "automated_ingestion/**" | |
| # The code location imports db/ models and domain/ rules in-process, | |
| # so a change to either alters what this image runs even when no | |
| # ingestion file moves. Without these, a domain fix merged to | |
| # production would leave the pipeline running the old rule against | |
| # the live database. The cost is that ordinary API changes to these | |
| # directories also trigger a build; a stale code location is worse. | |
| - "db/**" | |
| - "domain/**" | |
| - "dagster_cloud.yaml" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| - ".github/workflows/CD_dagster_prod.yml" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: dagster-prod-deploy | |
| cancel-in-progress: false | |
| jobs: | |
| dagster-prod-deploy: | |
| runs-on: ubuntu-latest | |
| # Deliberately not bound to the `production` GitHub environment. This job | |
| # reads only repository-level DAGSTER_CLOUD_API_TOKEN and | |
| # DAGSTER_CLOUD_ORGANIZATION_ID -- none of that environment's secrets -- and | |
| # it runs on every push to `staging` that touches the code location. Binding | |
| # it would put an approval gate on routine merges once `production` requires | |
| # reviewers, which is a gate on the wrong thing: this publishes code, not | |
| # data. | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v7.0.1 | |
| # parse_workspace performs its own `actions/checkout`, which cleans the | |
| # working tree. It has to run *before* requirements.txt is generated, or | |
| # the generated file is deleted before the deploy step can use it. | |
| - name: Parse dagster_cloud.yaml | |
| id: parse | |
| uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 | |
| with: | |
| dagster_cloud_file: dagster_cloud.yaml | |
| - name: Install uv in container | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "latest" | |
| # Dagster+ builds from a requirements.txt, which the repo does not keep | |
| # under version control. `--group ingestion` adds dagster and dlt on top | |
| # of the runtime dependencies; the runtime ones are needed too, because | |
| # the loader imports `db/` and `domain/`. | |
| - name: Generate requirements.txt | |
| run: | | |
| uv export \ | |
| --format requirements-txt \ | |
| --no-emit-project \ | |
| --no-dev \ | |
| --group ingestion \ | |
| --output-file requirements.txt | |
| # checkout_repo is false because requirements.txt is generated above and | |
| # a second checkout would discard it. | |
| - name: Deploy to Dagster+ prod | |
| uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v1.13.18 | |
| with: | |
| organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} | |
| dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} | |
| location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} | |
| checkout_repo: false | |
| # The action defaults to python:3.8-slim, which cannot install a | |
| # lockfile resolved for requires-python >= 3.13 -- pip reports the | |
| # pins as having no matching distribution rather than as a version | |
| # conflict, which reads like a broken requirements file. | |
| base_image: python:3.13-slim |