Merge pull request #869 from DataIntegrationGroup/fix/seed-reference-… #449
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CD (Staging) | |
| on: | |
| push: | |
| branches: [staging] | |
| permissions: | |
| contents: write | |
| jobs: | |
| staging-deploy: | |
| runs-on: ubuntu-latest | |
| environment: staging | |
| steps: | |
| - name: Check out source repository | |
| uses: actions/checkout@v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install uv in container | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "latest" | |
| - name: Generate requirements.txt | |
| run: | | |
| uv export \ | |
| --format requirements-txt \ | |
| --no-emit-project \ | |
| --no-dev \ | |
| --output-file requirements.txt | |
| - name: Authenticate to Google Cloud | |
| uses: 'google-github-actions/auth@v3' | |
| with: | |
| credentials_json: ${{ secrets.CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY }} | |
| # Application credentials live in Google Secret Manager, not GitHub | |
| # secrets. The deploy service account needs | |
| # roles/secretmanager.secretAccessor on these secrets. Every secret | |
| # listed here must already exist in the target project or the deploy | |
| # fails -- see docs/internal-ogc-desktop-gis.md for the placeholder | |
| # value to seed internal-ogc-api-keys with. | |
| - name: Fetch application secrets from Secret Manager | |
| id: app-secrets | |
| uses: 'google-github-actions/get-secretmanager-secrets@v3' | |
| with: | |
| secrets: |- | |
| jira_email:${{ vars.GCP_PROJECT_ID }}/jira-email | |
| jira_api_token:${{ vars.GCP_PROJECT_ID }}/jira-api-token | |
| slack_feedback_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-feedback-webhook-url | |
| slack_edits_webhook_url:${{ vars.GCP_PROJECT_ID }}/slack-edits-webhook-url | |
| internal_ogc_api_keys:${{ vars.GCP_PROJECT_ID }}/internal-ogc-api-keys | |
| - name: Run Alembic migrations on staging database | |
| env: | |
| DB_DRIVER: "cloudsql" | |
| CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" | |
| CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" | |
| CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" | |
| CLOUD_SQL_IAM_AUTH: true | |
| run: | | |
| uv run --no-dev alembic upgrade head | |
| # Data migrations are deliberately not applied here -- they change content | |
| # rather than structure, are often irreversible, and applying one is a | |
| # decision made in the Data Migrations workflow. This only reports, so a | |
| # merged migration cannot sit unnoticed. It never fails the deploy: the | |
| # deploy worked, and a pipeline that goes red for something else is a | |
| # pipeline people learn to ignore. | |
| - name: Report pending data migrations | |
| env: | |
| DB_DRIVER: "cloudsql" | |
| CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" | |
| CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" | |
| CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" | |
| CLOUD_SQL_IAM_AUTH: true | |
| run: | | |
| uv run --no-dev python -m scripts.report_pending_data_migrations | |
| - name: Ensure envsubst is available | |
| run: | | |
| if ! command -v envsubst >/dev/null 2>&1; then | |
| sudo apt-get update | |
| sudo apt-get install -y gettext-base | |
| fi | |
| - name: Render App Engine configs | |
| env: | |
| APP_VERSION: "${{ github.ref_name }}-${{ github.sha }}" | |
| ENVIRONMENT: "staging" | |
| CLOUD_SQL_INSTANCE_NAME: "${{ secrets.CLOUD_SQL_INSTANCE_NAME }}" | |
| CLOUD_SQL_DATABASE: "${{ vars.CLOUD_SQL_DATABASE }}" | |
| CLOUD_SQL_USER: "${{ secrets.CLOUD_SQL_USER }}" | |
| PYGEOAPI_POSTGRES_DB: "${{ vars.CLOUD_SQL_DATABASE }}" | |
| PYGEOAPI_POSTGRES_USER: "${{ secrets.PYGEOAPI_POSTGRES_USER }}" | |
| PYGEOAPI_POSTGRES_HOST: "${{ vars.PYGEOAPI_POSTGRES_HOST || '127.0.0.1' }}" | |
| PYGEOAPI_POSTGRES_PORT: "${{ vars.PYGEOAPI_POSTGRES_PORT || '5432' }}" | |
| PYGEOAPI_POSTGRES_PASSWORD: "${{ secrets.PYGEOAPI_POSTGRES_PASSWORD }}" | |
| PYGEOAPI_SERVER_URL: "${{ vars.PYGEOAPI_SERVER_URL }}" | |
| INTERNAL_OGC_API_KEYS: "${{ steps.app-secrets.outputs.internal_ogc_api_keys }}" | |
| CLOUD_SQL_IAM_AUTH: "true" | |
| GCS_SERVICE_ACCOUNT_KEY: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" | |
| GCS_BUCKET_NAME: "${{ vars.GCS_BUCKET_NAME }}" | |
| AUTHENTIK_URL: "${{ vars.AUTHENTIK_URL }}" | |
| AUTHENTIK_CLIENT_ID: "${{ vars.AUTHENTIK_CLIENT_ID }}" | |
| AUTHENTIK_AUTHORIZE_URL: "${{ vars.AUTHENTIK_AUTHORIZE_URL }}" | |
| AUTHENTIK_TOKEN_URL: "${{ vars.AUTHENTIK_TOKEN_URL }}" | |
| APITALLY_CLIENT_ID: "${{ vars.APITALLY_CLIENT_ID }}" | |
| JIRA_BASE_URL: "${{ vars.JIRA_BASE_URL || 'https://nmbgmr.atlassian.net' }}" | |
| JIRA_EMAIL: "${{ steps.app-secrets.outputs.jira_email }}" | |
| JIRA_API_TOKEN: "${{ steps.app-secrets.outputs.jira_api_token }}" | |
| JIRA_DEFAULT_PROJECT: "${{ vars.JIRA_DEFAULT_PROJECT || 'BDMS' }}" | |
| SLACK_FEEDBACK_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_feedback_webhook_url }}" | |
| SLACK_EDITS_WEBHOOK_URL: "${{ steps.app-secrets.outputs.slack_edits_webhook_url }}" | |
| OCOTILLO_UI_BASE_URL: "${{ vars.OCOTILLO_UI_BASE_URL || 'https://ocotillo-staging.newmexicowaterdata.org' }}" | |
| run: | | |
| export MAX_INSTANCES="10" | |
| export SERVICE_NAME="ocotillo-api-staging" | |
| export ENTRYPOINT="gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app" | |
| export MIN_INSTANCES="0" | |
| envsubst < .github/app.template.yaml > app.yaml | |
| - name: Deploy to Google Cloud | |
| run: | | |
| gcloud app deploy \ | |
| app.yaml \ | |
| --quiet \ | |
| --project ${{ vars.GCP_PROJECT_ID }} | |
| - name: Clean up oldest versions | |
| run: | | |
| SERVICE="ocotillo-api-staging" | |
| VERSIONS_JSON="$(gcloud app versions list --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --format=json --sort-by="version.createTime" 2>/dev/null || printf '[]')" | |
| export VERSIONS_JSON | |
| DELETE_VERSION="$(python - <<'PY' | |
| import json | |
| import os | |
| versions = json.loads(os.environ.get("VERSIONS_JSON", "[]") or "[]") | |
| if len(versions) <= 1: | |
| print("") | |
| raise SystemExit(0) | |
| def traffic_split(version): | |
| for key in ("traffic_split", "trafficSplit"): | |
| value = version.get(key) | |
| if value is not None: | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return 0.0 | |
| return 0.0 | |
| for version in versions: | |
| if traffic_split(version) == 0.0: | |
| print(version.get("id", "")) | |
| break | |
| else: | |
| print("") | |
| PY | |
| )" | |
| if [ -n "$DELETE_VERSION" ]; then | |
| echo "Deleting old non-serving version for $SERVICE: $DELETE_VERSION" | |
| gcloud app versions delete "$DELETE_VERSION" --service="$SERVICE" --project=${{ vars.GCP_PROJECT_ID }} --quiet | |
| else | |
| echo "No old non-serving versions to delete for $SERVICE" | |
| fi | |
| - name: Remove rendered configs | |
| run: | | |
| rm app.yaml | |
| # Use the workflow actor's username as git user name | |
| - name: Set up git user | |
| run: | | |
| git config --global user.name "${{ github.actor }}" | |
| git config --global user.email "${{ github.actor }}@users.noreply.github.com" | |
| # ":" are not allowed in git tags, so replace with "-" | |
| - name: Tag commit | |
| run: | | |
| git tag -a "staging-deploy-$(date -u +%Y-%m-%d)T$(date -u +%H-%M-%S%z)" -m "staging gcloud deployment: $(date -u +%Y-%m-%d)T$(date -u +%H:%M:%S%z)" | |
| git push origin --tags |