From dcac3ec6cff9c9e5a157302cd39ba44773e462f1 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:38:40 +0530 Subject: [PATCH 1/5] feat: read-only tests and rollback on prod, two-stage gate on dev Prod had no tests at all: a release was marked last-known-good straight after deploy. Now main runs only the readonly allowlist against prod and promotes on pass or rolls back on failure; dev runs smoke, then functional only if smoke passed. api_base_url is explicit per branch -- run-smoke's api job otherwise falls back to the dev API. --- .github/workflows/deploy-Dataspace.yml | 71 ++++++++++++++++---------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 6d769c6a..132d87a2 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -182,21 +182,20 @@ jobs: # Production has no smoke-test job today, so its only quality gate is the # boot health check above. Mark this release good immediately so a FUTURE # deploy knows what to roll back to if it fails. - - name: Mark release as last-known-good (prod only) - if: github.ref_name == 'main' - uses: appleboy/ssh-action@v1.0.3 - with: - host: ${{ vars.EC2_HOST }} - username: ${{ secrets.EC2_USERNAME }} - key: ${{ secrets.EC2_PRIVATE_KEY }} - script: echo "${{ steps.meta.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good - smoke-tests: needs: build-and-deploy - if: github.ref_name == 'dev' uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + # main: only the readonly allowlist ever runs against prod. + # dev: stage 1 of 2 -- functional-tests runs only if this passes. + suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }} + # Explicit: run-smoke's api job falls back to the dev API when this is + # unset, which would test dev's backend while gating a prod deploy. + api_base_url: ${{ github.ref_name == 'main' && 'https://api.datakeep.civicdays.in' || 'https://dev.api.civicdataspace.in' }} secrets: - HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + # Literal, not a secret, so a missing value can't silently fall back to + # the dev site through the `&& ||` idiom. + HOME_URL_DEV: ${{ github.ref_name == 'main' && 'https://civicdataspace.in' || secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} @@ -206,18 +205,36 @@ jobs: # and the job fails its preflight. KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} - # dev only: smoke tests are the real quality gate here. Only once they pass - # does this release become the thing a future rollback would target. - promote-dev: - needs: [build-and-deploy, smoke-tests] - if: github.ref_name == 'dev' && needs.smoke-tests.result == 'success' + functional-tests: + needs: smoke-tests + # dev only, stage 2: runs only once smoke has passed. Functional tests + # write (datasets, use cases), so they never run against prod. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + api_base_url: https://dev.api.civicdataspace.in + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + # Only once the tests pass does this release become the thing a future + # rollback would target -- on prod too, which used to be marked + # last-known-good straight after deploy with no tests at all. + promote: + needs: [build-and-deploy, smoke-tests, functional-tests] + # Not success(): on main functional-tests is skipped by design, and a + # skipped need would skip this job too. Nothing failed + smoke passed. + if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} runs-on: ubuntu-latest - # vars.EC2_HOST is an environment-scoped variable (Settings -> Environments - # -> development), only visible to jobs that declare `environment:`. - # Without this, appleboy/ssh-action fails with "missing server host" - # (found via a live run - build-and-deploy/smoke-tests both succeeded but - # this job still failed, which is why `if:` here is already gated to dev). - environment: development + # vars.EC2_HOST is an environment-scoped variable, only visible to jobs + # that declare `environment:` -- without it appleboy/ssh-action fails + # with "missing server host". + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Mark this release as last-known-good uses: appleboy/ssh-action@v1.0.3 @@ -227,13 +244,13 @@ jobs: key: ${{ secrets.EC2_PRIVATE_KEY }} script: echo "${{ needs.build-and-deploy.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good - rollback-dev: - needs: [build-and-deploy, smoke-tests] - if: github.ref_name == 'dev' && needs.smoke-tests.result == 'failure' + rollback: + needs: [build-and-deploy, smoke-tests, functional-tests] + if: failure() && needs.build-and-deploy.result == 'success' runs-on: ubuntu-latest - # Same environment-scoped vars.EC2_HOST issue as promote-dev - see the + # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. - environment: development + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Revert to last known-good release uses: appleboy/ssh-action@v1.0.3 From 1d8bcd25717934c0c9381cbfaf9af7d5fb252ff5 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:38:40 +0530 Subject: [PATCH 2/5] feat: full-suite PR gate against dev for PRs into main --- .github/workflows/pr-gate.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/pr-gate.yml diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml new file mode 100644 index 00000000..d11ea871 --- /dev/null +++ b/.github/workflows/pr-gate.yml @@ -0,0 +1,27 @@ +# Full test suite against dev for every PR into main. Required by branch +# protection on main, so a PR merges (and prod deploys) only once it's green. +# dev already runs the PR's code (it was merged to dev first). +name: PR Gate + +on: + pull_request: + branches: [main] + +concurrency: + group: pr-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + full-suite: + name: Full Suite (dev) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + api_base_url: https://dev.api.civicdataspace.in + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} From 9f5379375d364bf71235e7a8d504234da8be16e8 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 11:35:12 +0530 Subject: [PATCH 3/5] feat: run provider functional report-only while it is flaky against dev Provider functional tests fail at shifting points against dev (autosave races, slow create flows), so gating on them would roll back dev deploys spuriously. The gating functional stage and the PR gate now skip provider; separate provider jobs run with only_provider and report, without being a dependency of rollback/promote or a required check. --- .github/workflows/deploy-Dataspace.yml | 21 +++++++++++++++++++++ .github/workflows/pr-gate.yml | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 132d87a2..3372b293 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -214,6 +214,9 @@ jobs: with: suite: functional api_base_url: https://dev.api.civicdataspace.in + # Provider functional is flaky against dev, so it runs report-only in + # provider-functional-report instead of gating the deploy. + skip_provider: true secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} @@ -222,6 +225,24 @@ jobs: TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + provider-functional-report: + name: Provider Functional (report-only) + needs: smoke-tests + # Report-only: promote and rollback do not list this job in needs, so its + # result never rolls back or blocks a dev deploy. Promote it to a gate + # once the provider suite is stable against dev. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # Only once the tests pass does this release become the thing a future # rollback would target -- on prod too, which used to be marked # last-known-good straight after deploy with no tests at all. diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index d11ea871..06ed599e 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -17,6 +17,9 @@ jobs: uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI with: suite: full + # Provider is flaky against dev: it reports in provider-report below, which + # branch protection does not require. + skip_provider: true api_base_url: https://dev.api.civicdataspace.in secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} @@ -25,3 +28,16 @@ jobs: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + provider-report: + name: Provider Full Suite (report-only) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} From 7403543334960c7b70a3139335128762cc29f468 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 12:25:58 +0530 Subject: [PATCH 4/5] fix: wait for every test before finalizing or rolling back Finalize and rollback did not list the report-only provider job in needs, so a deploy asked for finalize approval while provider tests were still running. Both now wait for it, and condition on the gating jobs' results explicitly instead of failure()/!failure(), which would count the report-only job's result. --- .github/workflows/deploy-Dataspace.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 3372b293..59e773d1 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -247,10 +247,15 @@ jobs: # rollback would target -- on prod too, which used to be marked # last-known-good straight after deploy with no tests at all. promote: - needs: [build-and-deploy, smoke-tests, functional-tests] - # Not success(): on main functional-tests is skipped by design, and a - # skipped need would skip this job too. Nothing failed + smoke passed. - if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} + # provider-functional-report is listed so this waits for every test to + # finish, but its result is deliberately not checked (report-only). + needs: [build-and-deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not success()/!failure(): on main the functional + # jobs are skipped by design, and the provider job must not block. + if: >- + ${{ !cancelled() && needs.build-and-deploy.result == 'success' && + needs.smoke-tests.result == 'success' && + (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }} runs-on: ubuntu-latest # vars.EC2_HOST is an environment-scoped variable, only visible to jobs # that declare `environment:` -- without it appleboy/ssh-action fails @@ -266,8 +271,13 @@ jobs: script: echo "${{ needs.build-and-deploy.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good rollback: - needs: [build-and-deploy, smoke-tests, functional-tests] - if: failure() && needs.build-and-deploy.result == 'success' + # Waits for the report-only provider job too, without checking its result. + needs: [build-and-deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not failure(): failure() would also count the + # report-only provider job. + if: >- + ${{ !cancelled() && needs.build-and-deploy.result == 'success' && + (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }} runs-on: ubuntu-latest # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. From 1348e396d81bd212a6bdfe2068bfda5ddf177bb9 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 14:25:18 +0530 Subject: [PATCH 5/5] fix: run prod finalize/rollback in production-ops, without a second approval production requires a reviewer, and every job referencing it asked again: approve the deploy, then approve finalize or rollback -- so a failing prod deploy stayed live until someone clicked. production-ops holds the same host and SSH secrets with no reviewer and a main-only branch policy; the human approval stays on the deploy job. --- .github/workflows/deploy-Dataspace.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 59e773d1..f90c4924 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -260,7 +260,9 @@ jobs: # vars.EC2_HOST is an environment-scoped variable, only visible to jobs # that declare `environment:` -- without it appleboy/ssh-action fails # with "missing server host". - environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} + # production-ops: same prod host/SSH secrets, no required reviewer, main-only. + # The human approval is on the deploy job; finalize/rollback must not wait. + environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} steps: - name: Mark this release as last-known-good uses: appleboy/ssh-action@v1.0.3 @@ -281,7 +283,9 @@ jobs: runs-on: ubuntu-latest # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. - environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} + # production-ops: same prod host/SSH secrets, no required reviewer, main-only. + # The human approval is on the deploy job; finalize/rollback must not wait. + environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} steps: - name: Revert to last known-good release uses: appleboy/ssh-action@v1.0.3