From 08e66ce586048d06f8fa1a1b6bd980a2022aed02 Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:38:40 +0530 Subject: [PATCH 1/8] feat: read-only tests and rollback on prod, two-stage gate on dev Prod had no tests at all: a release was marked last-known-good straight after deploy. Now main runs only the readonly allowlist against prod and promotes on pass or rolls back on failure; dev runs smoke, then functional only if smoke passed. api_base_url is explicit per branch -- run-smoke's api job otherwise falls back to the dev API. --- .github/workflows/deploy-Dataspace.yml | 71 ++++++++++++++++---------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 6d769c6a..132d87a2 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -182,21 +182,20 @@ jobs: # Production has no smoke-test job today, so its only quality gate is the # boot health check above. Mark this release good immediately so a FUTURE # deploy knows what to roll back to if it fails. - - name: Mark release as last-known-good (prod only) - if: github.ref_name == 'main' - uses: appleboy/ssh-action@v1.0.3 - with: - host: ${{ vars.EC2_HOST }} - username: ${{ secrets.EC2_USERNAME }} - key: ${{ secrets.EC2_PRIVATE_KEY }} - script: echo "${{ steps.meta.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good - smoke-tests: needs: build-and-deploy - if: github.ref_name == 'dev' uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + # main: only the readonly allowlist ever runs against prod. + # dev: stage 1 of 2 -- functional-tests runs only if this passes. + suite: ${{ github.ref_name == 'main' && 'readonly' || 'smoke' }} + # Explicit: run-smoke's api job falls back to the dev API when this is + # unset, which would test dev's backend while gating a prod deploy. + api_base_url: ${{ github.ref_name == 'main' && 'https://api.datakeep.civicdays.in' || 'https://dev.api.civicdataspace.in' }} secrets: - HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + # Literal, not a secret, so a missing value can't silently fall back to + # the dev site through the `&& ||` idiom. + HOME_URL_DEV: ${{ github.ref_name == 'main' && 'https://civicdataspace.in' || secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} @@ -206,18 +205,36 @@ jobs: # and the job fails its preflight. KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} - # dev only: smoke tests are the real quality gate here. Only once they pass - # does this release become the thing a future rollback would target. - promote-dev: - needs: [build-and-deploy, smoke-tests] - if: github.ref_name == 'dev' && needs.smoke-tests.result == 'success' + functional-tests: + needs: smoke-tests + # dev only, stage 2: runs only once smoke has passed. Functional tests + # write (datasets, use cases), so they never run against prod. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + api_base_url: https://dev.api.civicdataspace.in + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + # Only once the tests pass does this release become the thing a future + # rollback would target -- on prod too, which used to be marked + # last-known-good straight after deploy with no tests at all. + promote: + needs: [build-and-deploy, smoke-tests, functional-tests] + # Not success(): on main functional-tests is skipped by design, and a + # skipped need would skip this job too. Nothing failed + smoke passed. + if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} runs-on: ubuntu-latest - # vars.EC2_HOST is an environment-scoped variable (Settings -> Environments - # -> development), only visible to jobs that declare `environment:`. - # Without this, appleboy/ssh-action fails with "missing server host" - # (found via a live run - build-and-deploy/smoke-tests both succeeded but - # this job still failed, which is why `if:` here is already gated to dev). - environment: development + # vars.EC2_HOST is an environment-scoped variable, only visible to jobs + # that declare `environment:` -- without it appleboy/ssh-action fails + # with "missing server host". + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Mark this release as last-known-good uses: appleboy/ssh-action@v1.0.3 @@ -227,13 +244,13 @@ jobs: key: ${{ secrets.EC2_PRIVATE_KEY }} script: echo "${{ needs.build-and-deploy.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good - rollback-dev: - needs: [build-and-deploy, smoke-tests] - if: github.ref_name == 'dev' && needs.smoke-tests.result == 'failure' + rollback: + needs: [build-and-deploy, smoke-tests, functional-tests] + if: failure() && needs.build-and-deploy.result == 'success' runs-on: ubuntu-latest - # Same environment-scoped vars.EC2_HOST issue as promote-dev - see the + # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. - environment: development + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Revert to last known-good release uses: appleboy/ssh-action@v1.0.3 From f22ba8381ebf7804af7f71e7ff65158ae2c3aefe Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 11 Sep 2026 23:38:40 +0530 Subject: [PATCH 2/8] feat: full-suite PR gate against dev for PRs into main --- .github/workflows/pr-gate.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/pr-gate.yml diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml new file mode 100644 index 00000000..d11ea871 --- /dev/null +++ b/.github/workflows/pr-gate.yml @@ -0,0 +1,27 @@ +# Full test suite against dev for every PR into main. Required by branch +# protection on main, so a PR merges (and prod deploys) only once it's green. +# dev already runs the PR's code (it was merged to dev first). +name: PR Gate + +on: + pull_request: + branches: [main] + +concurrency: + group: pr-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + full-suite: + name: Full Suite (dev) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + api_base_url: https://dev.api.civicdataspace.in + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} From a63f78be00d0b0a5d2efef5d32244d6924fbde51 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 11:35:12 +0530 Subject: [PATCH 3/8] feat: run provider functional report-only while it is flaky against dev Provider functional tests fail at shifting points against dev (autosave races, slow create flows), so gating on them would roll back dev deploys spuriously. The gating functional stage and the PR gate now skip provider; separate provider jobs run with only_provider and report, without being a dependency of rollback/promote or a required check. --- .github/workflows/deploy-Dataspace.yml | 21 +++++++++++++++++++++ .github/workflows/pr-gate.yml | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 132d87a2..3372b293 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -214,6 +214,9 @@ jobs: with: suite: functional api_base_url: https://dev.api.civicdataspace.in + # Provider functional is flaky against dev, so it runs report-only in + # provider-functional-report instead of gating the deploy. + skip_provider: true secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} @@ -222,6 +225,24 @@ jobs: TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + provider-functional-report: + name: Provider Functional (report-only) + needs: smoke-tests + # Report-only: promote and rollback do not list this job in needs, so its + # result never rolls back or blocks a dev deploy. Promote it to a gate + # once the provider suite is stable against dev. + if: github.ref_name != 'main' + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: functional + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # Only once the tests pass does this release become the thing a future # rollback would target -- on prod too, which used to be marked # last-known-good straight after deploy with no tests at all. diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index d11ea871..06ed599e 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -17,6 +17,9 @@ jobs: uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI with: suite: full + # Provider is flaky against dev: it reports in provider-report below, which + # branch protection does not require. + skip_provider: true api_base_url: https://dev.api.civicdataspace.in secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} @@ -25,3 +28,16 @@ jobs: TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} + + provider-report: + name: Provider Full Suite (report-only) + uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI + with: + suite: full + only_provider: true + secrets: + HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} + TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} + TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} + TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} + TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} From ccae78aa542ec56262057ffe7b370681040b862d Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 12:25:58 +0530 Subject: [PATCH 4/8] fix: wait for every test before finalizing or rolling back Finalize and rollback did not list the report-only provider job in needs, so a deploy asked for finalize approval while provider tests were still running. Both now wait for it, and condition on the gating jobs' results explicitly instead of failure()/!failure(), which would count the report-only job's result. --- .github/workflows/deploy-Dataspace.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 3372b293..59e773d1 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -247,10 +247,15 @@ jobs: # rollback would target -- on prod too, which used to be marked # last-known-good straight after deploy with no tests at all. promote: - needs: [build-and-deploy, smoke-tests, functional-tests] - # Not success(): on main functional-tests is skipped by design, and a - # skipped need would skip this job too. Nothing failed + smoke passed. - if: ${{ !failure() && !cancelled() && needs.smoke-tests.result == 'success' }} + # provider-functional-report is listed so this waits for every test to + # finish, but its result is deliberately not checked (report-only). + needs: [build-and-deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not success()/!failure(): on main the functional + # jobs are skipped by design, and the provider job must not block. + if: >- + ${{ !cancelled() && needs.build-and-deploy.result == 'success' && + needs.smoke-tests.result == 'success' && + (needs.functional-tests.result == 'success' || needs.functional-tests.result == 'skipped') }} runs-on: ubuntu-latest # vars.EC2_HOST is an environment-scoped variable, only visible to jobs # that declare `environment:` -- without it appleboy/ssh-action fails @@ -266,8 +271,13 @@ jobs: script: echo "${{ needs.build-and-deploy.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good rollback: - needs: [build-and-deploy, smoke-tests, functional-tests] - if: failure() && needs.build-and-deploy.result == 'success' + # Waits for the report-only provider job too, without checking its result. + needs: [build-and-deploy, smoke-tests, functional-tests, provider-functional-report] + # Explicit gating results, not failure(): failure() would also count the + # report-only provider job. + if: >- + ${{ !cancelled() && needs.build-and-deploy.result == 'success' && + (needs.smoke-tests.result == 'failure' || needs.functional-tests.result == 'failure') }} runs-on: ubuntu-latest # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. From c05c0c7ef73cee5d042b4bc94c3653fd40370645 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 14:25:18 +0530 Subject: [PATCH 5/8] fix: run prod finalize/rollback in production-ops, without a second approval production requires a reviewer, and every job referencing it asked again: approve the deploy, then approve finalize or rollback -- so a failing prod deploy stayed live until someone clicked. production-ops holds the same host and SSH secrets with no reviewer and a main-only branch policy; the human approval stays on the deploy job. --- .github/workflows/deploy-Dataspace.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 59e773d1..f90c4924 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -260,7 +260,9 @@ jobs: # vars.EC2_HOST is an environment-scoped variable, only visible to jobs # that declare `environment:` -- without it appleboy/ssh-action fails # with "missing server host". - environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} + # production-ops: same prod host/SSH secrets, no required reviewer, main-only. + # The human approval is on the deploy job; finalize/rollback must not wait. + environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} steps: - name: Mark this release as last-known-good uses: appleboy/ssh-action@v1.0.3 @@ -281,7 +283,9 @@ jobs: runs-on: ubuntu-latest # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. - environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} + # production-ops: same prod host/SSH secrets, no required reviewer, main-only. + # The human approval is on the deploy job; finalize/rollback must not wait. + environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} steps: - name: Revert to last known-good release uses: appleboy/ssh-action@v1.0.3 From e392f27e36fdae44407a8a0b7c3d609400417c0c Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 14:38:58 +0530 Subject: [PATCH 6/8] Revert "fix: run prod finalize/rollback in production-ops, without a second approval" This reverts commit c05c0c7ef73cee5d042b4bc94c3653fd40370645. --- .github/workflows/deploy-Dataspace.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index f90c4924..59e773d1 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -260,9 +260,7 @@ jobs: # vars.EC2_HOST is an environment-scoped variable, only visible to jobs # that declare `environment:` -- without it appleboy/ssh-action fails # with "missing server host". - # production-ops: same prod host/SSH secrets, no required reviewer, main-only. - # The human approval is on the deploy job; finalize/rollback must not wait. - environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Mark this release as last-known-good uses: appleboy/ssh-action@v1.0.3 @@ -283,9 +281,7 @@ jobs: runs-on: ubuntu-latest # Same environment-scoped vars.EC2_HOST issue as promote - see the # comment there. - # production-ops: same prod host/SSH secrets, no required reviewer, main-only. - # The human approval is on the deploy job; finalize/rollback must not wait. - environment: ${{ github.ref_name == 'main' && 'production-ops' || 'development' }} + environment: ${{ github.ref_name == 'main' && 'production' || 'development' }} steps: - name: Revert to last known-good release uses: appleboy/ssh-action@v1.0.3 From fa448fd5168b8d528ecb48cf24605f570a582e73 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 15:18:45 +0530 Subject: [PATCH 7/8] fix: pass KEYCLOAK_CLIENT_SECRET to the report-only provider job org_add_permission needs a Keycloak token to check canAdd. A called workflow only sees secrets its caller passes, and the report-only job passed none, so every org-create test skipped on both workers -- the provider report looked green while never exercising org flows. --- .github/workflows/deploy-Dataspace.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-Dataspace.yml b/.github/workflows/deploy-Dataspace.yml index 59e773d1..d58d89b6 100644 --- a/.github/workflows/deploy-Dataspace.yml +++ b/.github/workflows/deploy-Dataspace.yml @@ -236,12 +236,16 @@ jobs: with: suite: functional only_provider: true + api_base_url: https://dev.api.civicdataspace.in secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # org_add_permission needs a token to check canAdd; without this every + # org-create test skips on both workers and the report is hollow. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }} # Only once the tests pass does this release become the thing a future # rollback would target -- on prod too, which used to be marked From aa5a6303d7a5218de0db27f7557ce14a03a40230 Mon Sep 17 00:00:00 2001 From: Saqib Date: Tue, 15 Sep 2026 15:18:45 +0530 Subject: [PATCH 8/8] fix: pass KEYCLOAK_CLIENT_SECRET to the report-only provider-report job Same gap as the deploy workflow's report job: without it every org-create test skips on both workers. --- .github/workflows/pr-gate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-gate.yml b/.github/workflows/pr-gate.yml index 06ed599e..f68c08a0 100644 --- a/.github/workflows/pr-gate.yml +++ b/.github/workflows/pr-gate.yml @@ -35,9 +35,13 @@ jobs: with: suite: full only_provider: true + api_base_url: https://dev.api.civicdataspace.in secrets: HOME_URL_DEV: ${{ secrets.HOME_URL_DEV }} TEST_EMAIL_1: ${{ secrets.TEST_EMAIL_1 }} TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }} TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }} TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }} + # org_add_permission needs a token to check canAdd; without this every + # org-create test skips on both workers and the report is hollow. + KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}