feat(ci): monitors marker + three-way Better Stack heartbeat split

.github/workflows/_scheduled-test-hourly.yml

@@ -22,6 +22,12 @@
         required: true
       BETTERSTACK_HEARTBEAT_URL_STAGING:
         required: true
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING:
+        # Optional until the HE-TME Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING:
+        # Optional until the Platform API Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION:
         required: true
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION:
@@ -30,6 +36,12 @@
         required: true
       BETTERSTACK_HEARTBEAT_URL_PRODUCTION:
         required: true
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION:
+        # Optional until the HE-TME Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION:
+        # Optional until the Platform API Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
       SENTRY_DSN:
         required: true
 
@@ -90,18 +102,77 @@
           echo "$GCP_CREDENTIALS" | base64 -d > credentials.json
           echo "GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/credentials.json" >> $GITHUB_ENV
 
-      - name: Test / scheduled
+      # Tests are split into three runs so failures can be routed to the correct Better Stack monitor:
+      #   - SDK-layer failures (token management, service wiring): no monitors marker → SDK monitor
+      #   - Platform API failures (auth, listing, connectivity): monitors_platform_api → Platform API monitor
+      #   - Application failures (HE-TME, test-app processing): monitors("he-tme"/"test-app") → HE-TME monitor
+      # Tests declare which system they monitor via @pytest.mark.monitors("he-tme") /
+      # @pytest.mark.monitors_platform_api; tests with neither marker are SDK health checks.
+      - name: Test / scheduled / sdk
+        id: test_sdk
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+        shell: bash
+        run: |
+          # set +e so a test failure does not abort the step — we capture the exit code
+          # manually and send it to Better Stack regardless of outcome.
+          set +e
+          XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
+            -m "(scheduled or scheduled_only) and not monitors and not monitors_platform_api and not stress_only" \
+            --junit-xml=reports/junit_sdk.xml
+          echo "exit_code=$?" >> $GITHUB_OUTPUT
+
+      - name: Test / scheduled / platform-api
+        id: test_platform_api
         env:
-          BETTERSTACK_HEARTBEAT_URL: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}"
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
         shell: bash
         run: |
+          # set +e so a test failure does not abort the step — we capture the exit code
+          # manually and send it to Better Stack regardless of outcome.
           set +e
-          make test_scheduled
-          EXIT_CODE=$?
+          XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
+            -m "(scheduled or scheduled_only) and monitors_platform_api and not stress_only" \
+            --junit-xml=reports/junit_platform_api.xml
+          echo "exit_code=$?" >> $GITHUB_OUTPUT
+
+      - name: Test / scheduled / he-tme
+        id: test_he_tme
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+        shell: bash
+        run: |
+          # set +e so a test failure does not abort the step — we capture the exit code
+          # manually and send it to Better Stack regardless of outcome.
+          set +e
+          XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
+            -m "(scheduled or scheduled_only) and monitors and not monitors_platform_api and not stress_only" \
+            --junit-xml=reports/junit_he_tme.xml
+          echo "exit_code=$?" >> $GITHUB_OUTPUT
+
+      - name: Collect combined exit code and publish summary
+        id: collect
+        shell: bash
+        run: |
+          # Default to 1 (failure) if a step never wrote its output — e.g. a setup step
+          # errored before tests ran. Prevents sending a false "healthy" heartbeat.
+          SDK_EXIT=${{ steps.test_sdk.outputs.exit_code || '1' }}
+          PLATFORM_API_EXIT=${{ steps.test_platform_api.outputs.exit_code || '1' }}
+          HE_TME_EXIT=${{ steps.test_he_tme.outputs.exit_code || '1' }}
+          # Combined exit code: non-zero if any run failed
+          if [ "$SDK_EXIT" != "0" ] || [ "$PLATFORM_API_EXIT" != "0" ] || [ "$HE_TME_EXIT" != "0" ]; then
+            COMBINED_EXIT=1
+          else
+            COMBINED_EXIT=0
+          fi
+          echo "sdk_exit=${SDK_EXIT}" >> $GITHUB_OUTPUT
+          echo "platform_api_exit=${PLATFORM_API_EXIT}" >> $GITHUB_OUTPUT
+          echo "he_tme_exit=${HE_TME_EXIT}" >> $GITHUB_OUTPUT
+          echo "combined_exit=${COMBINED_EXIT}" >> $GITHUB_OUTPUT
+
           # Show test execution in GitHub Job summary
           found_files=0
-          for file in reports/pytest_*.md; do
+          for file in reports/pytest_*.md reports/pytest.md; do
             if [ -f "$file" ]; then
               cat "$file" >> $GITHUB_STEP_SUMMARY
               echo "" >> $GITHUB_STEP_SUMMARY
@@ -121,63 +192,182 @@
             echo "" >> $GITHUB_STEP_SUMMARY
           fi
 
-          # Send heartbeat to Sentry, defining the schedule on the fly
-          SENTRY_EXIT_CODE=$(sentry-cli monitors run -e CI --schedule "0 * * * *" --check-in-margin 30 --max-runtime 1 scheduled-testing-${{ inputs.platform_environment }}-hourly --timezone "Europe/Berlin" -- sh -c "exit $EXIT_CODE")
-
-          # Provide heartbeat to BetterStack for monitoring/alerting if heartbeat url is configured as secret
-          if [ -n "$BETTERSTACK_HEARTBEAT_URL" ]; then
-            BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
-              --arg github_workflow "${{ github.workflow }}" \
-              --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
-              --arg github_run_id "${{ github.run_id }}" \
-              --arg github_job "${{ github.job }}" \
-              --arg github_sha "${{ github.sha }}" \
-              --arg github_actor "${{ github.actor }}" \
-              --arg github_repository "${{ github.repository }}" \
-              --arg github_ref "${{ github.ref }}" \
-              --arg job_status "${{ job.status }}" \
-              --arg github_event_name "${{ github.event_name }}" \
-              --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
-              '{
-                github: {
-                  workflow: $github_workflow,
-                  run_url: $github_run_url,
-                  run_id: $github_run_id,
-                  job: $github_job,
-                  sha: $github_sha,
-                  actor: $github_actor,
-                  repository: $github_repository,
-                  ref: $github_ref,
-                  event_name: $github_event_name
-                },
-                job: {
-                  status: $job_status,
-                },
-                timestamp: $timestamp,
-              }'
-            )
-            curl \
-              --fail-with-body \
-              --silent \
-              --request POST \
-              --header "Content-Type: application/json" \
-              --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
-              "${BETTERSTACK_HEARTBEAT_URL}/${EXIT_CODE}"
-            echo "INFO: Sent heartbeat to betterstack with exit code '${EXIT_CODE}'"
-          else
-            echo "WARNING: No BetterStack heartbeat URL configured, skipped heartbeat notification."
+      - name: Heartbeat / Sentry
+        if: always()
+        shell: bash
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          COMBINED_EXIT: ${{ steps.collect.outputs.combined_exit }}
+        run: |
+          sentry-cli monitors run -e CI --schedule "0 * * * *" --check-in-margin 30 --max-runtime 1 \
+            scheduled-testing-${{ inputs.platform_environment }}-hourly \
+            --timezone "Europe/Berlin" -- sh -c "exit ${COMBINED_EXIT}" || true
+
+      - name: Heartbeat / BetterStack / SDK
+        if: always()
+        shell: bash
+        env:
+          BETTERSTACK_HEARTBEAT_URL: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}"
+          SDK_EXIT: ${{ steps.collect.outputs.sdk_exit }}
+        run: |
+          if [ -z "$BETTERSTACK_HEARTBEAT_URL" ]; then
+            echo "WARNING: No BetterStack SDK heartbeat URL configured, skipped."
+            exit 0
+          fi
+          BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
+            --arg github_workflow "${{ github.workflow }}" \
+            --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg github_run_id "${{ github.run_id }}" \
+            --arg github_job "${{ github.job }}" \
+            --arg github_sha "${{ github.sha }}" \
+            --arg github_actor "${{ github.actor }}" \
+            --arg github_repository "${{ github.repository }}" \
+            --arg github_ref "${{ github.ref }}" \
+            --arg job_status "${{ job.status }}" \
+            --arg github_event_name "${{ github.event_name }}" \
+            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            '{
+              github: {
+                workflow: $github_workflow,
+                run_url: $github_run_url,
+                run_id: $github_run_id,
+                job: $github_job,
+                sha: $github_sha,
+                actor: $github_actor,
+                repository: $github_repository,
+                ref: $github_ref,
+                event_name: $github_event_name
+              },
+              job: {
+                status: $job_status,
+              },
+              timestamp: $timestamp,
+            }'
+          )
+          curl \
+            --fail-with-body \
+            --silent \
+            --request POST \
+            --header "Content-Type: application/json" \
+            --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
+            "${BETTERSTACK_HEARTBEAT_URL}/${SDK_EXIT}"
+          echo "INFO: Sent SDK heartbeat to BetterStack with exit code '${SDK_EXIT}'"
+
+      - name: Heartbeat / BetterStack / Platform API
+        if: always()
+        shell: bash
+        env:
+          BETTERSTACK_HEARTBEAT_URL_PLATFORM_API: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}"
+          PLATFORM_API_EXIT: ${{ steps.collect.outputs.platform_api_exit }}
+        run: |
+          if [ -z "$BETTERSTACK_HEARTBEAT_URL_PLATFORM_API" ]; then
+            echo "INFO: No BetterStack Platform API heartbeat URL configured, skipped."
+            exit 0
           fi
-          exit $EXIT_CODE
+          BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
+            --arg github_workflow "${{ github.workflow }}" \
+            --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg github_run_id "${{ github.run_id }}" \
+            --arg github_job "${{ github.job }}" \
+            --arg github_sha "${{ github.sha }}" \
+            --arg github_actor "${{ github.actor }}" \
+            --arg github_repository "${{ github.repository }}" \
+            --arg github_ref "${{ github.ref }}" \
+            --arg job_status "${{ job.status }}" \
+            --arg github_event_name "${{ github.event_name }}" \
+            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            '{
+              github: {
+                workflow: $github_workflow,
+                run_url: $github_run_url,
+                run_id: $github_run_id,
+                job: $github_job,
+                sha: $github_sha,
+                actor: $github_actor,
+                repository: $github_repository,
+                ref: $github_ref,
+                event_name: $github_event_name
+              },
+              job: {
+                status: $job_status,
+              },
+              timestamp: $timestamp,
+            }'
+          )
+          curl \
+            --fail-with-body \
+            --silent \
+            --request POST \
+            --header "Content-Type: application/json" \
+            --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
+            "${BETTERSTACK_HEARTBEAT_URL_PLATFORM_API}/${PLATFORM_API_EXIT}"
+          echo "INFO: Sent Platform API heartbeat to BetterStack with exit code '${PLATFORM_API_EXIT}'"
+
+      - name: Heartbeat / BetterStack / HE-TME
+        if: always()
+        shell: bash
+        env:
+          BETTERSTACK_HEARTBEAT_URL_HE_TME: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION }}"
+          HE_TME_EXIT: ${{ steps.collect.outputs.he_tme_exit }}
+        run: |
+          if [ -z "$BETTERSTACK_HEARTBEAT_URL_HE_TME" ]; then
+            echo "INFO: No BetterStack HE-TME heartbeat URL configured, skipped."
+            exit 0
+          fi
+          BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
+            --arg github_workflow "${{ github.workflow }}" \
+            --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg github_run_id "${{ github.run_id }}" \
+            --arg github_job "${{ github.job }}" \
+            --arg github_sha "${{ github.sha }}" \
+            --arg github_actor "${{ github.actor }}" \
+            --arg github_repository "${{ github.repository }}" \
+            --arg github_ref "${{ github.ref }}" \
+            --arg job_status "${{ job.status }}" \
+            --arg github_event_name "${{ github.event_name }}" \
+            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            '{
+              github: {
+                workflow: $github_workflow,
+                run_url: $github_run_url,
+                run_id: $github_run_id,
+                job: $github_job,
+                sha: $github_sha,
+                actor: $github_actor,
+                repository: $github_repository,
+                ref: $github_ref,
+                event_name: $github_event_name
+              },
+              job: {
+                status: $job_status,
+              },
+              timestamp: $timestamp,
+            }'
+          )
+          curl \
+            --fail-with-body \
+            --silent \
+            --request POST \
+            --header "Content-Type: application/json" \
+            --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
+            "${BETTERSTACK_HEARTBEAT_URL_HE_TME}/${HE_TME_EXIT}"
+          echo "INFO: Sent HE-TME heartbeat to BetterStack with exit code '${HE_TME_EXIT}'"
 
       - name: Upload test results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: ${{ always() && (env.GITHUB_WORKFLOW_RUNTIME != 'ACT') }}
         with:
           name: test-results-scheduled
           path: |
-            reports/junit.xml
+            reports/junit_sdk.xml
+            reports/junit_platform_api.xml
+            reports/junit_he_tme.xml
             reports/coverage.xml
             reports/coverage.md
             reports/coverage_html
             aignostics.log
           retention-days: 7
+
+      - name: Fail job if any tests failed
+        shell: bash
+        run: exit ${{ steps.collect.outputs.combined_exit }}

.github/workflows/scheduled-testing-production-hourly.yml

@@ -24,8 +24,12 @@ jobs:
       AIGNOSTICS_REFRESH_TOKEN_STAGING: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_STAGING }}
       GCP_CREDENTIALS_STAGING: ${{ secrets.GCP_CREDENTIALS_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING }}
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION: ${{ secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
       GCP_CREDENTIALS_PRODUCTION: ${{ secrets.GCP_CREDENTIALS_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }} # For metrics and heartbeat

.github/workflows/scheduled-testing-staging-hourly.yml

@@ -24,8 +24,12 @@ jobs:
       AIGNOSTICS_REFRESH_TOKEN_STAGING: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_STAGING }}
       GCP_CREDENTIALS_STAGING: ${{ secrets.GCP_CREDENTIALS_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING }}
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION: ${{ secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
       GCP_CREDENTIALS_PRODUCTION: ${{ secrets.GCP_CREDENTIALS_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }} # For metrics and heartbeat

pyproject.toml

@@ -383,6 +383,8 @@ markers = [
     "unit: Solitary unit tests - test a layer of a module in isolation with all dependencies mocked, except interaction with shared utils and the systems module. Unit tests must be able to pass offline, i.e. not calls to external services. The timeout should not be bigger than the default 10s, and must be <5 min.",
     "integration: Sociable integration tests - test interactions across architectural layers (e.g. CLI/GUI→Service, Service→Utils) or between modules (e.g. Application→Platform), using real SDK collaborators, real file I/O, real subprocesses, and real Docker containers. Integration test must be able to pass offline, i.e. mock external services (Aignostics Platform API, Auth0, S3/GCS buckets, IDC). The timeout should not be bigger than the default 10s, and must be <5 min.",
     "e2e: End-to-end tests - test complete workflows with real external network services (Aignostics Platform API, cloud storage, IDC, etc). If the test timeout is >= 5 min and < 60 min, additionally mark as `long_running`, if >= 60min mark as 'very_long_running'.",
+    "monitors: Tag a scheduled test with the application it monitors, e.g. @pytest.mark.monitors('he-tme'). Tests without this marker are considered SDK-layer health checks. Used to route Better Stack heartbeats to the correct monitor.",
+    "monitors_platform_api: Tag a scheduled test that monitors the Platform API layer (auth, listing, connectivity). Used to route Platform API Better Stack heartbeats separately from SDK and application monitors.",
 ]
 md_report = true
 md_report_output = "reports/pytest.md"