ci.yml: fix flakiness

.github/workflows/ci.yml

@@ -77,8 +77,8 @@ jobs:
             exit 1
           fi
 
-          echo "=== Waiting for job to reach finished/failed (timeout 10 min) ==="
-          DEADLINE=$((SECONDS + 600))
+          echo "=== Waiting for job to reach finished/failed (timeout 20 min) ==="
+          DEADLINE=$((SECONDS + 1200))
           POLL=0
           while [[ $SECONDS -lt $DEADLINE ]]; do
             STATUS_JSON=$(python3 $PANDA_COMPOSE_DIR/scripts/pandajob-status "$JOB_ID")
@@ -138,7 +138,7 @@ jobs:
             --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' 2>/dev/null || true
 
           if [[ "$JOB_STATUS" != "finished" ]]; then
-            echo "ERROR: job did not reach 'finished' within 10 minutes (last status: $JOB_STATUS)"
+            echo "ERROR: job did not reach 'finished' within 20 minutes (last status: $JOB_STATUS)"
             exit 1
           fi
 

action.yml

@@ -57,10 +57,53 @@ runs:
     - name: Start PanDA stack
       shell: bash
       run: |
-        docker compose \
-          -f __panda_compose__/docker-compose.yml \
-          -p ${{ inputs.project-name }} \
-          up -d
+        compose() {
+          docker compose \
+            -f __panda_compose__/docker-compose.yml \
+            -p "${{ inputs.project-name }}" "$@"
+        }
+
+        # Bring up the base services first and let Postgres complete its
+        # first-run initialization in isolation. On a fresh CI volume the
+        # image runs initdb + panda_db_init.sh and restarts the container
+        # once (RestartCount=1); during that window it listens on the Unix
+        # socket only and briefly transitions through unhealthy. Attaching
+        # dependents via depends_on: service_healthy at the same time races
+        # this and intermittently aborts with
+        # 'dependency failed to start: container ...-postgres-1 is unhealthy'.
+        compose up -d postgres activemq mariadb
+
+        pg="$(compose ps -q postgres)"
+        if [ -z "$pg" ]; then
+          echo "ERROR: could not find postgres container for project '${{ inputs.project-name }}'" >&2
+          compose ps || true
+          exit 1
+        fi
+        echo "Waiting for Postgres to settle into a stable healthy state..."
+        stable=0
+        for _ in $(seq 1 60); do
+          status=$(docker inspect --format '{{.State.Health.Status}}' "$pg" 2>/dev/null || echo starting)
+          restarts=$(docker inspect --format '{{.RestartCount}}' "$pg" 2>/dev/null || echo 0)
+          echo "  postgres health=$status restarts=$restarts"
+          if [ "$status" = healthy ]; then
+            # Require two consecutive healthy reads so we don't catch it
+            # right before the init restart.
+            stable=$((stable + 1))
+            [ "$stable" -ge 2 ] && break
+          else
+            stable=0
+          fi
+          sleep 5
+        done
+        if [ "$stable" -lt 2 ]; then
+          echo "ERROR: Postgres did not reach a stable healthy state" >&2
+          docker logs "$pg" 2>&1 | tail -n 80 || true
+          exit 1
+        fi
+
+        # Postgres is stable; bring up the rest. Dependents' health
+        # conditions are already satisfied, so there is no startup race.
+        compose up -d
 
     - name: Wait for PanDA server
       shell: bash

docker-compose.yml

@@ -29,9 +29,9 @@ services:
     volumes:
       - panda-db-data:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db}"]
-      interval: 20s
-      timeout: 60s
+      test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db} -q"]
+      interval: 10s
+      timeout: 5s
       retries: 15
       start_period: 60s