From d3e79e48fbcb8c28b110b493e081509649f6fa1a Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 2 Sep 2026 04:08:01 -0400 Subject: [PATCH 1/5] ci.yml: increase job timeout jobs are flaky on CI --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2160908..57d9034 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,8 +77,8 @@ jobs: exit 1 fi - echo "=== Waiting for job to reach finished/failed (timeout 10 min) ===" - DEADLINE=$((SECONDS + 600)) + echo "=== Waiting for job to reach finished/failed (timeout 20 min) ===" + DEADLINE=$((SECONDS + 1200)) POLL=0 while [[ $SECONDS -lt $DEADLINE ]]; do STATUS_JSON=$(python3 $PANDA_COMPOSE_DIR/scripts/pandajob-status "$JOB_ID") @@ -138,7 +138,7 @@ jobs: --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' 2>/dev/null || true if [[ "$JOB_STATUS" != "finished" ]]; then - echo "ERROR: job did not reach 'finished' within 10 minutes (last status: $JOB_STATUS)" + echo "ERROR: job did not reach 'finished' within 20 minutes (last status: $JOB_STATUS)" exit 1 fi From c9318f3bd7b301d0910bd52d0fb03c0511068a93 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 2 Sep 2026 04:29:14 -0400 Subject: [PATCH 2/5] Harden Postgres healthcheck to reduce CI flaky unhealthy failures Fix the inverted timeout>interval (now 5s timeout, 10s interval). --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1773776..096797d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,9 +29,9 @@ services: volumes: - panda-db-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db}"] - interval: 20s - timeout: 60s + test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db} -q"] + interval: 10s + timeout: 5s retries: 15 start_period: 60s From 9f70642fcd60e77ebc18a41c811c86489ea4db86 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 2 Sep 2026 09:29:20 -0400 Subject: [PATCH 3/5] action: bring up Postgres first and wait for stable healthy before dependents The panda-database image runs first-run init on a fresh CI volume (initdb + panda_db_init.sh) and restarts the container once. During that window it listens on the Unix socket only and transitions through unhealthy. Starting dependents via depends_on: service_healthy at the same time races this and intermittently aborts with 'container ...-postgres-1 is unhealthy'. Start postgres/activemq/mariadb first, wait for two consecutive healthy reads (past the init restart), then bring up the rest. --- action.yml | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 86b648a..4ef4a18 100644 --- a/action.yml +++ b/action.yml @@ -57,10 +57,48 @@ runs: - name: Start PanDA stack shell: bash run: | - docker compose \ - -f __panda_compose__/docker-compose.yml \ - -p ${{ inputs.project-name }} \ - up -d + compose() { + docker compose \ + -f __panda_compose__/docker-compose.yml \ + -p ${{ inputs.project-name }} "$@" + } + + # Bring up the base services first and let Postgres complete its + # first-run initialization in isolation. On a fresh CI volume the + # image runs initdb + panda_db_init.sh and restarts the container + # once (RestartCount=1); during that window it listens on the Unix + # socket only and briefly transitions through unhealthy. Attaching + # dependents via depends_on: service_healthy at the same time races + # this and intermittently aborts with + # 'dependency failed to start: container ...-postgres-1 is unhealthy'. + compose up -d postgres activemq mariadb + + pg="${{ inputs.project-name }}-postgres-1" + echo "Waiting for Postgres to settle into a stable healthy state..." + stable=0 + for _ in $(seq 1 60); do + status=$(docker inspect --format '{{.State.Health.Status}}' "$pg" 2>/dev/null || echo starting) + restarts=$(docker inspect --format '{{.RestartCount}}' "$pg" 2>/dev/null || echo 0) + echo " postgres health=$status restarts=$restarts" + if [ "$status" = healthy ]; then + # Require two consecutive healthy reads so we don't catch it + # right before the init restart. + stable=$((stable + 1)) + [ "$stable" -ge 2 ] && break + else + stable=0 + fi + sleep 5 + done + if [ "$stable" -lt 2 ]; then + echo "ERROR: Postgres did not reach a stable healthy state" >&2 + docker logs "$pg" 2>&1 | tail -n 80 || true + exit 1 + fi + + # Postgres is stable; bring up the rest. Dependents' health + # conditions are already satisfied, so there is no startup race. + compose up -d - name: Wait for PanDA server shell: bash From e389e460bfb09cc4b927876a83b489e6202758a7 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 2 Sep 2026 09:59:47 -0400 Subject: [PATCH 4/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 4ef4a18..62d1881 100644 --- a/action.yml +++ b/action.yml @@ -60,7 +60,7 @@ runs: compose() { docker compose \ -f __panda_compose__/docker-compose.yml \ - -p ${{ inputs.project-name }} "$@" + -p "${{ inputs.project-name }}" "$@" } # Bring up the base services first and let Postgres complete its From 730c01610458121973f21ec621f8bb6bedf415a0 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 2 Sep 2026 10:01:03 -0400 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 62d1881..5c7276b 100644 --- a/action.yml +++ b/action.yml @@ -73,7 +73,12 @@ runs: # 'dependency failed to start: container ...-postgres-1 is unhealthy'. compose up -d postgres activemq mariadb - pg="${{ inputs.project-name }}-postgres-1" + pg="$(compose ps -q postgres)" + if [ -z "$pg" ]; then + echo "ERROR: could not find postgres container for project '${{ inputs.project-name }}'" >&2 + compose ps || true + exit 1 + fi echo "Waiting for Postgres to settle into a stable healthy state..." stable=0 for _ in $(seq 1 60); do