diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2160908..57d9034 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,8 +77,8 @@ jobs: exit 1 fi - echo "=== Waiting for job to reach finished/failed (timeout 10 min) ===" - DEADLINE=$((SECONDS + 600)) + echo "=== Waiting for job to reach finished/failed (timeout 20 min) ===" + DEADLINE=$((SECONDS + 1200)) POLL=0 while [[ $SECONDS -lt $DEADLINE ]]; do STATUS_JSON=$(python3 $PANDA_COMPOSE_DIR/scripts/pandajob-status "$JOB_ID") @@ -138,7 +138,7 @@ jobs: --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' 2>/dev/null || true if [[ "$JOB_STATUS" != "finished" ]]; then - echo "ERROR: job did not reach 'finished' within 10 minutes (last status: $JOB_STATUS)" + echo "ERROR: job did not reach 'finished' within 20 minutes (last status: $JOB_STATUS)" exit 1 fi diff --git a/action.yml b/action.yml index 86b648a..5c7276b 100644 --- a/action.yml +++ b/action.yml @@ -57,10 +57,53 @@ runs: - name: Start PanDA stack shell: bash run: | - docker compose \ - -f __panda_compose__/docker-compose.yml \ - -p ${{ inputs.project-name }} \ - up -d + compose() { + docker compose \ + -f __panda_compose__/docker-compose.yml \ + -p "${{ inputs.project-name }}" "$@" + } + + # Bring up the base services first and let Postgres complete its + # first-run initialization in isolation. On a fresh CI volume the + # image runs initdb + panda_db_init.sh and restarts the container + # once (RestartCount=1); during that window it listens on the Unix + # socket only and briefly transitions through unhealthy. Attaching + # dependents via depends_on: service_healthy at the same time races + # this and intermittently aborts with + # 'dependency failed to start: container ...-postgres-1 is unhealthy'. + compose up -d postgres activemq mariadb + + pg="$(compose ps -q postgres)" + if [ -z "$pg" ]; then + echo "ERROR: could not find postgres container for project '${{ inputs.project-name }}'" >&2 + compose ps || true + exit 1 + fi + echo "Waiting for Postgres to settle into a stable healthy state..." + stable=0 + for _ in $(seq 1 60); do + status=$(docker inspect --format '{{.State.Health.Status}}' "$pg" 2>/dev/null || echo starting) + restarts=$(docker inspect --format '{{.RestartCount}}' "$pg" 2>/dev/null || echo 0) + echo " postgres health=$status restarts=$restarts" + if [ "$status" = healthy ]; then + # Require two consecutive healthy reads so we don't catch it + # right before the init restart. + stable=$((stable + 1)) + [ "$stable" -ge 2 ] && break + else + stable=0 + fi + sleep 5 + done + if [ "$stable" -lt 2 ]; then + echo "ERROR: Postgres did not reach a stable healthy state" >&2 + docker logs "$pg" 2>&1 | tail -n 80 || true + exit 1 + fi + + # Postgres is stable; bring up the rest. Dependents' health + # conditions are already satisfied, so there is no startup race. + compose up -d - name: Wait for PanDA server shell: bash diff --git a/docker-compose.yml b/docker-compose.yml index 1773776..096797d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,9 +29,9 @@ services: volumes: - panda-db-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db}"] - interval: 20s - timeout: 60s + test: ["CMD-SHELL", "pg_isready -h localhost -U postgres -d ${PANDA_DB_NAME:-panda_db} -q"] + interval: 10s + timeout: 5s retries: 15 start_period: 60s