diff --git a/devops/production/bin/backup-postgres.sh b/devops/production/bin/backup-postgres.sh index a15fbac..7d70c3a 100755 --- a/devops/production/bin/backup-postgres.sh +++ b/devops/production/bin/backup-postgres.sh @@ -6,7 +6,6 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SERVER_SERVICES_ROOT="/opt/syr/services/staging" DOCKER_WRAPPER="${SERVER_SERVICES_ROOT}/bin/docker" BACKUP_DIR="/opt/syr/backups/production/opensyria/postgres" -POSTGRES_CONTAINER="infra-postgres" DATABASE_NAME="opensyria_datasets_production" LOCK_FILE="${ROOT_DIR}/.backup.lock" TEMP_DUMP="" @@ -68,12 +67,10 @@ main() { && ! -e "${final_recovery}" && ! -L "${final_recovery}" ]] \ || fail "Backup timestamp collision: ${timestamp}" - docker_cmd exec "${POSTGRES_CONTAINER}" sh -ceu \ - 'exec pg_dump --format=custom --no-owner --no-acl --exclude-extension=postgis --username="$POSTGRES_USER" --dbname="$1"' \ - sh "${DATABASE_NAME}" > "${TEMP_DUMP}" + docker_cmd dump-opensyria-database > "${TEMP_DUMP}" [[ -s "${TEMP_DUMP}" ]] || fail "PostgreSQL produced an empty backup" - docker_cmd exec -i "${POSTGRES_CONTAINER}" pg_restore --list < "${TEMP_DUMP}" >/dev/null + docker_cmd validate-postgres-dump < "${TEMP_DUMP}" >/dev/null digest="$(sha256sum "${TEMP_DUMP}" | cut -d ' ' -f 1)" [[ "${digest}" =~ ^[0-9a-f]{64}$ ]] || fail "Could not calculate backup checksum" printf '%s %s\n' "${digest}" "$(basename "${final_dump}")" > "${TEMP_CHECKSUM}" diff --git a/devops/production/bin/deploy.sh b/devops/production/bin/deploy.sh index fd19596..715847b 100755 --- a/devops/production/bin/deploy.sh +++ b/devops/production/bin/deploy.sh @@ -9,6 +9,7 @@ INFISICAL_LOGIN_HELPER="${SERVER_SERVICES_ROOT}/bin/infisical-login" COMPOSE_FILE="${ROOT_DIR}/docker-compose.app.yml" COMPOSE_ENV_FILE="${ROOT_DIR}/.deploy.env" RUNTIME_ENV_FILE="${ROOT_DIR}/env/api.env" +API_RELEASE_PROBE="fetch('http://127.0.0.1:3000/health/ready').then(async (response)=>{const payload=await response.json();const data=payload.data??{};if(!response.ok||payload.success!==true)throw new Error('readiness failed');if(data.database?.status!=='up'||!data.database?.release)throw new Error('pinned read model is unavailable');if(!(data.database?.recordCount>0))throw new Error('pinned read model is empty');if(typeof data.app?.release!=='string'||!data.app.release)throw new Error('application release is unavailable');process.stdout.write(data.app.release)}).catch((error)=>{console.error(error.message);process.exit(1)})" INFISICAL_CONFIG_FILE="${ROOT_DIR}/.infisical.env" STATE_DIR="${ROOT_DIR}/state" ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot" @@ -217,29 +218,15 @@ wait_for_service_health() { verify_direct_release() { local slot="$1" local expected_release="$2" - local service + local service actual_release service="$(service_for_slot "${slot}")" - compose exec -T -e EXPECTED_RELEASE="${expected_release}" "${service}" node - <<'NODE' -fetch('http://127.0.0.1:3000/health/ready') - .then(async (response) => { - const payload = await response.json() - const data = payload.data ?? {} - if (!response.ok || payload.success !== true) throw new Error('readiness failed') - if (data.app?.release !== process.env.EXPECTED_RELEASE) { - throw new Error('application release mismatch') - } - if (data.database?.status !== 'up' || !data.database?.release) { - throw new Error('pinned read model is unavailable') - } - if (!(data.database?.recordCount > 0)) throw new Error('pinned read model is empty') - }) - .then(() => process.exit(0)) - .catch((error) => { - console.error(error.message) - process.exit(1) - }) -NODE + actual_release="$(compose exec -T "${service}" node -e "${API_RELEASE_PROBE}")" \ + || return 1 + if [[ "${actual_release}" != "${expected_release}" ]]; then + echo "Expected ${service} release ${expected_release}, got ${actual_release:-missing}." >&2 + return 1 + fi } verify_private_route() { diff --git a/devops/production/bin/test_production_deploy_contract.py b/devops/production/bin/test_production_deploy_contract.py index a59a60b..bf50a1f 100644 --- a/devops/production/bin/test_production_deploy_contract.py +++ b/devops/production/bin/test_production_deploy_contract.py @@ -3,6 +3,7 @@ DEPLOY_SCRIPT = Path(__file__).with_name("deploy.sh") +BACKUP_SCRIPT = Path(__file__).with_name("backup-postgres.sh") class ProductionDeployContractTest(unittest.TestCase): @@ -21,6 +22,22 @@ def test_uses_approved_container_diagnostics(self) -> None: self.assertNotIn("docker_cmd inspect", script) self.assertNotIn("compose ps -q", script) + def test_uses_fixed_api_release_probe(self) -> None: + script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + self.assertIn( + 'compose exec -T "${service}" node -e "${API_RELEASE_PROBE}"', + script, + ) + self.assertNotIn("compose exec -T -e EXPECTED_RELEASE", script) + + def test_uses_typed_postgres_backup_operations(self) -> None: + script = BACKUP_SCRIPT.read_text(encoding="utf-8") + + self.assertIn("docker_cmd dump-opensyria-database", script) + self.assertIn("docker_cmd validate-postgres-dump", script) + self.assertNotIn("docker_cmd exec", script) + if __name__ == "__main__": unittest.main()