Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions devops/production/bin/backup-postgres.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SERVER_SERVICES_ROOT="/opt/syr/services/staging"
DOCKER_WRAPPER="${SERVER_SERVICES_ROOT}/bin/docker"
BACKUP_DIR="/opt/syr/backups/production/opensyria/postgres"
POSTGRES_CONTAINER="infra-postgres"
DATABASE_NAME="opensyria_datasets_production"
LOCK_FILE="${ROOT_DIR}/.backup.lock"
TEMP_DUMP=""
Expand Down Expand Up @@ -68,12 +67,10 @@ main() {
&& ! -e "${final_recovery}" && ! -L "${final_recovery}" ]] \
|| fail "Backup timestamp collision: ${timestamp}"

docker_cmd exec "${POSTGRES_CONTAINER}" sh -ceu \
'exec pg_dump --format=custom --no-owner --no-acl --exclude-extension=postgis --username="$POSTGRES_USER" --dbname="$1"' \
sh "${DATABASE_NAME}" > "${TEMP_DUMP}"
docker_cmd dump-opensyria-database > "${TEMP_DUMP}"
[[ -s "${TEMP_DUMP}" ]] || fail "PostgreSQL produced an empty backup"

docker_cmd exec -i "${POSTGRES_CONTAINER}" pg_restore --list < "${TEMP_DUMP}" >/dev/null
docker_cmd validate-postgres-dump < "${TEMP_DUMP}" >/dev/null
digest="$(sha256sum "${TEMP_DUMP}" | cut -d ' ' -f 1)"
[[ "${digest}" =~ ^[0-9a-f]{64}$ ]] || fail "Could not calculate backup checksum"
printf '%s %s\n' "${digest}" "$(basename "${final_dump}")" > "${TEMP_CHECKSUM}"
Expand Down
29 changes: 8 additions & 21 deletions devops/production/bin/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ INFISICAL_LOGIN_HELPER="${SERVER_SERVICES_ROOT}/bin/infisical-login"
COMPOSE_FILE="${ROOT_DIR}/docker-compose.app.yml"
COMPOSE_ENV_FILE="${ROOT_DIR}/.deploy.env"
RUNTIME_ENV_FILE="${ROOT_DIR}/env/api.env"
API_RELEASE_PROBE="fetch('http://127.0.0.1:3000/health/ready').then(async (response)=>{const payload=await response.json();const data=payload.data??{};if(!response.ok||payload.success!==true)throw new Error('readiness failed');if(data.database?.status!=='up'||!data.database?.release)throw new Error('pinned read model is unavailable');if(!(data.database?.recordCount>0))throw new Error('pinned read model is empty');if(typeof data.app?.release!=='string'||!data.app.release)throw new Error('application release is unavailable');process.stdout.write(data.app.release)}).catch((error)=>{console.error(error.message);process.exit(1)})"
INFISICAL_CONFIG_FILE="${ROOT_DIR}/.infisical.env"
STATE_DIR="${ROOT_DIR}/state"
ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
Expand Down Expand Up @@ -217,29 +218,15 @@ wait_for_service_health() {
verify_direct_release() {
local slot="$1"
local expected_release="$2"
local service
local service actual_release

service="$(service_for_slot "${slot}")"
compose exec -T -e EXPECTED_RELEASE="${expected_release}" "${service}" node - <<'NODE'
fetch('http://127.0.0.1:3000/health/ready')
.then(async (response) => {
const payload = await response.json()
const data = payload.data ?? {}
if (!response.ok || payload.success !== true) throw new Error('readiness failed')
if (data.app?.release !== process.env.EXPECTED_RELEASE) {
throw new Error('application release mismatch')
}
if (data.database?.status !== 'up' || !data.database?.release) {
throw new Error('pinned read model is unavailable')
}
if (!(data.database?.recordCount > 0)) throw new Error('pinned read model is empty')
})
.then(() => process.exit(0))
.catch((error) => {
console.error(error.message)
process.exit(1)
})
NODE
actual_release="$(compose exec -T "${service}" node -e "${API_RELEASE_PROBE}")" \
|| return 1
if [[ "${actual_release}" != "${expected_release}" ]]; then
echo "Expected ${service} release ${expected_release}, got ${actual_release:-missing}." >&2
return 1
fi
}

verify_private_route() {
Expand Down
17 changes: 17 additions & 0 deletions devops/production/bin/test_production_deploy_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@


DEPLOY_SCRIPT = Path(__file__).with_name("deploy.sh")
BACKUP_SCRIPT = Path(__file__).with_name("backup-postgres.sh")


class ProductionDeployContractTest(unittest.TestCase):
Expand All @@ -21,6 +22,22 @@ def test_uses_approved_container_diagnostics(self) -> None:
self.assertNotIn("docker_cmd inspect", script)
self.assertNotIn("compose ps -q", script)

def test_uses_fixed_api_release_probe(self) -> None:
script = DEPLOY_SCRIPT.read_text(encoding="utf-8")

self.assertIn(
'compose exec -T "${service}" node -e "${API_RELEASE_PROBE}"',
script,
)
self.assertNotIn("compose exec -T -e EXPECTED_RELEASE", script)

def test_uses_typed_postgres_backup_operations(self) -> None:
script = BACKUP_SCRIPT.read_text(encoding="utf-8")

self.assertIn("docker_cmd dump-opensyria-database", script)
self.assertIn("docker_cmd validate-postgres-dump", script)
self.assertNotIn("docker_cmd exec", script)


if __name__ == "__main__":
unittest.main()