From cb3c7bb2ffbad770a52091bc7025078c84c142f2 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 07:57:11 +0800 Subject: [PATCH 1/6] fix: replace shared IBKR precheck dispatcher Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 154 +++++++++++----------- scripts/reconcile_cloud_runtime.py | 3 + tests/test_reconcile_cloud_runtime.py | 4 +- tests/test_scheduler_deadline_contract.py | 21 +++ tests/test_sync_cloud_run_env_workflow.sh | 22 +++- 5 files changed, 116 insertions(+), 88 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 76bb607..62422e2 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -614,6 +614,7 @@ jobs: shared_remove_env_vars=( "IB_GATEWAY_HOST" "IB_GATEWAY_PORT" + "IBKR_MONITOR_DISPATCH_TARGETS_JSON" "TELEGRAM_CHAT_ID" "IB_CLIENT_ID" "IB_GATEWAY_INSTANCE_NAME" @@ -889,44 +890,6 @@ jobs: shared_remove_env_vars+=("STRATEGY_PLUGIN_ALERT_TELEGRAM_BODY_MAX_CHARS") fi - monitor_dispatch_targets_file="$(mktemp)" - for target_index in "${!target_services[@]}"; do - cloud_run_service="${target_services[${target_index}]}" - service_url="$(gcloud run services describe "${cloud_run_service}" \ - --project="${GCP_PROJECT_ID}" \ - --region="${CLOUD_RUN_REGION}" \ - --format='value(status.url)' 2>/dev/null || true)" - if [ -z "${service_url}" ]; then - echo "Unable to resolve Cloud Run service URL for ${cloud_run_service}; cannot build monitor dispatcher targets." >&2 - exit 1 - fi - SYNC_TARGET_INDEX="${target_index}" SERVICE_URL="${service_url}" python - <<'PY' >> "${monitor_dispatch_targets_file}" - import json - import os - - plan = json.loads(os.environ["SYNC_PLAN_JSON"]) - target = plan["targets"][int(os.environ["SYNC_TARGET_INDEX"])] - env = target.get("env") or {} - scheduler = target.get("scheduler") or {} - payload = { - "service_name": target["service_name"], - "service_url": os.environ["SERVICE_URL"].rstrip("/"), - "strategy_profile": target.get("strategy_profile") or env.get("STRATEGY_PROFILE"), - "account_group": env.get("ACCOUNT_GROUP"), - "runtime_target_enabled": env.get("RUNTIME_TARGET_ENABLED", "true"), - # Dedicated Cloud Scheduler jobs already own /health warmups and - # /run execution. The shared dispatcher only owns prechecks. - "scheduler": { - "timezone": scheduler.get("timezone"), - "precheck_time": scheduler.get("precheck_time"), - }, - } - print(json.dumps(payload, separators=(",", ":"), sort_keys=True)) - PY - done - monitor_dispatch_targets_json="$(jq -sc '{targets:.}' "${monitor_dispatch_targets_file}")" - shared_env_pairs+=("IBKR_MONITOR_DISPATCH_TARGETS_JSON=${monitor_dispatch_targets_json}") - for target_index in "${!target_services[@]}"; do cloud_run_service="${target_services[${target_index}]}" mapfile -t target_env_pairs < <(emit_target_env_pairs "${target_index}") @@ -1022,6 +985,8 @@ jobs: timezone, str(scheduler.get("main_time") or configured_time("CLOUD_SCHEDULER_MAIN_TIME", "45 15")), str(scheduler.get("probe_time") or configured_time("CLOUD_SCHEDULER_PROBE_TIME", "35 9,15")), + str(scheduler.get("precheck_time") or configured_time("CLOUD_SCHEDULER_PRECHECK_TIME", "45 9")), + str(env.get("RUNTIME_TARGET_ENABLED", "true")).strip().lower() or "true", str(scheduler.get("attempt_deadline") or ""), ] ) @@ -1030,7 +995,7 @@ jobs: ) for update in "${scheduler_updates[@]}"; do - IFS=$'\t' read -r cloud_run_service market_timezone main_time warmup_time main_attempt_deadline <<< "${update}" + IFS=$'\t' read -r cloud_run_service market_timezone main_time warmup_time precheck_time runtime_target_enabled main_attempt_deadline <<< "${update}" if [ -z "${cloud_run_service}" ] || [ -z "${market_timezone}" ]; then continue fi @@ -1103,6 +1068,24 @@ jobs: desired_warmup_schedule="$(CURRENT_SCHEDULE="${desired_schedule}" SCHEDULE_TIME="${warmup_time}" python - <<'PY' import os + current_fields = os.environ["CURRENT_SCHEDULE"].split() + time_fields = os.environ["SCHEDULE_TIME"].split() + if len(current_fields) != 5: + raise SystemExit(f"Cloud Scheduler schedule must have 5 fields: {os.environ['CURRENT_SCHEDULE']!r}") + if len(time_fields) == 5: + print(" ".join(time_fields)) + elif len(time_fields) == 2: + print(" ".join([*time_fields, *current_fields[2:]])) + else: + raise SystemExit( + f"Cloud Scheduler override must have 2 time fields or 5 cron fields: {os.environ['SCHEDULE_TIME']!r}" + ) + PY + )" + + desired_precheck_schedule="$(CURRENT_SCHEDULE="${desired_schedule}" SCHEDULE_TIME="${precheck_time}" python - <<'PY' + import os + current_fields = os.environ["CURRENT_SCHEDULE"].split() time_fields = os.environ["SCHEDULE_TIME"].split() if len(current_fields) != 5: @@ -1187,49 +1170,62 @@ jobs: --max-doublings=2 \ --quiet fi - done - monitor_dispatch_service="$(printf '%s\n' "${scheduler_updates[@]}" | head -n 1 | cut -f1)" - if [ -z "${monitor_dispatch_service}" ]; then - echo "Unable to resolve monitor dispatch service." >&2 - exit 1 - fi - monitor_dispatch_base_url="$(gcloud run services describe "${monitor_dispatch_service}" \ - --project="${GCP_PROJECT_ID}" \ - --region="${CLOUD_RUN_REGION}" \ - --format='value(status.url)' 2>/dev/null || true)" - if [ -z "${monitor_dispatch_base_url}" ]; then - echo "Unable to resolve monitor dispatch URL for ${monitor_dispatch_service}." >&2 - exit 1 - fi - monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler" - monitor_uri="${monitor_dispatch_base_url}/monitor-dispatch" - if gcloud scheduler jobs describe "${monitor_job_name}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" >/dev/null 2>&1; then - echo "Updating Cloud Scheduler job ${monitor_job_name} to ${monitor_uri}." - gcloud scheduler jobs update http "${monitor_job_name}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" \ - --uri="${monitor_uri}" \ - --schedule="*/5 * * * *" \ - --time-zone="UTC" \ - --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ - --oidc-token-audience="${monitor_dispatch_base_url}" \ - --quiet - else - echo "Creating Cloud Scheduler job ${monitor_job_name} at ${monitor_uri}." - gcloud scheduler jobs create http "${monitor_job_name}" \ + precheck_job_name="${cloud_run_service%-service}-precheck-scheduler" + precheck_uri="${service_url}/dry-run" + precheck_state="$(gcloud scheduler jobs describe "${precheck_job_name}" \ --project="${GCP_PROJECT_ID}" \ --location="${scheduler_location}" \ - --uri="${monitor_uri}" \ - --schedule="*/5 * * * *" \ - --time-zone="UTC" \ - --http-method=POST \ - --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ - --oidc-token-audience="${monitor_dispatch_base_url}" \ - --quiet - fi + --format='value(state)' 2>/dev/null || true)" + case "${runtime_target_enabled}" in + 1|true|yes|on) + if [ -n "${precheck_state}" ]; then + echo "Updating Cloud Scheduler precheck ${precheck_job_name} to ${desired_precheck_schedule}." + gcloud scheduler jobs update http "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --uri="${precheck_uri}" \ + --schedule="${desired_precheck_schedule}" \ + --time-zone="${market_timezone}" \ + --http-method=POST \ + --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ + --oidc-token-audience="${service_url}" \ + --attempt-deadline=120s \ + --max-retry-attempts=0 \ + --quiet + else + echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}." + gcloud scheduler jobs create http "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --uri="${precheck_uri}" \ + --schedule="${desired_precheck_schedule}" \ + --time-zone="${market_timezone}" \ + --http-method=POST \ + --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ + --oidc-token-audience="${service_url}" \ + --attempt-deadline=120s \ + --max-retry-attempts=0 \ + --quiet + fi + if [ "${precheck_state}" = "PAUSED" ]; then + gcloud scheduler jobs resume "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --quiet + fi + ;; + *) + if [ -n "${precheck_state}" ] && [ "${precheck_state}" != "PAUSED" ]; then + echo "Pausing Cloud Scheduler precheck ${precheck_job_name} because ${cloud_run_service} is disabled." + gcloud scheduler jobs pause "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --quiet + fi + ;; + esac + done python3 scripts/reconcile_cloud_runtime.py \ --platform=ibkr \ diff --git a/scripts/reconcile_cloud_runtime.py b/scripts/reconcile_cloud_runtime.py index 219ae53..c17d11b 100755 --- a/scripts/reconcile_cloud_runtime.py +++ b/scripts/reconcile_cloud_runtime.py @@ -291,6 +291,9 @@ def _legacy_jobs_for_target(platform: str, target: RuntimeTarget) -> list[str]: if scope and scope != service: jobs.append(f"lb-{scope}-backup-execution") elif platform == "ibkr": + canonical_precheck = f"{service.removesuffix('-service')}-precheck-scheduler" + jobs.remove(canonical_precheck) + jobs.append("interactive-brokers-monitor-dispatcher-scheduler") prefix = "interactive-brokers-quant-live-" suffix = service if suffix.startswith(prefix): diff --git a/tests/test_reconcile_cloud_runtime.py b/tests/test_reconcile_cloud_runtime.py index 4e990e1..d5d4fe7 100644 --- a/tests/test_reconcile_cloud_runtime.py +++ b/tests/test_reconcile_cloud_runtime.py @@ -60,9 +60,9 @@ def test_legacy_jobs_for_ibkr_service_include_only_explicit_candidates(self) -> "interactive-brokers-quant-live-u1234-service-probe-scheduler", "interactive-brokers-quant-live-u1234-service-precheck-scheduler", "interactive-brokers-quant-live-u1234-probe-scheduler", - "interactive-brokers-quant-live-u1234-precheck-scheduler", "ibkr-u1234-backup-execution", "ibkr-u1234-pre-market-dry-run", + "interactive-brokers-monitor-dispatcher-scheduler", }, ) @@ -181,9 +181,9 @@ def test_delete_legacy_schedulers_deletes_only_known_jobs(self) -> None: "interactive-brokers-quant-live-u1234-service-probe-scheduler", "interactive-brokers-quant-live-u1234-service-precheck-scheduler", "interactive-brokers-quant-live-u1234-probe-scheduler", - "interactive-brokers-quant-live-u1234-precheck-scheduler", "ibkr-u1234-backup-execution", "ibkr-u1234-pre-market-dry-run", + "interactive-brokers-monitor-dispatcher-scheduler", } describe_calls: list[list[str]] = [] delete_calls: list[list[str]] = [] diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index 284aa7f..a444802 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -8,6 +8,7 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "sync-cloud-run-env.yml" +MAIN = ROOT / "main.py" def test_live_gateway_run_deadline_exceeds_cloud_run_timeout() -> None: @@ -41,3 +42,23 @@ def test_run_deadline_is_plan_scoped_and_warmup_remains_60_seconds() -> None: assert 'main_attempt_deadline_args+=(--attempt-deadline="${main_attempt_deadline}")' in workflow assert workflow.count('"${main_attempt_deadline_args[@]}"') == 2 assert workflow.count("--attempt-deadline=60s") == 2 + + +def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + main = MAIN.read_text(encoding="utf-8") + + assert 'precheck_job_name="${cloud_run_service%-service}-precheck-scheduler"' in workflow + assert 'precheck_uri="${service_url}/dry-run"' in workflow + assert workflow.count("--attempt-deadline=120s") == 2 + assert workflow.count("--max-retry-attempts=0") == 2 + assert 'gcloud scheduler jobs resume "${precheck_job_name}"' in workflow + assert 'gcloud scheduler jobs pause "${precheck_job_name}"' in workflow + assert 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' not in workflow + assert 'shared_env_pairs+=("IBKR_MONITOR_DISPATCH_TARGETS_JSON=' not in workflow + + strategy_deadline = re.search(r'IBKR_DRY_RUN_DEADLINE_SECONDS = get_positive_int_env\([^,]+, (\d+)\)', main) + report_grace = re.search(r'IBKR_DEADLINE_REPORT_GRACE_SECONDS = get_positive_int_env\([^,]+, (\d+)\)', main) + assert strategy_deadline is not None + assert report_grace is not None + assert 120 > int(strategy_deadline.group(1)) + int(report_grace.group(1)) diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 63f30d2..2e60897 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -119,9 +119,9 @@ grep -Fq -- '--max-instances 1' "$workflow_file" grep -Fq -- '--remove-env-vars "$(IFS=,; echo "${remove_env_vars[*]}")' "$workflow_file" grep -Fq -- '--update-env-vars "^|^$(join_by_delimiter "|" "${env_pairs[@]}")' "$workflow_file" grep -Fq -- '--update-labels "$(IFS=,; echo "${target_label_pairs[*]}")' "$workflow_file" -grep -Fq 'monitor_dispatch_targets_json="$(jq -sc '\''{targets:.}'\'' "${monitor_dispatch_targets_file}")"' "$workflow_file" -grep -Fq 'shared_env_pairs+=("IBKR_MONITOR_DISPATCH_TARGETS_JSON=${monitor_dispatch_targets_json}")' "$workflow_file" -grep -Fq '"precheck_time": scheduler.get("precheck_time")' "$workflow_file" +grep -Fq '"IBKR_MONITOR_DISPATCH_TARGETS_JSON"' "$workflow_file" +test "$(grep -Fc 'monitor_dispatch_targets_file=' "$workflow_file")" -eq 0 +test "$(grep -Fc 'shared_env_pairs+=("IBKR_MONITOR_DISPATCH_TARGETS_JSON=' "$workflow_file")" -eq 0 test "$(grep -Fc '"probe_time": scheduler.get("probe_time")' "$workflow_file")" -eq 0 grep -Fq 'Sync Cloud Scheduler schedule' "$workflow_file" grep -Fq 'scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}"' "$workflow_file" @@ -134,7 +134,8 @@ grep -Fq 'scheduler = target.get("scheduler") or {}' "$workflow_file" grep -Fq 'timezone = str(scheduler.get("timezone") or env.get("IBKR_MARKET_TIMEZONE") or "").strip()' "$workflow_file" grep -Fq 'timezone = "Asia/Hong_Kong" if market == "HK" else "America/New_York"' "$workflow_file" grep -Fq 'configured_time("CLOUD_SCHEDULER_MAIN_TIME", "45 15")' "$workflow_file" -grep -Fq 'IFS=$'\''\t'\'' read -r cloud_run_service market_timezone main_time warmup_time main_attempt_deadline <<< "${update}"' "$workflow_file" +grep -Fq 'str(scheduler.get("precheck_time") or configured_time("CLOUD_SCHEDULER_PRECHECK_TIME", "45 9"))' "$workflow_file" +grep -Fq 'IFS=$'\''\t'\'' read -r cloud_run_service market_timezone main_time warmup_time precheck_time runtime_target_enabled main_attempt_deadline <<< "${update}"' "$workflow_file" grep -Fq 'scheduler_job_candidates+=("${cloud_run_service%-service}-scheduler")' "$workflow_file" grep -Fq 'scheduler_job_candidates+=("${cloud_run_service}-scheduler")' "$workflow_file" grep -Fq 'for candidate_job in "${scheduler_job_candidates[@]}"; do' "$workflow_file" @@ -154,9 +155,16 @@ grep -Fq 'main_attempt_deadline_args+=(--attempt-deadline="${main_attempt_deadli test "$(grep -Fc '"${main_attempt_deadline_args[@]}"' "$workflow_file")" -eq 2 grep -Fq -- '--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}"' "$workflow_file" grep -Fq -- '--oidc-token-audience="${service_url}"' "$workflow_file" -grep -Fq 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' "$workflow_file" -grep -Fq 'monitor_uri="${monitor_dispatch_base_url}/monitor-dispatch"' "$workflow_file" -grep -Fq -- '--schedule="*/5 * * * *"' "$workflow_file" +grep -Fq 'desired_precheck_schedule="$(CURRENT_SCHEDULE="${desired_schedule}" SCHEDULE_TIME="${precheck_time}" python - <<' "$workflow_file" +grep -Fq 'precheck_job_name="${cloud_run_service%-service}-precheck-scheduler"' "$workflow_file" +grep -Fq 'precheck_uri="${service_url}/dry-run"' "$workflow_file" +grep -Fq 'gcloud scheduler jobs update http "${precheck_job_name}"' "$workflow_file" +grep -Fq 'gcloud scheduler jobs create http "${precheck_job_name}"' "$workflow_file" +test "$(grep -Fc -- '--attempt-deadline=120s' "$workflow_file")" -eq 2 +test "$(grep -Fc -- '--max-retry-attempts=0' "$workflow_file")" -eq 2 +grep -Fq 'gcloud scheduler jobs resume "${precheck_job_name}"' "$workflow_file" +grep -Fq 'gcloud scheduler jobs pause "${precheck_job_name}"' "$workflow_file" +test "$(grep -Fc 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' "$workflow_file")" -eq 0 grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_TO"' "$workflow_file" grep -Fq '"CRISIS_ALERT_GOOGLE_VOICE_SENDER_PASSWORD"' "$workflow_file" From eddda761653697969d97f9ebbae6ac95e0ecccfc Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:07:47 +0800 Subject: [PATCH 2/6] fix: harden precheck scheduler rollout Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 44 +++++++++++++++++++---- tests/test_scheduler_deadline_contract.py | 14 ++++++++ tests/test_sync_cloud_run_env_workflow.sh | 6 ++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 62422e2..d95d083 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -536,6 +536,32 @@ jobs: done done + - name: Pause legacy monitor dispatcher + if: steps.config.outputs.env_sync_enabled == 'true' + run: | + set -euo pipefail + + if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then + echo "Skipping global dispatcher retirement for the isolated HK verify target." + exit 0 + fi + + scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" + legacy_monitor_job="interactive-brokers-monitor-dispatcher-scheduler" + legacy_monitor_state="$(gcloud scheduler jobs describe "${legacy_monitor_job}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --format='value(state)' 2>/dev/null || true)" + if [ -z "${legacy_monitor_state}" ] || [ "${legacy_monitor_state}" = "PAUSED" ]; then + exit 0 + fi + + echo "Pausing legacy Cloud Scheduler job ${legacy_monitor_job} before removing its target configuration." + gcloud scheduler jobs pause "${legacy_monitor_job}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --quiet + - name: Sync Cloud Run environment if: steps.config.outputs.env_sync_enabled == 'true' env: @@ -1192,6 +1218,7 @@ jobs: --oidc-token-audience="${service_url}" \ --attempt-deadline=120s \ --max-retry-attempts=0 \ + --max-retry-duration=0s \ --quiet else echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}." @@ -1206,6 +1233,7 @@ jobs: --oidc-token-audience="${service_url}" \ --attempt-deadline=120s \ --max-retry-attempts=0 \ + --max-retry-duration=0s \ --quiet fi if [ "${precheck_state}" = "PAUSED" ]; then @@ -1227,12 +1255,16 @@ jobs: esac done - python3 scripts/reconcile_cloud_runtime.py \ - --platform=ibkr \ - --project="${GCP_PROJECT_ID}" \ - --region="${CLOUD_RUN_REGION}" \ - --scheduler-location="${scheduler_location}" \ - --delete-legacy-schedulers + reconcile_args=( + --platform=ibkr + --project="${GCP_PROJECT_ID}" + --region="${CLOUD_RUN_REGION}" + --scheduler-location="${scheduler_location}" + ) + if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then + reconcile_args+=(--delete-legacy-schedulers) + fi + python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}" - name: Prune old Cloud Run revisions if: steps.config.outputs.enabled == 'true' diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index a444802..2e64bba 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -52,6 +52,7 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None: assert 'precheck_uri="${service_url}/dry-run"' in workflow assert workflow.count("--attempt-deadline=120s") == 2 assert workflow.count("--max-retry-attempts=0") == 2 + assert workflow.count("--max-retry-duration=0s") == 2 assert 'gcloud scheduler jobs resume "${precheck_job_name}"' in workflow assert 'gcloud scheduler jobs pause "${precheck_job_name}"' in workflow assert 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' not in workflow @@ -62,3 +63,16 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None: assert strategy_deadline is not None assert report_grace is not None assert 120 > int(strategy_deadline.group(1)) + int(report_grace.group(1)) + + +def test_legacy_dispatcher_is_paused_before_removing_its_targets() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + + pause_step = workflow.index("- name: Pause legacy monitor dispatcher") + remove_targets = workflow.index('"IBKR_MONITOR_DISPATCH_TARGETS_JSON"') + + assert pause_step < remove_targets + assert 'gcloud scheduler jobs pause "${legacy_monitor_job}"' in workflow + assert 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' in workflow + assert 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' in workflow + assert 'reconcile_args+=(--delete-legacy-schedulers)' in workflow diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 2e60897..5c6034b 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -89,6 +89,11 @@ grep -Fq 'python3 scripts/reconcile_cloud_runtime.py' "$workflow_file" grep -Fq -- '--platform=ibkr' "$workflow_file" grep -Fq -- '--ensure-latest-traffic' "$workflow_file" grep -Fq -- '--delete-legacy-schedulers' "$workflow_file" +grep -Fq 'Pause legacy monitor dispatcher' "$workflow_file" +grep -Fq 'gcloud scheduler jobs pause "${legacy_monitor_job}"' "$workflow_file" +grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' "$workflow_file" +grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' "$workflow_file" +grep -Fq 'reconcile_args+=(--delete-legacy-schedulers)' "$workflow_file" grep -Fq 'emit_target_env_pairs()' "$workflow_file" grep -Fq 'emit_target_remove_env_vars()' "$workflow_file" @@ -162,6 +167,7 @@ grep -Fq 'gcloud scheduler jobs update http "${precheck_job_name}"' "$workflow_f grep -Fq 'gcloud scheduler jobs create http "${precheck_job_name}"' "$workflow_file" test "$(grep -Fc -- '--attempt-deadline=120s' "$workflow_file")" -eq 2 test "$(grep -Fc -- '--max-retry-attempts=0' "$workflow_file")" -eq 2 +test "$(grep -Fc -- '--max-retry-duration=0s' "$workflow_file")" -eq 2 grep -Fq 'gcloud scheduler jobs resume "${precheck_job_name}"' "$workflow_file" grep -Fq 'gcloud scheduler jobs pause "${precheck_job_name}"' "$workflow_file" test "$(grep -Fc 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' "$workflow_file")" -eq 0 From 8e27faba839e879bd47127f2243f8bdeae852059 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:15:46 +0800 Subject: [PATCH 3/6] fix: make precheck migration fail safe Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 49 +++++++++-------------- scripts/reconcile_cloud_runtime.py | 5 +++ tests/test_reconcile_cloud_runtime.py | 31 ++++++++++++++ tests/test_scheduler_deadline_contract.py | 17 ++++---- tests/test_sync_cloud_run_env_workflow.sh | 10 ++--- 5 files changed, 69 insertions(+), 43 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index d95d083..e8adc7a 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -536,32 +536,6 @@ jobs: done done - - name: Pause legacy monitor dispatcher - if: steps.config.outputs.env_sync_enabled == 'true' - run: | - set -euo pipefail - - if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then - echo "Skipping global dispatcher retirement for the isolated HK verify target." - exit 0 - fi - - scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" - legacy_monitor_job="interactive-brokers-monitor-dispatcher-scheduler" - legacy_monitor_state="$(gcloud scheduler jobs describe "${legacy_monitor_job}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" \ - --format='value(state)' 2>/dev/null || true)" - if [ -z "${legacy_monitor_state}" ] || [ "${legacy_monitor_state}" = "PAUSED" ]; then - exit 0 - fi - - echo "Pausing legacy Cloud Scheduler job ${legacy_monitor_job} before removing its target configuration." - gcloud scheduler jobs pause "${legacy_monitor_job}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" \ - --quiet - - name: Sync Cloud Run environment if: steps.config.outputs.env_sync_enabled == 'true' env: @@ -640,7 +614,6 @@ jobs: shared_remove_env_vars=( "IB_GATEWAY_HOST" "IB_GATEWAY_PORT" - "IBKR_MONITOR_DISPATCH_TARGETS_JSON" "TELEGRAM_CHAT_ID" "IB_CLIENT_ID" "IB_GATEWAY_INSTANCE_NAME" @@ -1260,12 +1233,30 @@ jobs: --project="${GCP_PROJECT_ID}" --region="${CLOUD_RUN_REGION}" --scheduler-location="${scheduler_location}" + --delete-legacy-schedulers ) - if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then - reconcile_args+=(--delete-legacy-schedulers) + if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then + reconcile_args+=(--preserve-shared-monitor-dispatcher) fi python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}" + for update in "${scheduler_updates[@]}"; do + IFS=$'\t' read -r cloud_run_service _ <<< "${update}" + legacy_env_names="$(gcloud run services describe "${cloud_run_service}" \ + --project="${GCP_PROJECT_ID}" \ + --region="${CLOUD_RUN_REGION}" \ + --format='value(spec.template.spec.containers[0].env[].name)')" + if ! printf '%s\n' "${legacy_env_names}" | tr ';' '\n' | grep -Fxq 'IBKR_MONITOR_DISPATCH_TARGETS_JSON'; then + continue + fi + echo "Removing retired monitor dispatcher configuration from ${cloud_run_service}." + gcloud run services update "${cloud_run_service}" \ + --project="${GCP_PROJECT_ID}" \ + --region="${CLOUD_RUN_REGION}" \ + --remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON \ + --quiet + done + - name: Prune old Cloud Run revisions if: steps.config.outputs.enabled == 'true' env: diff --git a/scripts/reconcile_cloud_runtime.py b/scripts/reconcile_cloud_runtime.py index c17d11b..eb071e4 100755 --- a/scripts/reconcile_cloud_runtime.py +++ b/scripts/reconcile_cloud_runtime.py @@ -328,6 +328,7 @@ def delete_legacy_schedulers( targets: Sequence[RuntimeTarget], env: Mapping[str, str], dry_run: bool, + preserve_shared_monitor_dispatcher: bool = False, ) -> None: locations = _scheduler_locations( region=region, @@ -337,6 +338,8 @@ def delete_legacy_schedulers( ) for target in targets: for job in _legacy_jobs_for_target(platform, target): + if preserve_shared_monitor_dispatcher and job == "interactive-brokers-monitor-dispatcher-scheduler": + continue for location in locations: if not _run_optional( [ @@ -376,6 +379,7 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: parser.add_argument("--expected-commit", default=os.environ.get("GITHUB_SHA", "")) parser.add_argument("--ensure-latest-traffic", action="store_true") parser.add_argument("--delete-legacy-schedulers", action="store_true") + parser.add_argument("--preserve-shared-monitor-dispatcher", action="store_true") parser.add_argument("--dry-run", action="store_true") return parser.parse_args(argv) @@ -404,6 +408,7 @@ def main(argv: Sequence[str] | None = None) -> int: targets=targets, env=os.environ, dry_run=args.dry_run, + preserve_shared_monitor_dispatcher=args.preserve_shared_monitor_dispatcher, ) return 0 diff --git a/tests/test_reconcile_cloud_runtime.py b/tests/test_reconcile_cloud_runtime.py index d5d4fe7..5dfcbb1 100644 --- a/tests/test_reconcile_cloud_runtime.py +++ b/tests/test_reconcile_cloud_runtime.py @@ -221,6 +221,37 @@ def fake_run(args, *, json_output=False, dry_run=False): self.assertEqual({call[4] for call in describe_calls}, expected_jobs) self.assertEqual({call[4] for call in delete_calls}, expected_jobs) + def test_delete_legacy_schedulers_can_preserve_shared_dispatcher(self) -> None: + target = rcr.RuntimeTarget(service_name="interactive-brokers-hk-verify-service") + described_jobs: list[str] = [] + deleted_jobs: list[str] = [] + + def fake_run_optional(args, *, dry_run=False): + described_jobs.append(args[4]) + return True + + def fake_run(args, *, json_output=False, dry_run=False): + deleted_jobs.append(args[4]) + return "" + + with mock.patch.object(rcr, "_run_optional", side_effect=fake_run_optional), mock.patch.object( + rcr, "_run", side_effect=fake_run + ): + rcr.delete_legacy_schedulers( + platform="ibkr", + project="interactivebrokersquant", + region="us-central1", + scheduler_location="us-central1", + targets=[target], + env={}, + dry_run=False, + preserve_shared_monitor_dispatcher=True, + ) + + self.assertNotIn("interactive-brokers-monitor-dispatcher-scheduler", described_jobs) + self.assertNotIn("interactive-brokers-monitor-dispatcher-scheduler", deleted_jobs) + self.assertIn("interactive-brokers-hk-verify-service-precheck-scheduler", deleted_jobs) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index 2e64bba..8e4861c 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -65,14 +65,15 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None: assert 120 > int(strategy_deadline.group(1)) + int(report_grace.group(1)) -def test_legacy_dispatcher_is_paused_before_removing_its_targets() -> None: +def test_legacy_dispatcher_is_retired_after_replacement_jobs_are_ready() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") - pause_step = workflow.index("- name: Pause legacy monitor dispatcher") - remove_targets = workflow.index('"IBKR_MONITOR_DISPATCH_TARGETS_JSON"') + env_sync_start = workflow.index("shared_remove_env_vars=(") + env_sync_end = workflow.index("shared_remove_secret_vars=(") + replacement_jobs = workflow.index('precheck_job_name="${cloud_run_service%-service}-precheck-scheduler"') + retire_legacy = workflow.index('python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}"') + remove_targets = workflow.index("--remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON") - assert pause_step < remove_targets - assert 'gcloud scheduler jobs pause "${legacy_monitor_job}"' in workflow - assert 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' in workflow - assert 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' in workflow - assert 'reconcile_args+=(--delete-legacy-schedulers)' in workflow + assert "IBKR_MONITOR_DISPATCH_TARGETS_JSON" not in workflow[env_sync_start:env_sync_end] + assert replacement_jobs < retire_legacy < remove_targets + assert 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' in workflow diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 5c6034b..198b439 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -89,11 +89,9 @@ grep -Fq 'python3 scripts/reconcile_cloud_runtime.py' "$workflow_file" grep -Fq -- '--platform=ibkr' "$workflow_file" grep -Fq -- '--ensure-latest-traffic' "$workflow_file" grep -Fq -- '--delete-legacy-schedulers' "$workflow_file" -grep -Fq 'Pause legacy monitor dispatcher' "$workflow_file" -grep -Fq 'gcloud scheduler jobs pause "${legacy_monitor_job}"' "$workflow_file" -grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' "$workflow_file" -grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' "$workflow_file" -grep -Fq 'reconcile_args+=(--delete-legacy-schedulers)' "$workflow_file" +grep -Fq ' --delete-legacy-schedulers' "$workflow_file" +grep -Fq 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' "$workflow_file" +grep -Fq -- '--remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON' "$workflow_file" grep -Fq 'emit_target_env_pairs()' "$workflow_file" grep -Fq 'emit_target_remove_env_vars()' "$workflow_file" @@ -124,7 +122,7 @@ grep -Fq -- '--max-instances 1' "$workflow_file" grep -Fq -- '--remove-env-vars "$(IFS=,; echo "${remove_env_vars[*]}")' "$workflow_file" grep -Fq -- '--update-env-vars "^|^$(join_by_delimiter "|" "${env_pairs[@]}")' "$workflow_file" grep -Fq -- '--update-labels "$(IFS=,; echo "${target_label_pairs[*]}")' "$workflow_file" -grep -Fq '"IBKR_MONITOR_DISPATCH_TARGETS_JSON"' "$workflow_file" +test "$(grep -Fc '"IBKR_MONITOR_DISPATCH_TARGETS_JSON"' "$workflow_file")" -eq 0 test "$(grep -Fc 'monitor_dispatch_targets_file=' "$workflow_file")" -eq 0 test "$(grep -Fc 'shared_env_pairs+=("IBKR_MONITOR_DISPATCH_TARGETS_JSON=' "$workflow_file")" -eq 0 test "$(grep -Fc '"probe_time": scheduler.get("probe_time")' "$workflow_file")" -eq 0 From 03f6e5b9fe2bf0fd3c00fbe8bddd1f7eb60943e0 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:23:09 +0800 Subject: [PATCH 4/6] fix: isolate HK scheduler migration Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 12 ++++++--- scripts/reconcile_cloud_runtime.py | 5 ---- tests/test_reconcile_cloud_runtime.py | 32 ----------------------- tests/test_scheduler_deadline_contract.py | 3 ++- tests/test_sync_cloud_run_env_workflow.sh | 3 ++- 5 files changed, 13 insertions(+), 42 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index e8adc7a..7338cab 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -1170,6 +1170,11 @@ jobs: --quiet fi + if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then + echo "Skipping precheck scheduler migration for the isolated HK verify target." + continue + fi + precheck_job_name="${cloud_run_service%-service}-precheck-scheduler" precheck_uri="${service_url}/dry-run" precheck_state="$(gcloud scheduler jobs describe "${precheck_job_name}" \ @@ -1228,6 +1233,10 @@ jobs: esac done + if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then + exit 0 + fi + reconcile_args=( --platform=ibkr --project="${GCP_PROJECT_ID}" @@ -1235,9 +1244,6 @@ jobs: --scheduler-location="${scheduler_location}" --delete-legacy-schedulers ) - if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then - reconcile_args+=(--preserve-shared-monitor-dispatcher) - fi python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}" for update in "${scheduler_updates[@]}"; do diff --git a/scripts/reconcile_cloud_runtime.py b/scripts/reconcile_cloud_runtime.py index eb071e4..c17d11b 100755 --- a/scripts/reconcile_cloud_runtime.py +++ b/scripts/reconcile_cloud_runtime.py @@ -328,7 +328,6 @@ def delete_legacy_schedulers( targets: Sequence[RuntimeTarget], env: Mapping[str, str], dry_run: bool, - preserve_shared_monitor_dispatcher: bool = False, ) -> None: locations = _scheduler_locations( region=region, @@ -338,8 +337,6 @@ def delete_legacy_schedulers( ) for target in targets: for job in _legacy_jobs_for_target(platform, target): - if preserve_shared_monitor_dispatcher and job == "interactive-brokers-monitor-dispatcher-scheduler": - continue for location in locations: if not _run_optional( [ @@ -379,7 +376,6 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: parser.add_argument("--expected-commit", default=os.environ.get("GITHUB_SHA", "")) parser.add_argument("--ensure-latest-traffic", action="store_true") parser.add_argument("--delete-legacy-schedulers", action="store_true") - parser.add_argument("--preserve-shared-monitor-dispatcher", action="store_true") parser.add_argument("--dry-run", action="store_true") return parser.parse_args(argv) @@ -408,7 +404,6 @@ def main(argv: Sequence[str] | None = None) -> int: targets=targets, env=os.environ, dry_run=args.dry_run, - preserve_shared_monitor_dispatcher=args.preserve_shared_monitor_dispatcher, ) return 0 diff --git a/tests/test_reconcile_cloud_runtime.py b/tests/test_reconcile_cloud_runtime.py index 5dfcbb1..5b3b9eb 100644 --- a/tests/test_reconcile_cloud_runtime.py +++ b/tests/test_reconcile_cloud_runtime.py @@ -221,37 +221,5 @@ def fake_run(args, *, json_output=False, dry_run=False): self.assertEqual({call[4] for call in describe_calls}, expected_jobs) self.assertEqual({call[4] for call in delete_calls}, expected_jobs) - def test_delete_legacy_schedulers_can_preserve_shared_dispatcher(self) -> None: - target = rcr.RuntimeTarget(service_name="interactive-brokers-hk-verify-service") - described_jobs: list[str] = [] - deleted_jobs: list[str] = [] - - def fake_run_optional(args, *, dry_run=False): - described_jobs.append(args[4]) - return True - - def fake_run(args, *, json_output=False, dry_run=False): - deleted_jobs.append(args[4]) - return "" - - with mock.patch.object(rcr, "_run_optional", side_effect=fake_run_optional), mock.patch.object( - rcr, "_run", side_effect=fake_run - ): - rcr.delete_legacy_schedulers( - platform="ibkr", - project="interactivebrokersquant", - region="us-central1", - scheduler_location="us-central1", - targets=[target], - env={}, - dry_run=False, - preserve_shared_monitor_dispatcher=True, - ) - - self.assertNotIn("interactive-brokers-monitor-dispatcher-scheduler", described_jobs) - self.assertNotIn("interactive-brokers-monitor-dispatcher-scheduler", deleted_jobs) - self.assertIn("interactive-brokers-hk-verify-service-precheck-scheduler", deleted_jobs) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index 8e4861c..c1eb77f 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -76,4 +76,5 @@ def test_legacy_dispatcher_is_retired_after_replacement_jobs_are_ready() -> None assert "IBKR_MONITOR_DISPATCH_TARGETS_JSON" not in workflow[env_sync_start:env_sync_end] assert replacement_jobs < retire_legacy < remove_targets - assert 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' in workflow + assert 'Skipping precheck scheduler migration for the isolated HK verify target.' in workflow + assert 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' not in workflow diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 198b439..4878928 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -90,7 +90,8 @@ grep -Fq -- '--platform=ibkr' "$workflow_file" grep -Fq -- '--ensure-latest-traffic' "$workflow_file" grep -Fq -- '--delete-legacy-schedulers' "$workflow_file" grep -Fq ' --delete-legacy-schedulers' "$workflow_file" -grep -Fq 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' "$workflow_file" +grep -Fq 'Skipping precheck scheduler migration for the isolated HK verify target.' "$workflow_file" +test "$(grep -Fc -- '--preserve-shared-monitor-dispatcher' "$workflow_file")" -eq 0 grep -Fq -- '--remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON' "$workflow_file" grep -Fq 'emit_target_env_pairs()' "$workflow_file" From a2ddeba751b7ea9144e2f55e8902420f5c304490 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:32:57 +0800 Subject: [PATCH 5/6] fix: preserve disabled precheck contracts Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 64 +++++++++++------------ tests/test_scheduler_deadline_contract.py | 11 ++++ 2 files changed, 43 insertions(+), 32 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 7338cab..0c5bb50 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -1181,39 +1181,39 @@ jobs: --project="${GCP_PROJECT_ID}" \ --location="${scheduler_location}" \ --format='value(state)' 2>/dev/null || true)" + if [ -n "${precheck_state}" ]; then + echo "Updating Cloud Scheduler precheck ${precheck_job_name} to ${desired_precheck_schedule}." + gcloud scheduler jobs update http "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --uri="${precheck_uri}" \ + --schedule="${desired_precheck_schedule}" \ + --time-zone="${market_timezone}" \ + --http-method=POST \ + --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ + --oidc-token-audience="${service_url}" \ + --attempt-deadline=120s \ + --max-retry-attempts=0 \ + --max-retry-duration=0s \ + --quiet + else + echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}." + gcloud scheduler jobs create http "${precheck_job_name}" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" \ + --uri="${precheck_uri}" \ + --schedule="${desired_precheck_schedule}" \ + --time-zone="${market_timezone}" \ + --http-method=POST \ + --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ + --oidc-token-audience="${service_url}" \ + --attempt-deadline=120s \ + --max-retry-attempts=0 \ + --max-retry-duration=0s \ + --quiet + fi case "${runtime_target_enabled}" in 1|true|yes|on) - if [ -n "${precheck_state}" ]; then - echo "Updating Cloud Scheduler precheck ${precheck_job_name} to ${desired_precheck_schedule}." - gcloud scheduler jobs update http "${precheck_job_name}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" \ - --uri="${precheck_uri}" \ - --schedule="${desired_precheck_schedule}" \ - --time-zone="${market_timezone}" \ - --http-method=POST \ - --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ - --oidc-token-audience="${service_url}" \ - --attempt-deadline=120s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ - --quiet - else - echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}." - gcloud scheduler jobs create http "${precheck_job_name}" \ - --project="${GCP_PROJECT_ID}" \ - --location="${scheduler_location}" \ - --uri="${precheck_uri}" \ - --schedule="${desired_precheck_schedule}" \ - --time-zone="${market_timezone}" \ - --http-method=POST \ - --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ - --oidc-token-audience="${service_url}" \ - --attempt-deadline=120s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ - --quiet - fi if [ "${precheck_state}" = "PAUSED" ]; then gcloud scheduler jobs resume "${precheck_job_name}" \ --project="${GCP_PROJECT_ID}" \ @@ -1222,7 +1222,7 @@ jobs: fi ;; *) - if [ -n "${precheck_state}" ] && [ "${precheck_state}" != "PAUSED" ]; then + if [ "${precheck_state}" != "PAUSED" ]; then echo "Pausing Cloud Scheduler precheck ${precheck_job_name} because ${cloud_run_service} is disabled." gcloud scheduler jobs pause "${precheck_job_name}" \ --project="${GCP_PROJECT_ID}" \ diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index c1eb77f..a7149f2 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -65,6 +65,17 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None: assert 120 > int(strategy_deadline.group(1)) + int(report_grace.group(1)) +def test_disabled_target_gets_a_paused_canonical_precheck_before_legacy_cleanup() -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + + ensure_precheck = workflow.index('if [ -n "${precheck_state}" ]; then') + enabled_state = workflow.index('case "${runtime_target_enabled}" in') + pause_precheck = workflow.index('gcloud scheduler jobs pause "${precheck_job_name}"') + retire_legacy = workflow.index('python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}"') + + assert ensure_precheck < enabled_state < pause_precheck < retire_legacy + + def test_legacy_dispatcher_is_retired_after_replacement_jobs_are_ready() -> None: workflow = WORKFLOW.read_text(encoding="utf-8") From 54cb1eb323afa873f050f3ec1a0c27cf3c693ba7 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:39:45 +0800 Subject: [PATCH 6/6] fix: gate isolated HK precheck migration Co-Authored-By: Codex --- .github/workflows/sync-cloud-run-env.yml | 20 +++++++++++--------- tests/test_scheduler_deadline_contract.py | 5 +++-- tests/test_sync_cloud_run_env_workflow.sh | 3 ++- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 0c5bb50..b557559 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -422,6 +422,17 @@ jobs: project_id: ${{ env.GCP_PROJECT_ID }} version: ">= 416.0.0" + - name: Require configured scheduler migration before HK verify + if: steps.config.outputs.env_sync_enabled == 'true' && env.WORKFLOW_TARGET == 'hk-verify' + run: | + set -euo pipefail + scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" + if gcloud scheduler jobs describe "interactive-brokers-monitor-dispatcher-scheduler" \ + --project="${GCP_PROJECT_ID}" \ + --location="${scheduler_location}" >/dev/null 2>&1; then + echo "HK verify scheduler migration requires the configured migration first." >&2 + exit 1 + fi - name: Build, push, and deploy Cloud Run image run: | @@ -1170,11 +1181,6 @@ jobs: --quiet fi - if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then - echo "Skipping precheck scheduler migration for the isolated HK verify target." - continue - fi - precheck_job_name="${cloud_run_service%-service}-precheck-scheduler" precheck_uri="${service_url}/dry-run" precheck_state="$(gcloud scheduler jobs describe "${precheck_job_name}" \ @@ -1233,10 +1239,6 @@ jobs: esac done - if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then - exit 0 - fi - reconcile_args=( --platform=ibkr --project="${GCP_PROJECT_ID}" diff --git a/tests/test_scheduler_deadline_contract.py b/tests/test_scheduler_deadline_contract.py index a7149f2..777f222 100644 --- a/tests/test_scheduler_deadline_contract.py +++ b/tests/test_scheduler_deadline_contract.py @@ -81,11 +81,12 @@ def test_legacy_dispatcher_is_retired_after_replacement_jobs_are_ready() -> None env_sync_start = workflow.index("shared_remove_env_vars=(") env_sync_end = workflow.index("shared_remove_secret_vars=(") + hk_migration_guard = workflow.index("HK verify scheduler migration requires the configured migration first.") replacement_jobs = workflow.index('precheck_job_name="${cloud_run_service%-service}-precheck-scheduler"') retire_legacy = workflow.index('python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}"') remove_targets = workflow.index("--remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON") assert "IBKR_MONITOR_DISPATCH_TARGETS_JSON" not in workflow[env_sync_start:env_sync_end] - assert replacement_jobs < retire_legacy < remove_targets - assert 'Skipping precheck scheduler migration for the isolated HK verify target.' in workflow + assert hk_migration_guard < replacement_jobs < retire_legacy < remove_targets + assert 'Skipping precheck scheduler migration for the isolated HK verify target.' not in workflow assert 'reconcile_args+=(--preserve-shared-monitor-dispatcher)' not in workflow diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 4878928..43a1c36 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -90,7 +90,8 @@ grep -Fq -- '--platform=ibkr' "$workflow_file" grep -Fq -- '--ensure-latest-traffic' "$workflow_file" grep -Fq -- '--delete-legacy-schedulers' "$workflow_file" grep -Fq ' --delete-legacy-schedulers' "$workflow_file" -grep -Fq 'Skipping precheck scheduler migration for the isolated HK verify target.' "$workflow_file" +grep -Fq 'HK verify scheduler migration requires the configured migration first.' "$workflow_file" +test "$(grep -Fc 'Skipping precheck scheduler migration for the isolated HK verify target.' "$workflow_file")" -eq 0 test "$(grep -Fc -- '--preserve-shared-monitor-dispatcher' "$workflow_file")" -eq 0 grep -Fq -- '--remove-env-vars=IBKR_MONITOR_DISPATCH_TARGETS_JSON' "$workflow_file"