Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 64 additions & 7 deletions scripts/cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ def _load_services() -> list[str]:
for target in targets:
if not isinstance(target, dict):
continue
if not _target_enabled(target):
continue
Comment on lines +59 to +60

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Exclude disabled targets already listed in env

When CLOUD_RUN_SERVICE or CLOUD_RUN_SERVICES is still set for a service that is disabled in CLOUD_RUN_SERVICE_TARGETS_JSON, the service has already been appended before this continue, so the guard still queries and alerts on it. Since the runtime-guard workflow supplies the legacy service envs alongside target JSON, disabling a target will not suppress monitoring unless those legacy vars are also cleared.

Useful? React with 👍 / 👎.

runtime_target = target.get("runtime_target") or target.get(
"runtime_target_json"
)
Expand Down Expand Up @@ -85,19 +87,43 @@ def _load_services() -> list[str]:
return unique


def _service_job_aliases(service: str) -> list[str]:
service_name = str(service or "").strip()
if not service_name:
return []
aliases = [service_name]
if service_name.endswith("-service"):
aliases.append(service_name.removesuffix("-service"))
return list(dict.fromkeys(aliases))


def _scheduler_job_pattern_for_services(services: list[str]) -> str:
candidates: list[str] = []
for service in services:
service_name = str(service or "").strip()
if not service_name:
continue
candidates.append(service_name)
if service_name.endswith("-service"):
candidates.append(service_name.removesuffix("-service"))
candidates.extend(_service_job_aliases(service))
unique = list(dict.fromkeys(candidates))
return "|".join(re.escape(candidate) for candidate in unique)


def _entry_job_name(entry: dict[str, Any]) -> str:
labels = _labels(entry)
return str(labels.get("job_id") or labels.get("job_name") or "")


def _scheduler_entry_since(
entry: dict[str, Any],
service_since_by_name: dict[str, dt.datetime],
fallback: dt.datetime,
) -> dt.datetime:
job_name = _entry_job_name(entry)
matches = [
service_since
for service, service_since in service_since_by_name.items()
if any(alias and alias in job_name for alias in _service_job_aliases(service))
]
return max(matches) if matches else fallback
Comment on lines +120 to +124

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Match scheduler jobs by exact alias boundary

When multiple monitored services share a prefix, such as foo-service and foo-bar-service, this substring match makes foo-bar-service-scheduler match both foo and foo-bar-service; because the function then uses max(matches), a real failure for foo-bar-service before foo's later revision time is skipped as stale. Match the job id to a concrete scheduler name or alias boundary instead of using unbounded substring containment.

Useful? React with 👍 / 👎.



def _run_gcloud(args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(args, text=True, capture_output=True, check=False)

Expand Down Expand Up @@ -184,6 +210,27 @@ def _runtime_target(target: dict[str, Any]) -> dict[str, Any]:
return runtime_target if isinstance(runtime_target, dict) else {}


def _coerce_bool(value: Any, default: bool) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if not text:
return default
return text in {"1", "true", "yes", "y", "on"}
Comment on lines +218 to +221

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Reject invalid target enabled values

When a target has a typo such as "RUNTIME_TARGET_ENABLED":"treu" or "runtime_target_enabled":"maybe", this helper treats the value as false, so _load_services() silently drops the target and the guard can report OK instead of surfacing a configuration error. The same setting is validated elsewhere as true/false, so invalid target JSON should not disable monitoring.

Useful? React with 👍 / 👎.



def _target_enabled(target: dict[str, Any]) -> bool:
runtime_target = _runtime_target(target)
for key in ("runtime_target_enabled", "RUNTIME_TARGET_ENABLED"):
if key in target:
return _coerce_bool(target.get(key), True)
if key in runtime_target:
return _coerce_bool(runtime_target.get(key), True)
Comment on lines +226 to +230

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor disabled flags from target env blocks

When CLOUD_RUN_SERVICE_TARGETS_JSON uses the same schema as the env-sync plan, RUNTIME_TARGET_ENABLED can live under a target's env block or defaults, but this new guard only checks the top-level target and runtime_target. In that configuration, e.g. {"env":{"RUNTIME_TARGET_ENABLED":"false"}}, _load_services() still includes the disabled service and the runtime guard continues querying and alerting on it.

Useful? React with 👍 / 👎.

return True


def _target_service_names(target: dict[str, Any]) -> list[str]:
runtime_target = _runtime_target(target)
for key in ("service", "service_name", "cloud_run_service"):
Expand Down Expand Up @@ -428,6 +475,7 @@ def main() -> int:
issues: list[str] = []
details: list[str] = []
success_count = 0
service_since_by_name: dict[str, dt.datetime] = {}

try:
services = _load_services()
Expand All @@ -441,6 +489,7 @@ def main() -> int:

for service in services:
service_since = _cloud_run_log_since(project, service, since) if ignore_pre_ready_logs else since
service_since_by_name[service] = service_since
service_since_text = _format_timestamp(service_since)
log_filter = (
'resource.type="cloud_run_revision" '
Expand Down Expand Up @@ -474,7 +523,15 @@ def main() -> int:
for entry in entries
if regex.search(str(_labels(entry).get("job_id") or _labels(entry).get("job_name") or ""))
]
failures = [entry for entry in entries if _is_failure(entry)]
failures = []
for entry in entries:
if not _is_failure(entry):
continue
entry_timestamp = _parse_timestamp(entry.get("timestamp"))
entry_since = _scheduler_entry_since(entry, service_since_by_name, since)
if entry_timestamp and entry_timestamp < entry_since:
Comment on lines +530 to +532

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Apply revision windows to custom scheduler patterns

When RUNTIME_GUARD_SCHEDULER_JOB_PATTERN is configured for a job name that does not contain the Cloud Run service alias, entries matching that explicit pattern reach this loop but _scheduler_entry_since() falls back to the global lookback window. In that setup, scheduler failures from before the service's latest-ready revision are still reported as current, so the stale-log filter does not work for custom scheduler job names.

Useful? React with 👍 / 👎.

continue
failures.append(entry)
if failures:
issues.append(f"{len(failures)} Cloud Scheduler failure log(s)")
details.extend(_summarize(entry) for entry in failures[:5])
Expand Down
35 changes: 35 additions & 0 deletions tests/test_cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,38 @@ def test_region_for_service_prefers_target_region(monkeypatch):
)

assert guard._region_for_service("firstrade-service") == "asia-east1"


def test_load_services_ignores_disabled_runtime_targets(monkeypatch):
monkeypatch.delenv("RUNTIME_GUARD_CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICE", raising=False)
monkeypatch.setenv(
"CLOUD_RUN_SERVICE_TARGETS_JSON",
json.dumps(
{
"targets": [
{"service": "enabled-service", "RUNTIME_TARGET_ENABLED": "true"},
{"service": "disabled-service", "RUNTIME_TARGET_ENABLED": "false"},
{"service": "disabled-lower-service", "runtime_target_enabled": "false"},
]
}
),
)

assert guard._load_services() == ["enabled-service"]


def test_scheduler_entry_since_uses_matching_service_revision_window():
fallback = dt.datetime(2026, 7, 1, 1, 0, tzinfo=dt.timezone.utc)
service_since = dt.datetime(2026, 7, 1, 2, 0, tzinfo=dt.timezone.utc)
entry = {"resource": {"labels": {"job_id": "enabled-service-scheduler"}}}

assert (
guard._scheduler_entry_since(entry, {"enabled-service": service_since}, fallback)
== service_since
)
assert (
guard._scheduler_entry_since(entry, {"other-service": service_since}, fallback)
== fallback
)
Loading