From 0eedf6796f098c9311c6fb6b91ea4e2dec1d7224 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 2 Jul 2026 03:04:56 +0800 Subject: [PATCH] fix: harden Firstrade runtime audit gates --- .github/workflows/ci.yml | 8 +++-- Dockerfile | 5 ++- constraints.txt | 2 +- main.py | 52 +++++++++++++++++++++------- notifications/telegram.py | 14 ++++++-- pyproject.toml | 4 ++- requirements.txt | 2 +- runtime_config_support.py | 24 ++++++++----- scripts/check_qpk_pin_consistency.py | 12 ++++--- tests/test_request_handling.py | 37 ++++++++++++++++++++ 10 files changed, 126 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 259ecd1..1a1307e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,6 @@ jobs: run: | set -euo pipefail python -m pip install --upgrade pip - python -m pip install --upgrade pip # Install non-git packages first to avoid cross-package dependency conflicts grep -vE "^\s*(#|$)|git\+" requirements.txt | xargs -r python -m pip install -c https://raw.githubusercontent.com/QuantStrategyLab/QuantPlatformKit/main/constraints.txt # Install git-based packages individually with --no-deps @@ -91,11 +90,16 @@ jobs: set -euo pipefail python -m pip install --no-deps -e external/QuantPlatformKit -e external/UsEquityStrategies + - name: Verify Python dependencies + run: python -m pip check + - name: Run ruff run: | set -euo pipefail ruff check --exclude external . + - name: Run tests + run: python -m pytest -q + - name: Check QPK pin consistency run: python scripts/check_qpk_pin_consistency.py - continue-on-error: true diff --git a/Dockerfile b/Dockerfile index 3bd9434..fbbd8f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,10 @@ RUN apt-get update \ COPY requirements.txt ./ COPY constraints.txt ./ RUN python -m pip install --upgrade pip \ - && python -m pip install -r requirements.txt -c constraints.txt \ + && grep -vE "^\s*(#|$)|git\+" requirements.txt | xargs -r python -m pip install -c constraints.txt \ + && grep "git+" requirements.txt | while IFS= read -r pkg; do \ + [ -n "$pkg" ] && python -m pip install --no-deps "$pkg"; \ + done \ && apt-get purge -y git \ && apt-get autoremove -y --purge \ && rm -rf /var/lib/apt/lists/* diff --git a/constraints.txt b/constraints.txt index 2a6081a..742aa4a 100644 --- a/constraints.txt +++ b/constraints.txt @@ -3,7 +3,7 @@ # Generated: 2026-07-01 # Auto-updated by update-qpk-pin.yml on every push to QPK main. -quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@d786c1140967f0e96e35599d057f0655e5a9ba25 +quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@b0eacd2fe4884f7f2447b704a232e9a121f396c4 us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@c8df5f9659340965bd7f53998892ed1018ed4254 hk-equity-strategies @ git+https://github.com/QuantStrategyLab/HkEquityStrategies.git@700c8b19c46336d3d8fcba687e58553afcf0235f cn-equity-strategies @ git+https://github.com/QuantStrategyLab/CnEquityStrategies.git@e09fd557c2bd9ae5f4d44228915c4e52c4b0dd21 diff --git a/main.py b/main.py index 6bd987c..c8664f9 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import re import traceback from dataclasses import replace from datetime import datetime, timezone @@ -42,6 +43,16 @@ app = Flask(__name__) register_health_endpoint(app) # GET /health /healthz +_REDACTED = "" +_TELEGRAM_BOT_PATH_RE = re.compile(r"(?i)(/bot)([^/\s]+)") +_SENSITIVE_QUERY_RE = re.compile( + r"(?i)([?&](?:access[_-]?token|api[_-]?key|auth[_-]?token|key|password|secret|signature|token)=)([^&\s]+)" +) +_AUTH_HEADER_RE = re.compile(r"(?i)\b(Bearer|Basic)\s+([A-Za-z0-9._~+/=-]{8,})") +_ASSIGNMENT_RE = re.compile( + r"(?i)\b(api[_-]?key|auth[_-]?token|credential|password|private[_-]?key|secret|token)\s*[:=]\s*([\"']?)([^\"'\s,;]{8,})([\"']?)" +) + def get_project_id() -> str | None: return os.getenv("GOOGLE_CLOUD_PROJECT") @@ -59,6 +70,14 @@ def _split_env_list(value: str | None) -> tuple[str, ...]: ) +def redact_sensitive_text(value: object) -> str: + text = str(value) + text = _TELEGRAM_BOT_PATH_RE.sub(r"\1" + _REDACTED, text) + text = _SENSITIVE_QUERY_RE.sub(r"\1" + _REDACTED, text) + text = _AUTH_HEADER_RE.sub(r"\1 " + _REDACTED, text) + return _ASSIGNMENT_RE.sub(lambda match: f"{match.group(1)}={_REDACTED}", text) + + def _get_telegram_token() -> str: try: from quant_platform_kit.cloud import get_secret_store @@ -86,7 +105,7 @@ def _telegram_notification_targets() -> tuple[tuple[str, str], ...]: def _runtime_error_notification_message(exc: Exception) -> str: - error_text = f"{type(exc).__name__}: {exc}" + error_text = _safe_exception_text(exc, include_type=True) if len(error_text) > 1200: error_text = error_text[:1197] + "..." is_health_check = request.path == "/probe" @@ -131,13 +150,22 @@ def _notify_runtime_error(exc: Exception) -> bool: try: build_sender(token, chat_id)(message) except Exception as send_exc: # pragma: no cover - build_sender normally handles this. - print(f"Firstrade runtime error Telegram send failed: {send_exc}", flush=True) + print( + f"Firstrade runtime error Telegram send failed: {redact_sensitive_text(send_exc)}", + flush=True, + ) return attempted +def _safe_exception_text(exc: Exception, *, include_type: bool = False) -> str: + text = redact_sensitive_text(exc) + return f"{type(exc).__name__}: {text}" if include_type else text + + def _handle_strategy_run_exception(exc: Exception) -> bool: - print(f"Firstrade strategy run failed: {type(exc).__name__}: {exc}", flush=True) - traceback.print_exc() + print(f"Firstrade strategy run failed: {_safe_exception_text(exc, include_type=True)}", flush=True) + for line in traceback.format_exception(type(exc), exc, exc.__traceback__): + print(redact_sensitive_text(line.rstrip()), flush=True) return _notify_runtime_error(exc) @@ -412,7 +440,7 @@ def smoke(): } ) except FirstradePlatformError as exc: - return jsonify({"ok": False, "error": str(exc)}), 500 + return jsonify({"ok": False, "error": _safe_exception_text(exc)}), 500 def session_check(): @@ -437,7 +465,7 @@ def session_check(): jsonify( { "ok": False, - "error": str(exc), + "error": _safe_exception_text(exc), "runtime_error_notification_attempted": notification_attempted, } ), @@ -449,7 +477,7 @@ def session_check(): jsonify( { "ok": False, - "error": f"{type(exc).__name__}: {exc}", + "error": _safe_exception_text(exc, include_type=True), "runtime_error_notification_attempted": notification_attempted, } ), @@ -487,7 +515,7 @@ def run_strategy(): jsonify( { "ok": False, - "error": str(exc), + "error": _safe_exception_text(exc), "runtime_error_notification_attempted": notification_attempted, } ), @@ -499,7 +527,7 @@ def run_strategy(): jsonify( { "ok": False, - "error": f"{type(exc).__name__}: {exc}", + "error": _safe_exception_text(exc, include_type=True), "runtime_error_notification_attempted": notification_attempted, } ), @@ -527,7 +555,7 @@ def dry_run(): jsonify( { "ok": False, - "error": str(exc), + "error": _safe_exception_text(exc), "runtime_error_notification_attempted": notification_attempted, } ), @@ -539,7 +567,7 @@ def dry_run(): jsonify( { "ok": False, - "error": f"{type(exc).__name__}: {exc}", + "error": _safe_exception_text(exc, include_type=True), "runtime_error_notification_attempted": notification_attempted, } ), @@ -566,7 +594,7 @@ def monitor_dispatch(): jsonify( { "ok": False, - "error": f"{type(exc).__name__}: {exc}", + "error": _safe_exception_text(exc, include_type=True), "runtime_error_notification_attempted": notification_attempted, } ), diff --git a/notifications/telegram.py b/notifications/telegram.py index 92d8ae2..7a03363 100644 --- a/notifications/telegram.py +++ b/notifications/telegram.py @@ -1203,19 +1203,29 @@ def render_cycle_summary(result: Mapping[str, Any], *, lang: str = "en") -> str: account = str(result.get("account") or "").strip() lines = [translator("rebalance_title" if has_rebalance_attempt else "heartbeat_title")] if strategy_name: - lines.append(f"策略: {strategy_name} | 账户: {account}" if lang == "zh" else f"Strategy: {strategy_name} | Account: {account}") + lines.append(translator("strategy_label", name=strategy_name)) + if account: + lines.append(translator("account_label", account=account)) if dry_run_only: lines.append(translator("dry_run_banner")) if bool(result.get("execution_blocked")): blocked = list(result.get("execution_blocking_skips") or skipped) reason = _format_skipped_reason(blocked, translator=translator) - lines.append(translator("execution_blocked_banner", reason=reason)) + if bool(result.get("funding_blocked")): + banner_key = "funding_blocked_banner" + elif bool(result.get("execution_block_retryable")): + banner_key = "execution_blocked_retryable_banner" + else: + banner_key = "execution_blocked_banner" + lines.append(translator(banner_key, reason=reason)) dashboard_lines = _format_dashboard_lines(portfolio, execution, translator=translator) if dashboard_lines: lines.append(SEPARATOR) lines.extend(dashboard_lines) lines.append(SEPARATOR) + lines.extend(_format_timing_lines(execution, translator=translator)) + lines.extend(_format_signal_lines(execution, translator=translator)) lines.extend(target_diff_lines) lines.extend( str(line).strip() diff --git a/pyproject.toml b/pyproject.toml index d063d03..5fcfb60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,10 +40,12 @@ pythonpath = [ testpaths = ["tests"] [tool.ruff] -ignore = ["E701", "E702", "E741", "F841", "F821", "B008", "F401", "F601", "E402", "E401"] target-version = "py311" line-length = 100 +[tool.ruff.lint] +ignore = ["E701", "E702", "E741", "F841", "F821", "B008", "F401", "F601", "E402", "E401"] + [tool.ruff.lint.per-file-ignores] "tests/**" = ["E402", "F841"] diff --git a/requirements.txt b/requirements.txt index a706558..5efb380 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ pandas_market_calendars pytest pytz requests -quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@d786c1140967f0e96e35599d057f0655e5a9ba25 +quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@b0eacd2fe4884f7f2447b704a232e9a121f396c4 us-equity-strategies @ git+https://github.com/QuantStrategyLab/UsEquityStrategies.git@c8df5f9659340965bd7f53998892ed1018ed4254 diff --git a/runtime_config_support.py b/runtime_config_support.py index 71b81ac..b49e9ce 100644 --- a/runtime_config_support.py +++ b/runtime_config_support.py @@ -41,16 +41,16 @@ IBIT_SMART_DCA_PROFILE = "ibit_smart_dca" -def _get_credential(secret_name: str, env_var: str) -> str: +def _get_credential(secret_name: str, env_var: str) -> str | None: + val = os.environ.get(env_var) + if val is not None: + return val try: from quant_platform_kit.cloud import get_secret_store return get_secret_store().get_secret(secret_name, project_id="firstradequant") except Exception: - val = os.environ.get(env_var) - if val is None: - raise ValueError(f"Missing credential: {secret_name} (tried env {env_var})") - return val + return None @dataclass(frozen=True) @@ -218,19 +218,19 @@ def load_platform_runtime_settings( debug_position_snapshot=resolve_bool_value(os.getenv("FIRSTRADE_DEBUG_POSITION_SNAPSHOT")), income_threshold_usd=resolve_optional_float_env(os.environ, "INCOME_THRESHOLD_USD"), qqqi_income_ratio=_qqqi_income_ratio_env(), - income_layer_enabled=resolve_optional_bool_env("INCOME_LAYER_ENABLED"), + income_layer_enabled=_optional_bool_env("INCOME_LAYER_ENABLED"), income_layer_start_usd=_optional_non_negative_float_env("INCOME_LAYER_START_USD"), income_layer_max_ratio=resolve_optional_ratio_env("INCOME_LAYER_MAX_RATIO"), dca_mode=resolve_optional_dca_mode_env("DCA_MODE"), dca_base_investment_usd=resolve_optional_positive_float_env("DCA_BASE_INVESTMENT_USD"), - ibit_zscore_exit_enabled=resolve_optional_bool_env("IBIT_ZSCORE_EXIT_ENABLED"), + ibit_zscore_exit_enabled=_optional_bool_env("IBIT_ZSCORE_EXIT_ENABLED"), ibit_zscore_exit_mode=resolve_optional_ibit_zscore_exit_mode_env("IBIT_ZSCORE_EXIT_MODE"), ibit_zscore_exit_parking_symbol=resolve_optional_symbol_env("IBIT_ZSCORE_EXIT_PARKING_SYMBOL"), ibit_zscore_exit_risk_reduced_exposure=resolve_optional_ratio_env( "IBIT_ZSCORE_EXIT_RISK_REDUCED_EXPOSURE" ), ibit_zscore_exit_risk_off_exposure=resolve_optional_ratio_env("IBIT_ZSCORE_EXIT_RISK_OFF_EXPOSURE"), - ibit_zscore_exit_allow_outside_execution_window=resolve_optional_bool_env( + ibit_zscore_exit_allow_outside_execution_window=_optional_bool_env( "IBIT_ZSCORE_EXIT_ALLOW_OUTSIDE_EXECUTION_WINDOW" ), runtime_execution_window_trading_days=_runtime_execution_window_trading_days_env( @@ -384,10 +384,16 @@ def _qqqi_income_ratio_env() -> float | None: def _runtime_target_enabled_env() -> bool: - value = resolve_optional_bool_env("RUNTIME_TARGET_ENABLED") + value = _optional_bool_env("RUNTIME_TARGET_ENABLED") return True if value is None else value +def _optional_bool_env(name: str) -> bool | None: + raw_value = os.getenv(f"QSL_{name}") or os.getenv(name) + if raw_value is None or str(raw_value).strip() == "": + return None + return resolve_optional_bool_env(name) + def _optional_non_negative_float_env(name: str) -> float | None: value = resolve_optional_float_env(os.environ, name) diff --git a/scripts/check_qpk_pin_consistency.py b/scripts/check_qpk_pin_consistency.py index 360f881..02349db 100644 --- a/scripts/check_qpk_pin_consistency.py +++ b/scripts/check_qpk_pin_consistency.py @@ -2,11 +2,11 @@ """Check that all QPK git references match the canonical QPK_PIN. Usage: python scripts/check_qpk_pin_consistency.py [--fix] """ -import re, subprocess, sys +import re, sys from pathlib import Path QPK_PIN_URL = "https://raw.githubusercontent.com/QuantStrategyLab/QuantPlatformKit/main/QPK_PIN" -SHA_RE = re.compile(r"@([a-f0-9]{40})") +QPK_REF_RE = re.compile(r"QuantPlatformKit\.git@([a-f0-9]{40})") def fetch_pin() -> str: import urllib.request @@ -23,15 +23,17 @@ def main(): for path in sorted(Path.cwd().glob("**/requirements*.txt")) + sorted(Path.cwd().glob("**/pyproject.toml")): if "external" in str(path): continue content = path.read_text() - for m in SHA_RE.finditer(content): + updated = content + for m in QPK_REF_RE.finditer(content): sha = m.group(1) - if "QuantPlatformKit" not in content[max(0,m.start()-200):m.end()]: continue if sha != target: errors += 1 print(f" ❌ {path}: QPK@{sha[:12]} (expected {target[:12]})") if fix: - path.write_text(content.replace(sha, target)) + updated = updated.replace(f"QuantPlatformKit.git@{sha}", f"QuantPlatformKit.git@{target}") print(" → fixed") + if fix and updated != content: + path.write_text(updated) if errors: print(f"\n{errors} mismatch(es). Run with --fix to auto-fix.") return 1 diff --git a/tests/test_request_handling.py b/tests/test_request_handling.py index dc6a72f..65537a4 100644 --- a/tests/test_request_handling.py +++ b/tests/test_request_handling.py @@ -24,6 +24,8 @@ def route_methods(): def test_cloud_run_route_contracts_are_registered(): assert route_methods() == { "/": ["GET"], + "/health": ["GET"], + "/healthz": ["GET"], "/profiles": ["GET"], "/smoke": ["GET"], "/run": ["GET", "POST"], @@ -288,6 +290,41 @@ def send(message): assert "错误: ValueError: snapshot denied" in text +def test_run_endpoint_redacts_sensitive_error_text(monkeypatch): + monkeypatch.setenv("FIRSTRADE_RUN_STRATEGY_ON_HTTP", "true") + monkeypatch.setenv("TELEGRAM_TOKEN", "token-1") + monkeypatch.setenv("GLOBAL_TELEGRAM_CHAT_ID", "chat-1") + sent_messages = [] + + def fake_build_sender(token, chat_id): + def send(message): + sent_messages.append((token, chat_id, message)) + + return send + + sensitive_error = ( + "request failed: password=supersecret123 token=abcd1234efgh " + "https://api.telegram.org/bot123456789:ABC/sendMessage?api_key=key987654" + ) + monkeypatch.setattr(main, "build_sender", fake_build_sender) + monkeypatch.setattr( + main, + "_run_strategy_cycle_with_report", + lambda: (_ for _ in ()).throw(RuntimeError(sensitive_error)), + ) + client = main.app.test_client() + + response = client.post("/run") + + assert response.status_code == 500 + payload = response.get_json() + assert "" in payload["error"] + assert "" in sent_messages[0][2] + for raw_secret in ("supersecret123", "abcd1234efgh", "123456789:ABC", "key987654"): + assert raw_secret not in payload["error"] + assert raw_secret not in sent_messages[0][2] + + def test_run_endpoint_error_does_not_require_telegram_config(monkeypatch): monkeypatch.setenv("FIRSTRADE_RUN_STRATEGY_ON_HTTP", "true") monkeypatch.delenv("TELEGRAM_TOKEN", raising=False)