From bfa251f72c4434b04268cf33a01854b98924c2ac Mon Sep 17 00:00:00 2001 From: Mike McDougall Date: Sun, 13 Sep 2026 08:55:04 -1000 Subject: [PATCH 1/3] fix(ci): pin ephemeral-server-smoke to the candidate identity, add a moving canary The blocking live GP smoke lane consumed the moving nightly-aot tag and an unpinned honua-server seed checkout, so a commit's verdict depended on what honua-server merged that day. - pinned (push/pull_request): image digest + seed revision of the 2026.1-rc.2 governed candidate (honua-server@7ba4226, sha256:dd50cd81...), checked against the pulled image's digest and revision label; seed checked out at that exact revision. Documented promotion path in the job comment. - canary (daily schedule / dispatch mode=canary): resolves nightly-aot to a digest + revision and seeds from it; every step is non-fatal so the check-run never concludes failure; the verdict creates/updates the "honua-gp moving canary drift" issue. - a verdict step separates environment-startup-failure, gp-correctness-failure, eval-incomplete and not-executed from passed, records smoke-identity.json, and a missing compose fixture now fails instead of skipping green. - the persistent HONUA_GP_EVAL_BASE_URL target is honoured on manual dispatch only; the canary has its own concurrency group. Closes #201 --- .github/workflows/honua-gp-eval.yml | 443 +++++++++++++++++++++++++--- 1 file changed, 408 insertions(+), 35 deletions(-) diff --git a/.github/workflows/honua-gp-eval.yml b/.github/workflows/honua-gp-eval.yml index f38d901..6f1f30c 100644 --- a/.github/workflows/honua-gp-eval.yml +++ b/.github/workflows/honua-gp-eval.yml @@ -2,6 +2,24 @@ name: honua-gp eval on: workflow_dispatch: + inputs: + mode: + description: "pinned = the blocking pinned baseline (or a one-off override below); canary = the moving nightly-aot canary (reports drift, never fails its check-run)." + type: choice + options: [pinned, canary] + default: pinned + server_image: + description: "Override the honua-server image (repo:tag or repo@sha256:digest). Blank = the pinned candidate digest (pinned) or nightly-aot (canary)." + required: false + default: "" + seed_ref: + description: "pinned mode only: honua-server revision for the seed fixtures; required with server_image and must equal the image's org.opencontainers.image.revision label. The canary always seeds from that label." + required: false + default: "" + schedule: + # Moving canary (#201): the current nightly-aot build against its own seed + # revision. See the ephemeral-server-smoke job comment. + - cron: "17 14 * * *" push: branches: [trunk] paths: @@ -23,7 +41,9 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # The canary gets its own group so a scheduled run on trunk never cancels + # (or is cancelled by) the pinned verdict of a trunk push. + group: ${{ github.workflow }}-${{ (github.event_name == 'schedule' || inputs.mode == 'canary') && 'canary-' || '' }}${{ github.ref }} cancel-in-progress: true env: @@ -134,23 +154,75 @@ jobs: # layer) -- several eval scripts mutate the seeded layer (InsertCursor / # UpdateCursor -> applyEdits) and those edits persist, so the oracles are # only deterministic against a database that has not been touched by a prior - # run. This job therefore targets a DEDICATED ``vars.HONUA_GP_EVAL_BASE_URL`` - # variable, not the generic ``vars.HONUA_BASE_URL`` other lanes (e.g. - # conformance) point at a persistent shared staging server -- reusing that - # generic var here previously ran this job's value diff against a drifted, - # non-fresh database and produced spurious response-mismatch failures with - # no real regression (see PR discussion). Leaving the dedicated var unset - # (the default posture) spins up a fresh, ephemeral, per-run Docker target so - # the response oracles reproduce. Set ``vars.HONUA_GP_EVAL_BASE_URL`` - # explicitly ONLY if you intentionally want this lane to smoke-test against a - # persistent target instead of the fresh seed -- expect the seed-pinned - # response oracles to drift/flake against a non-fresh database in that case - # (see docs/golden-eval.md). + # run. The lane therefore always spins up a fresh, ephemeral, per-run Docker + # target. ``vars.HONUA_GP_EVAL_BASE_URL`` (a persistent target) is honoured + # ONLY on a manual workflow_dispatch: a persistent server has no identity + # this job can bind, so it can never carry a push/pull_request verdict + # (#201), and the seed-pinned oracles drift against a non-fresh database + # (see docs/golden-eval.md). It deliberately does NOT read the generic + # ``vars.HONUA_BASE_URL`` other lanes point at shared staging. + # + # BASELINE DECISION (#201): this lane may NOT consume a moving upstream while + # it blocks a commit. It used to default to the moving ``nightly-aot`` tag + # plus an unpinned honua-server seed checkout (default-branch HEAD at run + # time), so a commit's verdict depended on whatever honua-server merged that + # day -- the class of problem honua-sdk-dotnet#300 fixed. It now runs in one + # of two modes: + # + # * pinned (push / pull_request / default dispatch) -- BLOCKING. The server + # image digest and the seed revision below are one identity: the seed is + # checked out at exactly the revision the image was built from, and the + # job fails if the pulled image's digest or its + # ``org.opencontainers.image.revision`` label disagree with the pin. + # * canary (daily schedule / dispatch mode=canary) -- NON-BLOCKING. Pulls + # the moving ``nightly-aot`` tag, resolves it to an immutable digest and + # revision, and seeds from that revision. Every step is non-fatal, so its + # check-run on trunk HEAD never carries a failure conclusion; the verdict + # goes to the ``gp-eval-canary-alert`` job, which creates/updates the open + # issue titled "honua-gp moving canary drift" (the drift destination), + # and to the ``honua-gp-canary-evidence`` artifact. + # + # Both modes record the verdict in ``smoke-identity.json`` (uploaded with the + # smoke artifacts) and the step summary, and separate the failure classes: + # ``environment-startup-failure`` (install, image identity, seed checkout, + # stack start/seed -- no GP was evaluated), ``gp-correctness-failure`` (the + # live eval ran and a script failed), ``eval-incomplete`` (the eval ran but + # skipped scripts, ran in stub mode, or produced no results) and + # ``not-executed``. Only ``passed`` -- every script executed live and passed + # -- counts as success. + # + # PINNED IDENTITY: 2026.1-rc.2 governed candidate (honua-release + # platform-manifest.yaml components.honua-server, snapshot 2026-09-12): + # honua-server@7ba422672e0c751843b17beb36e954a019cc19fb, image + # nightly-aot-7ba4226 = sha256:dd50cd81.... Verified live against this + # repo's #234 oracles: stack ready, SpatialJoin layer served, 50/50 live. + # (The previous 2026-08-19 conformance.yml pin, f897700, predates + # honua-server#4645's SRID-correct buffer that #234's MultiPolygon oracles + # encode.) The layer-aware ops (Buffer / SpatialJoin / Dissolve / Project, + # issue #157) depend on honua-server#2327, an ancestor of this revision. + # + # PROMOTION PATH -- move the pin only in a reviewed PR, both values together: + # 1. Pick the new identity: the governed candidate in honua-release's + # platform-manifest.yaml (components.honua-server sha + digest), or a + # nightly the canary reported ``passed`` for (its + # ``smoke-identity.json`` records digest + revision). + # 2. Confirm the digest carries the revision label: + # docker pull ghcr.io/honua-io/honua-server@ + # docker image inspect ghcr.io/honua-io/honua-server@ \ + # --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' + # and that the revision is fetchable from honua-server. + # 3. Dispatch this workflow on the PR branch with mode=pinned, + # server_image=ghcr.io/honua-io/honua-server@, seed_ref= + # and require verdict ``passed``. If oracles legitimately changed + # upstream, refresh eval/golden in the same PR with the evidence. + # 4. Update HONUA_GP_EVAL_PINNED_IMAGE and HONUA_GP_EVAL_PINNED_SEED_REVISION + # below, in that PR. # # This is NOT ArcGIS Pro parity: a licensed arcpy baseline does not exist in # this environment, so arcpy-level output equivalence stays out of scope and # is tracked separately. ephemeral-server-smoke: + name: ${{ (github.event_name == 'schedule' || inputs.mode == 'canary') && 'gp-eval-moving-canary' || 'ephemeral-server-smoke' }} if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest needs: eval @@ -158,19 +230,31 @@ jobs: contents: read packages: read environment: staging + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} + image: ${{ steps.identity.outputs.resolved }} + revision: ${{ steps.identity.outputs.revision }} + identity-source: ${{ steps.identity.outputs.source }} env: - # Dedicated var -- see the job-level comment above for why this must NOT - # read the generic ``vars.HONUA_BASE_URL``. - HONUA_BASE_URL: ${{ vars.HONUA_GP_EVAL_BASE_URL }} + HONUA_GP_EVAL_MODE: ${{ (github.event_name == 'schedule' || inputs.mode == 'canary') && 'canary' || 'pinned' }} + HONUA_GP_EVAL_PINNED_IMAGE: ghcr.io/honua-io/honua-server@sha256:dd50cd81c057e37e73a6144572abdfc90d48de314d7625c54c4ef3b6eb65b0fd + HONUA_GP_EVAL_PINNED_SEED_REVISION: 7ba422672e0c751843b17beb36e954a019cc19fb + HONUA_GP_EVAL_CANARY_IMAGE: ghcr.io/honua-io/honua-server:nightly-aot + # Persistent-target override: manual dispatch only -- see the job comment. + HONUA_BASE_URL: ${{ github.event_name == 'workflow_dispatch' && vars.HONUA_GP_EVAL_BASE_URL || '' }} HONUA_API_KEY: ${{ secrets.HONUA_API_KEY }} - # Use the immutable image/source pair verified by the conformance lane. - # Keep operator overrides paired so seed schema cannot drift from the image. - HONUA_LOCAL_SERVER_IMAGE: ${{ vars.HONUA_LOCAL_SERVER_IMAGE || 'ghcr.io/honua-io/honua-server@sha256:78e3088d64d832d3e2752c87d80bfcad201b414f4525989ca5d9a242cd5fee8a' }} - HONUA_LOCAL_SERVER_REVISION: ${{ vars.HONUA_LOCAL_SERVER_REVISION || 'f897700159e2791c9468c6ca85bb4e2a3a8d8433' }} steps: + # On the canary every step is non-fatal (continue-on-error) so the + # check-run never concludes failure; each step is gated on the outcome + # of the one it depends on, and the "Record smoke verdict" step turns + # the outcomes into the verdict. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + id: checkout + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} - name: Setup Python + id: setup-python + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: ${{ env.PYTHON_VERSION }} @@ -178,6 +262,9 @@ jobs: cache-dependency-path: packages/honua-gp/pyproject.toml - name: Install packages + id: install + if: ${{ steps.checkout.outcome == 'success' && steps.setup-python.outcome == 'success' }} + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} run: | python -m pip install --require-hashes --only-binary=:all: -r .github/requirements/ci.lock pip install --no-deps --no-build-isolation -e packages/honua-sdk @@ -188,41 +275,124 @@ jobs: id: target run: | if [[ -n "${HONUA_BASE_URL}" ]]; then - echo "configured=true" >> "${GITHUB_OUTPUT}" echo "local_stack=false" >> "${GITHUB_OUTPUT}" { echo "## honua-gp smoke" - echo '- ``vars.HONUA_GP_EVAL_BASE_URL`` is configured; smoke-testing against that persistent target instead of a fresh local seed.' - echo '- WARNING: the response-value oracles in eval/golden/*.json are pinned to a fresh client-compat seed and MAY drift/flake against a non-fresh database.' + echo '- manual dispatch with ``vars.HONUA_GP_EVAL_BASE_URL`` configured; smoke-testing against that persistent target instead of a fresh local seed.' + echo '- WARNING: the server identity is unverified and the response-value oracles in eval/golden/*.json are pinned to a fresh client-compat seed, so they MAY drift/flake against a non-fresh database.' } >> "${GITHUB_STEP_SUMMARY}" else - echo "configured=true" >> "${GITHUB_OUTPUT}" echo "local_stack=true" >> "${GITHUB_OUTPUT}" { - echo "## honua-gp smoke" - echo '- ``vars.HONUA_GP_EVAL_BASE_URL`` not configured; standing up a fresh, ephemeral, seeded client-compat Docker target so the response-value oracles are deterministic.' + echo "## honua-gp smoke (${HONUA_GP_EVAL_MODE})" + echo '- standing up a fresh, ephemeral, seeded client-compat Docker target so the response-value oracles are deterministic.' } >> "${GITHUB_STEP_SUMMARY}" fi - - name: Checkout Honua Server seed fixtures + - name: Resolve honua-server image identity + id: identity if: ${{ steps.target.outputs.local_stack == 'true' }} + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} + env: + REQUESTED_IMAGE: ${{ inputs.server_image }} + REQUESTED_SEED_REF: ${{ inputs.seed_ref }} + run: | + set -euo pipefail + if [[ "${HONUA_GP_EVAL_MODE}" == "canary" ]]; then + image="${REQUESTED_IMAGE:-${HONUA_GP_EVAL_CANARY_IMAGE}}" + seed_ref="" + source="moving-canary" + elif [[ -n "${REQUESTED_IMAGE}${REQUESTED_SEED_REF}" ]]; then + if [[ -z "${REQUESTED_IMAGE}" || -z "${REQUESTED_SEED_REF}" ]]; then + echo "::error::server_image and seed_ref override the pinned baseline as one identity; pass both." + exit 1 + fi + image="${REQUESTED_IMAGE}" + seed_ref="${REQUESTED_SEED_REF}" + source="dispatch-override" + else + image="${HONUA_GP_EVAL_PINNED_IMAGE}" + seed_ref="${HONUA_GP_EVAL_PINNED_SEED_REVISION}" + source="pinned" + fi + + docker pull "${image}" + resolved="$(docker image inspect "${image}" --format '{{index .RepoDigests 0}}')" + digest="${resolved##*@}" + revision="$(docker image inspect "${image}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" + if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::could not resolve an immutable digest for ${image} (got '${resolved}')." + exit 1 + fi + if [[ ! "${revision}" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::${image} carries no org.opencontainers.image.revision label (got '${revision}'); the seed fixtures cannot be bound to it." + exit 1 + fi + if [[ "${source}" == "pinned" ]]; then + if [[ "${image}" != *"@${digest}" ]]; then + echo "::error::pulled digest ${digest} does not match the pinned image ${image}." + exit 1 + fi + if [[ "${revision}" != "${seed_ref}" ]]; then + echo "::error::pinned image revision ${revision} does not match the pinned seed revision ${seed_ref}; update both pins together." + exit 1 + fi + fi + # The canary seeds from the revision the moving image was built from, + # so image and seed still move as one identity. + seed_ref="${seed_ref:-${revision}}" + + { + echo "image=${image}" + echo "resolved=${resolved}" + echo "digest=${digest}" + echo "revision=${revision}" + echo "seed_ref=${seed_ref}" + echo "source=${source}" + } >> "${GITHUB_OUTPUT}" + { + echo "- Identity source: \`${source}\`" + echo "- Server image: \`${image}\`" + echo "- Resolved digest: \`${resolved}\`" + echo "- Image revision: \`${revision}\`" + echo "- Seed ref: \`${seed_ref}\`" + } >> "${GITHUB_STEP_SUMMARY}" + + - name: Checkout Honua Server seed fixtures + id: seed-checkout + if: ${{ steps.target.outputs.local_stack == 'true' && steps.identity.outcome == 'success' }} + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: honua-io/honua-server path: honua-server - ref: ${{ env.HONUA_LOCAL_SERVER_REVISION }} + # The exact revision the server image was built from (see the + # identity step) -- never the default branch. + ref: ${{ steps.identity.outputs.seed_ref }} - name: Start seeded local Honua target - if: ${{ steps.target.outputs.local_stack == 'true' }} + id: stack + if: ${{ steps.target.outputs.local_stack == 'true' && steps.seed-checkout.outcome == 'success' }} + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} env: COMPOSE_PROJECT_NAME: honua-gp-smoke-${{ github.run_id }}-${{ github.run_attempt }} + # Digest-qualified, so compose runs exactly the image the identity + # step resolved even if the canary's moving tag advances meanwhile. + HONUA_LOCAL_SERVER_IMAGE: ${{ steps.identity.outputs.resolved }} + HONUA_SERVER_REVISION: ${{ steps.identity.outputs.revision }} run: | set -euo pipefail server_root="${GITHUB_WORKSPACE}/honua-server" + seed_revision="$(git -C "${server_root}" rev-parse HEAD)" + if [[ "${seed_revision}" != "${HONUA_SERVER_REVISION}" ]]; then + echo "::error::checked-out seed revision ${seed_revision} does not match image revision ${HONUA_SERVER_REVISION}." + exit 1 + fi compose_file="${server_root}/docker/client-compat/compose.yml" if [[ ! -f "${compose_file}" ]]; then - echo "::warning::honua-server compose file not found; skipping ephemeral smoke." - exit 0 + # A missing fixture is failed evidence, never a skipped-green smoke. + echo "::error::honua-server@${seed_revision} has no docker/client-compat/compose.yml; the seeded target cannot be started." + exit 1 fi override_file="${RUNNER_TEMP}/honua-gp-smoke.override.yml" cat > "${override_file}" < real honua-server) + id: live + if: ${{ steps.install.outcome == 'success' && (steps.target.outputs.local_stack == 'false' || steps.stack.outcome == 'success') }} + continue-on-error: ${{ env.HONUA_GP_EVAL_MODE == 'canary' }} env: HONUA_GP_EVAL_USE_STUB: "0" HONUA_GP_AUDIT_DIR: ${{ runner.temp }}/honua-gp-smoke-audit @@ -384,15 +554,218 @@ jobs: --pass-threshold 0.70 \ --require-supported-pass-rate 1.0 + - name: Record smoke verdict + id: verdict + if: always() + env: + LOCAL_STACK: ${{ steps.target.outputs.local_stack }} + OUTCOME_CHECKOUT: ${{ steps.checkout.outcome }} + OUTCOME_SETUP_PYTHON: ${{ steps.setup-python.outcome }} + OUTCOME_INSTALL: ${{ steps.install.outcome }} + OUTCOME_TARGET: ${{ steps.target.outcome }} + OUTCOME_IDENTITY: ${{ steps.identity.outcome }} + OUTCOME_SEED_CHECKOUT: ${{ steps.seed-checkout.outcome }} + OUTCOME_STACK: ${{ steps.stack.outcome }} + OUTCOME_LIVE: ${{ steps.live.outcome }} + IDENTITY_SOURCE: ${{ steps.identity.outputs.source }} + IDENTITY_IMAGE: ${{ steps.identity.outputs.image }} + IDENTITY_DIGEST: ${{ steps.identity.outputs.digest }} + IDENTITY_REVISION: ${{ steps.identity.outputs.revision }} + IDENTITY_SEED_REF: ${{ steps.identity.outputs.seed_ref }} + SMOKE_RESULTS: packages/honua-gp/smoke-eval.json + SMOKE_IDENTITY: packages/honua-gp/smoke-identity.json + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + + env = os.environ + local_stack = env.get("LOCAL_STACK") == "true" + harness = [ + ("checkout", "OUTCOME_CHECKOUT"), + ("setup-python", "OUTCOME_SETUP_PYTHON"), + ("install", "OUTCOME_INSTALL"), + ("target", "OUTCOME_TARGET"), + ] + if local_stack: + harness += [ + ("image-identity", "OUTCOME_IDENTITY"), + ("seed-checkout", "OUTCOME_SEED_CHECKOUT"), + ("stack-start", "OUTCOME_STACK"), + ] + outcomes = {name: env.get(key) or "not-run" for name, key in harness} + outcomes["live-eval"] = env.get("OUTCOME_LIVE") or "not-run" + + results = None + results_path = Path(env["SMOKE_RESULTS"]) + if results_path.is_file(): + try: + results = json.loads(results_path.read_text(encoding="utf-8")) + except ValueError: + results = None + + failed_harness = [name for name, _ in harness if outcomes[name] != "success"] + live = outcomes["live-eval"] + if failed_harness: + verdict = "environment-startup-failure" + reason = "harness step(s) did not succeed: " + ", ".join( + f"{name}={outcomes[name]}" for name in failed_harness + ) + elif live in ("not-run", "skipped", "cancelled"): + verdict = "not-executed" + reason = f"live eval outcome {live}" + elif results is None: + verdict = "eval-incomplete" + reason = f"live eval outcome {live} but no readable {results_path}" + elif not results.get("liveMode"): + verdict = "eval-incomplete" + reason = "eval ran in stub mode, not against the live server" + elif results.get("failed", 0) > 0: + verdict = "gp-correctness-failure" + reason = "; ".join( + f"{r['name']}: {r.get('reason') or 'failed'}" + for r in results.get("results", []) + if r.get("status") == "fail" + ) + elif ( + live != "success" + or results.get("skipped", 0) > 0 + or results.get("supportedTotal", 0) == 0 + or results.get("passed") != results.get("total") + ): + verdict = "eval-incomplete" + reason = ( + f"live eval outcome {live}; passed {results.get('passed')}/{results.get('total')}, " + f"skipped {results.get('skipped')}, supported {results.get('supportedTotal')}" + ) + else: + verdict = "passed" + reason = f"{results['passed']}/{results['total']} scripts passed live" + + record = { + "schema": "honua-gp-smoke-identity.v1", + "mode": env["HONUA_GP_EVAL_MODE"], + "verdict": verdict, + "reason": reason, + "sdkCommit": env["GITHUB_SHA"], + "event": env["GITHUB_EVENT_NAME"], + "runUrl": env["RUN_URL"], + "target": "ephemeral-local-stack" if local_stack else "persistent-unverified", + "server": { + "identitySource": env.get("IDENTITY_SOURCE") or None, + "image": env.get("IDENTITY_IMAGE") or None, + "digest": env.get("IDENTITY_DIGEST") or None, + "revision": env.get("IDENTITY_REVISION") or None, + "seedRef": env.get("IDENTITY_SEED_REF") or None, + "pinnedImage": env["HONUA_GP_EVAL_PINNED_IMAGE"], + "pinnedSeedRevision": env["HONUA_GP_EVAL_PINNED_SEED_REVISION"], + }, + "outcomes": outcomes, + "eval": None + if results is None + else { + key: results.get(key) + for key in ("total", "passed", "failed", "skipped", "supportedTotal", "supportedPassed", "liveMode") + }, + } + identity_path = Path(env["SMOKE_IDENTITY"]) + identity_path.parent.mkdir(parents=True, exist_ok=True) + identity_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"verdict={verdict}\n") + with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(f"\n### Smoke verdict ({record['mode']}): `{verdict}`\n\n{reason}\n\n") + summary.write("| step | outcome |\n| --- | --- |\n") + for name, outcome in outcomes.items(): + summary.write(f"| {name} | {outcome} |\n") + print(json.dumps(record, indent=2, sort_keys=True)) + PY + verdict="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["verdict"])' packages/honua-gp/smoke-identity.json)" + if [[ "${verdict}" == "passed" ]]; then + exit 0 + fi + if [[ "${HONUA_GP_EVAL_MODE}" == "canary" ]]; then + echo "::warning::honua-gp moving canary verdict: ${verdict} (reported by gp-eval-canary-alert; this check-run does not fail)." + exit 0 + fi + echo "::error::honua-gp pinned smoke verdict: ${verdict}" + exit 1 + # Without this the only trace of a live failure is the aggregate pass-rate # line -- the failing script names and their reasons live in the JSON. - name: Upload ephemeral smoke artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: honua-gp-smoke-eval + name: ${{ env.HONUA_GP_EVAL_MODE == 'canary' && 'honua-gp-canary-evidence' || 'honua-gp-smoke-eval' }} path: | packages/honua-gp/smoke-eval.json packages/honua-gp/smoke-eval.xml + packages/honua-gp/smoke-identity.json if-no-files-found: warn retention-days: 30 + + # Drift destination for the moving canary (#201). Keys off the smoke job's + # verdict output, not its result: the canary job always concludes success. + # An empty verdict (the canary was skipped, cancelled or timed out) is not + # ``passed`` and therefore raises the alert. This job itself may fail: a + # broken alert path is a real, actionable red. + gp-eval-canary-alert: + name: gp-eval moving canary alert + needs: [eval, ephemeral-server-smoke] + if: ${{ always() && (github.event_name == 'schedule' || inputs.mode == 'canary') }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + ISSUE_TITLE: honua-gp moving canary drift + VERDICT: ${{ needs.ephemeral-server-smoke.outputs.verdict }} + SMOKE_RESULT: ${{ needs.ephemeral-server-smoke.result }} + EVAL_RESULT: ${{ needs.eval.result }} + CANARY_IMAGE: ${{ needs.ephemeral-server-smoke.outputs.image }} + CANARY_REVISION: ${{ needs.ephemeral-server-smoke.outputs.revision }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SDK_COMMIT: ${{ github.sha }} + steps: + - name: Create or update the canary drift issue + run: | + set -euo pipefail + verdict="${VERDICT:-not-executed}" + existing="$(gh issue list --state open --search "\"${ISSUE_TITLE}\" in:title" --json number,title \ + --jq "map(select(.title == \"${ISSUE_TITLE}\")) | .[0].number // empty")" + + if [[ "${verdict}" == "passed" ]]; then + echo "## honua-gp moving canary: passed" >> "${GITHUB_STEP_SUMMARY}" + echo "- \`${CANARY_IMAGE}\` (honua-server@${CANARY_REVISION}) is a promotion candidate for the pinned baseline." >> "${GITHUB_STEP_SUMMARY}" + if [[ -n "${existing}" ]]; then + gh issue comment "${existing}" --body "Canary verdict \`passed\` again on ${RUN_URL} against \`${CANARY_IMAGE}\` (honua-server@${CANARY_REVISION}). Close this issue once the drift that opened it is understood." + fi + exit 0 + fi + + body_file="${RUNNER_TEMP}/canary-drift.md" + { + echo "The scheduled \`honua-gp eval\` moving canary reported verdict **\`${verdict}\`**." + echo + echo "- Run: ${RUN_URL} (artifact \`honua-gp-canary-evidence\` holds \`smoke-identity.json\` and the per-script results)" + echo "- Server image: \`${CANARY_IMAGE:-unresolved}\`" + echo "- Server revision: \`${CANARY_REVISION:-unresolved}\`" + echo "- SDK commit: \`${SDK_COMMIT}\`" + echo "- Job results: eval=\`${EVAL_RESULT}\`, canary=\`${SMOKE_RESULT}\`" + echo + echo "Verdicts: \`environment-startup-failure\` means the image/seed/stack never came up (no GP was evaluated); \`gp-correctness-failure\` means a live GP script returned a wrong result; \`eval-incomplete\` / \`not-executed\` mean there is no full verdict." + echo + echo "This canary deliberately does not fail its check-run (#201): the blocking \`ephemeral-server-smoke\` lane stays on its pinned identity. Triage against honua-server; promote the pin only through the promotion path in \`.github/workflows/honua-gp-eval.yml\`." + } > "${body_file}" + + if [[ -n "${existing}" ]]; then + gh issue comment "${existing}" --body-file "${body_file}" + else + gh issue create --title "${ISSUE_TITLE}" --body-file "${body_file}" + fi From e22e9708990449fb530e2637bf47d13b8045633d Mon Sep 17 00:00:00 2001 From: Mike McDougall Date: Thu, 10 Sep 2026 11:38:41 -1000 Subject: [PATCH 2/3] fix(gp): bless live response oracles for 9 unblessed scripts + gate future regrowth Live smoke graded no response oracle for 9 of 25 supported honua-gp eval scripts (Describe x2, ListFields x3, InsertCursor, UpdateCursor x2), so a response-parsing regression in schema introspection or write-cursor round trips would pass the lane silently as an "unblessed" pass. Blessing the two write-cursor scripts against a fresh seed surfaced a real bug rather than a missing oracle: honua_gp.da._values_for_row/_extract_oid only recognized a feature's .attributes, but the live SDK's honua_sdk.models.QueryFeature exposes GeoJSON-shaped .properties/.id instead -- SearchCursor/UpdateCursor silently returned None for every field (OID included) against a real server. Stub-mode CI never caught this because the eval stub's _StubFeature happens to have .attributes. Fixed both helpers to read the real feature shape (preferring the SDK's already-resolved .id for OID@, since the seed's object-id field is lower-case objectid and never matched the existing OBJECTID/oid/FID key list), added regression tests against the actual QueryFeature type, and re-blessed against a fresh stack -- update_cursor_close_status now genuinely proves an update round-trip (update_count: 1) instead of masking the bug behind a vacuous zero. Describe/ListFields (schema fetch) and InsertCursor were already correct; those got a schema/apply-edits fingerprint oracle. MakeTableView makes no server round trip at all (a client-side session alias registration) -- no response to fingerprint, so it goes in a new UNBLESSED_ALLOWLIST.json with the reason instead of a silent pass. run_eval.py now fails a live-mode supported script that has no response oracle and no UNBLESSED_ALLOWLIST.json entry, so this set cannot regrow unnoticed (verified: an emptied allow-list correctly fails make_table_view.py with the new reason). Closes #202 --- packages/honua-gp/docs/golden-eval.md | 22 ++++++ .../honua-gp/eval/UNBLESSED_ALLOWLIST.json | 3 + packages/honua-gp/eval/_emit.py | 69 ++++++++++++++++- .../eval/golden/describe_segments.json | 41 ++++++++++ .../eval/golden/describe_segments_fields.json | 41 ++++++++++ .../golden/insert_cursor_append_rows.json | 6 ++ .../eval/golden/list_fields_segments.json | 41 ++++++++++ .../golden/list_fields_segments_filtered.json | 23 ++++++ .../golden/list_fields_segments_wildcard.json | 13 ++++ .../golden/update_cursor_close_status.json | 6 ++ .../golden/update_cursor_delete_closed.json | 6 ++ packages/honua-gp/eval/run_eval.py | 61 ++++++++++++++- .../eval/scripts/describe_segments.py | 5 ++ .../eval/scripts/describe_segments_fields.py | 6 ++ .../eval/scripts/insert_cursor_append_rows.py | 6 ++ .../eval/scripts/list_fields_segments.py | 2 + .../scripts/list_fields_segments_filtered.py | 2 + .../scripts/list_fields_segments_wildcard.py | 2 + .../scripts/update_cursor_close_status.py | 6 ++ .../scripts/update_cursor_delete_closed.py | 9 +++ packages/honua-gp/honua_gp/da/__init__.py | 49 +++++++++--- packages/honua-gp/tests/test_da_cursors.py | 74 +++++++++++++++++++ 22 files changed, 481 insertions(+), 12 deletions(-) create mode 100644 packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json diff --git a/packages/honua-gp/docs/golden-eval.md b/packages/honua-gp/docs/golden-eval.md index cbff390..2706495 100644 --- a/packages/honua-gp/docs/golden-eval.md +++ b/packages/honua-gp/docs/golden-eval.md @@ -85,6 +85,28 @@ Request fingerprints are written in any mode; response fingerprints only in live mode (the stub's canned `href` carries no real values and must never be frozen as an oracle). +## Unblessed supported scripts must be explicit (`UNBLESSED_ALLOWLIST.json`) + +A live-mode run grades every non-`expected_failure` script's response layer +against its golden `response` block. If a supported script has no `response` +block at all, the harness does **not** silently pass it: it fails unless the +script's stem is listed in `eval/UNBLESSED_ALLOWLIST.json` with a documented +reason. This closes the gap issue #202 found -- an un-oracled supported +script used to grade as an "unblessed" pass, so a response-parsing regression +in an unblessed script's operation could sail through the live smoke lane +silently. + +Two ways a supported script ends up here: + +* **Not yet blessed** -- bless it (see above) and commit the oracle. This is + the expected outcome for almost every case; a newly-added supported script + must be blessed before merge, or the live smoke lane fails on it. +* **Genuinely not capturable** -- the operation makes no honua-server HTTP + round trip to fingerprint (e.g. `MakeTableView`/`MakeFeatureLayer`, which + only register a client-side session alias). Add an entry to + `UNBLESSED_ALLOWLIST.json` explaining why, instead of leaving the gap + implicit. + ## Determinism note (live mode) The response oracles for count/row scripts record exact values pinned to the diff --git a/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json b/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json new file mode 100644 index 0000000..18cc9cd --- /dev/null +++ b/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json @@ -0,0 +1,3 @@ +{ + "make_table_view": "management.MakeTableView (and MakeFeatureLayer) resolve to honua_gp.management._make_layer_handler, which only registers a client-side LayerAlias in the session -- it makes no honua-server HTTP round trip. There is no server response for the harness to fingerprint; the request/plumbing layers (already graded) are this script's full coverage. See issue #202 and docs/golden-eval.md." +} diff --git a/packages/honua-gp/eval/_emit.py b/packages/honua-gp/eval/_emit.py index 4df92f5..e6ccacc 100644 --- a/packages/honua-gp/eval/_emit.py +++ b/packages/honua-gp/eval/_emit.py @@ -123,4 +123,71 @@ def feature_layer_fingerprint(result: Any) -> dict[str, Any]: return fingerprint -__all__ = ["emit_response", "feature_layer_fingerprint"] +def schema_fingerprint(fields: Any, *, shape_type: Any = None, oid_field: Any = None, srid: Any = None) -> dict[str, Any]: + """Normalize a Describe/ListFields result into a seed-stable fingerprint. + + ``fields`` is a sequence of ``FieldDescribe`` (or anything exposing + ``.name`` / ``.type``). The seeded ``segments`` / ``roads`` schemas are + fixed by ``tests/seed/client-compat-v1.sql`` and do not change between + live runs, so field names + types are a stable oracle -- unlike geometry + coordinates or generated object ids. + """ + + return { + "field_count": len(fields), + "field_names": [str(getattr(f, "name", "")) for f in fields], + "field_types": {str(getattr(f, "name", "")): getattr(f, "type", None) for f in fields}, + "shape_type": shape_type, + "oid_field": oid_field, + "srid": srid, + } + + +def apply_edits_fingerprint(result: Any) -> dict[str, Any]: + """Normalize an ``InsertCursor``/``UpdateCursor`` ``flush()`` return value. + + ``flush()`` returns different shapes depending on transport: the stub's + ``_StubApplyEditsResult.to_dict()`` (a plain dict with ``adds`` / + ``updates`` / ``deletes`` lists) versus the live SDK's + ``honua_sdk.models.ApplyEditsResult`` dataclass (``add_results`` / + ``update_results`` / ``delete_results`` sequences of typed + ``EditOperationResult``, each carrying a server-assigned ``object_id``). + Only success *counts* are captured, never object ids -- those are not + stable oracles across repeated seed runs. ``result`` is ``None`` when + ``flush()`` had nothing buffered (e.g. an UpdateCursor predicate matched + zero rows against the current seed state) -- that is itself a valid, + deterministic oracle (all counts zero, vacuously succeeded), not an + absence of one. + """ + + if result is None: + return {"add_count": 0, "update_count": 0, "delete_count": 0, "all_succeeded": True} + if isinstance(result, Mapping): + adds, updates, deletes = result.get("adds", []), result.get("updates", []), result.get("deletes", []) + return { + "add_count": len(adds), + "update_count": len(updates), + "delete_count": len(deletes), + "all_succeeded": True, + } + add_results = getattr(result, "add_results", ()) + update_results = getattr(result, "update_results", ()) + delete_results = getattr(result, "delete_results", ()) + all_succeeded = getattr(result, "all_succeeded", None) + if all_succeeded is None: + combined = [*add_results, *update_results, *delete_results] + all_succeeded = bool(combined) and all(getattr(r, "success", False) for r in combined) + return { + "add_count": len(add_results), + "update_count": len(update_results), + "delete_count": len(delete_results), + "all_succeeded": bool(all_succeeded), + } + + +__all__ = [ + "apply_edits_fingerprint", + "emit_response", + "feature_layer_fingerprint", + "schema_fingerprint", +] diff --git a/packages/honua-gp/eval/golden/describe_segments.json b/packages/honua-gp/eval/golden/describe_segments.json index 03dfe97..53efea4 100644 --- a/packages/honua-gp/eval/golden/describe_segments.json +++ b/packages/honua-gp/eval/golden/describe_segments.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": "objectid", + "shape_type": "Point", + "srid": 4326 + }, + "schema_version": 2, "stdout_contains": "describe_segments ok" } diff --git a/packages/honua-gp/eval/golden/describe_segments_fields.json b/packages/honua-gp/eval/golden/describe_segments_fields.json index f15e202..6ce5f8f 100644 --- a/packages/honua-gp/eval/golden/describe_segments_fields.json +++ b/packages/honua-gp/eval/golden/describe_segments_fields.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": "objectid", + "shape_type": "Point", + "srid": 4326 + }, + "schema_version": 2, "stdout_contains": "describe_segments_fields ok" } diff --git a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json index 2c607bb..8cca5b1 100644 --- a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json +++ b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json @@ -4,5 +4,11 @@ "audit_lines": 1, "stdout_contains": "insert_cursor_append_rows ok" }, + "response": { + "add_count": 3, + "all_succeeded": true, + "delete_count": 0, + "update_count": 0 + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/list_fields_segments.json b/packages/honua-gp/eval/golden/list_fields_segments.json index e3247d9..7050282 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments.json +++ b/packages/honua-gp/eval/golden/list_fields_segments.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments ok" } diff --git a/packages/honua-gp/eval/golden/list_fields_segments_filtered.json b/packages/honua-gp/eval/golden/list_fields_segments_filtered.json index 37b7dcf..39d2f1c 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments_filtered.json +++ b/packages/honua-gp/eval/golden/list_fields_segments_filtered.json @@ -1,5 +1,28 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 6, + "field_names": [ + "name", + "description", + "status", + "event_time", + "tags", + "numbers" + ], + "field_types": { + "description": "String", + "event_time": "String", + "name": "String", + "numbers": "String", + "status": "String", + "tags": "String" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments_filtered ok" } diff --git a/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json b/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json index 549b5a6..f58f74d 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json +++ b/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json @@ -1,5 +1,18 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 1, + "field_names": [ + "status" + ], + "field_types": { + "status": "String" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments_wildcard ok" } diff --git a/packages/honua-gp/eval/golden/update_cursor_close_status.json b/packages/honua-gp/eval/golden/update_cursor_close_status.json index 085ffae..a6475fe 100644 --- a/packages/honua-gp/eval/golden/update_cursor_close_status.json +++ b/packages/honua-gp/eval/golden/update_cursor_close_status.json @@ -4,5 +4,11 @@ "audit_lines": 1, "stdout_contains": "update_cursor_close_status ok" }, + "response": { + "add_count": 0, + "all_succeeded": true, + "delete_count": 0, + "update_count": 1 + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json index 7725630..17e9844 100644 --- a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json +++ b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json @@ -4,5 +4,11 @@ "audit_lines": 1, "stdout_contains": "update_cursor_delete_closed ok" }, + "response": { + "add_count": 0, + "all_succeeded": true, + "delete_count": 0, + "update_count": 0 + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/run_eval.py b/packages/honua-gp/eval/run_eval.py index e170307..a8f4a4b 100644 --- a/packages/honua-gp/eval/run_eval.py +++ b/packages/honua-gp/eval/run_eval.py @@ -62,6 +62,7 @@ PACKAGE_ROOT = Path(__file__).resolve().parents[1] DEFAULT_SCRIPT_DIR = PACKAGE_ROOT / "eval" / "scripts" DEFAULT_GOLDEN_DIR = PACKAGE_ROOT / "eval" / "golden" +DEFAULT_UNBLESSED_ALLOWLIST = PACKAGE_ROOT / "eval" / "UNBLESSED_ALLOWLIST.json" DEFAULT_PASS_RATE = 0.70 @@ -177,6 +178,29 @@ def _golden_is_expected_failure(script: Path, golden: dict[str, Any] | None) -> return "expected_failure" in script.stem +def load_unblessed_allowlist(path: Path) -> dict[str, str]: + """Load the {script_stem: reason} map of supported scripts with no capturable response oracle. + + A live-mode supported script with no golden ``response`` block is either + a gap that has never been blessed (a regression risk -- see issue #202) + or a script whose operation genuinely has no server round trip to + fingerprint (e.g. ``MakeTableView``, a session-local alias registration). + Only the second case belongs here, with the reason documented; anything + else must fail so a newly-added supported script cannot silently regrow + the unblessed set. + """ + + if not path.exists(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + if not isinstance(data, dict): + return {} + return {str(k): str(v) for k, v in data.items()} + + # --------------------------------------------------------------------------- # Captured value oracles # --------------------------------------------------------------------------- @@ -313,6 +337,7 @@ def _grade( request_actual: list[dict[str, Any]], response_actual: dict[str, Any] | None, live_mode: bool, + unblessed_allowlist: dict[str, str] | None = None, ) -> tuple[str, bool, str | None, dict[str, str]]: """Grade one script across plumbing + request + response layers.""" @@ -391,8 +416,26 @@ def _grade( else: checks["response"] = "pass" elif live_mode and golden is not None and not expected_failure: - # Live run, supported script, but no response oracle recorded yet. - checks["response"] = "unblessed" + # Live run, supported script, but no response oracle recorded. Pass + # ONLY when the script is explicitly documented as non-capturable in + # the unblessed allow-list (see UNBLESSED_ALLOWLIST.json) -- otherwise + # this is exactly the silent-pass gap issue #202 closed: a new + # supported script (or a regression that dropped an oracle) must fail + # loudly, not disappear into an "unblessed" pass. + allowlist = unblessed_allowlist or {} + if script.stem in allowlist: + checks["response"] = "unblessed(allowed)" + else: + checks["response"] = "fail" + return ( + "fail", + False, + f"supported script {script.stem!r} has no response oracle and is not in " + "eval/UNBLESSED_ALLOWLIST.json -- bless it " + "(HONUA_GP_EVAL_USE_STUB=0 ... run_eval.py --update-golden) or document why it " + "has no capturable response in the allow-list", + checks, + ) return "pass", False, None, checks @@ -431,9 +474,11 @@ def run( timeout: float, pass_threshold: float, update_golden: bool = False, + unblessed_allowlist_path: Path = DEFAULT_UNBLESSED_ALLOWLIST, ) -> EvalSummary: scripts = sorted(p for p in script_dir.glob("*.py") if not p.name.startswith("_")) live_mode = live_values_available() + unblessed_allowlist = load_unblessed_allowlist(unblessed_allowlist_path) summary = EvalSummary(total=len(scripts), pass_threshold=pass_threshold, live_mode=live_mode) result_root = Path(tempfile.mkdtemp(prefix="honua-gp-eval-results-")) for script in scripts: @@ -487,6 +532,7 @@ def run( request_actual=request_actual, response_actual=response_actual, live_mode=live_mode, + unblessed_allowlist=unblessed_allowlist, ) result = ScriptResult( name=script.name, @@ -586,6 +632,16 @@ def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run the honua-gp compatibility eval suite.") parser.add_argument("--scripts", type=Path, default=DEFAULT_SCRIPT_DIR, help="Directory of eval scripts.") parser.add_argument("--golden", type=Path, default=DEFAULT_GOLDEN_DIR, help="Directory of golden reference values.") + parser.add_argument( + "--unblessed-allowlist", + type=Path, + default=DEFAULT_UNBLESSED_ALLOWLIST, + help=( + "JSON {script_stem: reason} map of supported scripts documented as having no " + "capturable live response oracle. A live-mode supported script with no response " + "oracle and no allow-list entry fails the run (see issue #202)." + ), + ) parser.add_argument( "--audit-root", type=Path, @@ -645,6 +701,7 @@ def main(argv: Sequence[str] | None = None) -> int: timeout=args.timeout, pass_threshold=args.pass_threshold, update_golden=args.update_golden, + unblessed_allowlist_path=args.unblessed_allowlist, ) write_json(summary, args.output_json) write_junit(summary, args.output_junit) diff --git a/packages/honua-gp/eval/scripts/describe_segments.py b/packages/honua-gp/eval/scripts/describe_segments.py index 301e61d..d09135d 100644 --- a/packages/honua-gp/eval/scripts/describe_segments.py +++ b/packages/honua-gp/eval/scripts/describe_segments.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,6 +26,10 @@ desc = arcpy.Describe("segments") srid = desc.spatialReference.factoryCode if desc.spatialReference else None +emit_response( + "describe_segments", + schema_fingerprint(desc.fields, shape_type=desc.shapeType, oid_field=desc.OIDFieldName, srid=srid), +) print( f"describe_segments ok shapeType={desc.shapeType} " f"fields={len(desc.fields)} oidField={desc.OIDFieldName} srid={srid}" diff --git a/packages/honua-gp/eval/scripts/describe_segments_fields.py b/packages/honua-gp/eval/scripts/describe_segments_fields.py index 6b9bfd6..9a24b64 100644 --- a/packages/honua-gp/eval/scripts/describe_segments_fields.py +++ b/packages/honua-gp/eval/scripts/describe_segments_fields.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,9 @@ desc = arcpy.Describe("segments") names = [field.name for field in desc.fields] +srid = desc.spatialReference.factoryCode if desc.spatialReference else None +emit_response( + "describe_segments_fields", + schema_fingerprint(desc.fields, shape_type=desc.shapeType, oid_field=desc.OIDFieldName, srid=srid), +) print(f"describe_segments_fields ok fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py index e11b487..fd57830 100644 --- a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py +++ b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -27,4 +28,9 @@ cursor.insertRow(["OPEN", "Main St"]) cursor.insertRow(["OPEN", "Elm Ave"]) cursor.insertRow(["CLOSED", "Side Rd"]) + # Flush explicitly (rather than relying on the implicit __exit__ flush) so + # the applyEdits result is available here to fingerprint. + result = cursor.flush() + +emit_response("insert_cursor_append_rows", apply_edits_fingerprint(result)) print("insert_cursor_append_rows ok") diff --git a/packages/honua-gp/eval/scripts/list_fields_segments.py b/packages/honua-gp/eval/scripts/list_fields_segments.py index a73bbc6..c386acf 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments") names = [f.name for f in fields] +emit_response("list_fields_segments", schema_fingerprint(fields)) print(f"list_fields_segments ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py b/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py index 6a87715..7490284 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments", field_type="String") names = [f.name for f in fields] +emit_response("list_fields_segments_filtered", schema_fingerprint(fields)) print(f"list_fields_segments_filtered ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py b/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py index b657679..3d3cfdc 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments", wild_card="STAT*") names = [f.name for f in fields] +emit_response("list_fields_segments_wildcard", schema_fingerprint(fields)) print(f"list_fields_segments_wildcard ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/update_cursor_close_status.py b/packages/honua-gp/eval/scripts/update_cursor_close_status.py index 1a18207..1e5790e 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_close_status.py +++ b/packages/honua-gp/eval/scripts/update_cursor_close_status.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -28,4 +29,9 @@ if row[1] == "CLOSED": row[1] = "ARCHIVED" cursor.updateRow(row) + # Flush explicitly (rather than relying on the implicit __exit__ flush) so + # the applyEdits result is available here to fingerprint. + result = cursor.flush() + +emit_response("update_cursor_close_status", apply_edits_fingerprint(result)) print("update_cursor_close_status ok") diff --git a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py index 8fa0f13..c1e8582 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py +++ b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -27,4 +28,12 @@ for row in cursor: if row[1] == "CLOSED": cursor.deleteRow() + # Flush explicitly (rather than relying on the implicit __exit__ flush) so + # the applyEdits result is available here to fingerprint. update_cursor_ + # close_status runs alphabetically first and archives every CLOSED row on + # the same seeded layer, so this deterministically finds zero rows to + # delete -- that is itself the stable oracle, not an absent one. + result = cursor.flush() + +emit_response("update_cursor_delete_closed", apply_edits_fingerprint(result)) print("update_cursor_delete_closed ok") diff --git a/packages/honua-gp/honua_gp/da/__init__.py b/packages/honua-gp/honua_gp/da/__init__.py index c76cf9c..6d165d8 100644 --- a/packages/honua-gp/honua_gp/da/__init__.py +++ b/packages/honua-gp/honua_gp/da/__init__.py @@ -102,13 +102,45 @@ def _oid_from_attrs(attrs: Mapping[str, Any]) -> Any: return None -def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: +def _attrs_for_feature(feature: Any) -> dict[str, Any]: + """Extract the attribute/property mapping from any feature shape a Source yields. + + The eval stub's ``_StubFeature`` exposes ``.attributes``; the real + ``honua_sdk.models.QueryFeature`` (what ``Source.iter_features`` actually + yields against a live server) exposes GeoJSON-shaped ``.properties`` + instead -- there is no ``.attributes``. Without the ``.properties`` + fallback every live cursor read silently degraded to an empty mapping + (every field value ``None``), which stub-mode CI could never catch since + the stub always has ``.attributes``. + """ + if hasattr(feature, "attributes"): - attrs = dict(feature.attributes or {}) - elif isinstance(feature, dict): - attrs = dict(feature.get("attributes") or feature.get("properties") or {}) - else: - attrs = {} + return dict(feature.attributes or {}) + if hasattr(feature, "properties"): + return dict(feature.properties or {}) + if isinstance(feature, dict): + return dict(feature.get("attributes") or feature.get("properties") or {}) + return {} + + +def _oid_from_feature(feature: Any, attrs: Mapping[str, Any]) -> Any: + """Resolve a feature's OID, preferring the SDK's protocol-neutral ``.id``. + + ``QueryFeature.id`` is the SDK's already-resolved stable identifier -- + it does not depend on the server's object-id field name matching one of + ``_OID_KEYS`` (the client-compat seed's object-id field is the lower-case + ``objectid``, which ``_OID_KEYS`` never matched). Fall back to scanning + ``attrs`` for the stub's legacy shape, which has no ``.id``. + """ + + feature_id = getattr(feature, "id", None) + if feature_id is not None: + return feature_id + return _oid_from_attrs(attrs) + + +def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: + attrs = _attrs_for_feature(feature) geometry = getattr(feature, "geometry", None) if geometry is None and isinstance(feature, dict): @@ -119,7 +151,7 @@ def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: if field.upper() in {"SHAPE@", "SHAPE@JSON"}: out.append(_shape_value(geometry, field.upper())) elif field.upper() == "OID@": - out.append(_oid_from_attrs(attrs)) + out.append(_oid_from_feature(feature, attrs)) else: out.append(attrs.get(field)) return tuple(out) @@ -554,8 +586,7 @@ def _close(self, exc_type, exc, tb) -> None: }) def _extract_oid(self, feature: Any) -> Any: - attrs = getattr(feature, "attributes", None) or (feature.get("attributes") if isinstance(feature, dict) else None) or {} - return _oid_from_attrs(attrs) + return _oid_from_feature(feature, _attrs_for_feature(feature)) class InsertCursor(_BaseCursor): diff --git a/packages/honua-gp/tests/test_da_cursors.py b/packages/honua-gp/tests/test_da_cursors.py index 9d52a9f..3947c17 100644 --- a/packages/honua-gp/tests/test_da_cursors.py +++ b/packages/honua-gp/tests/test_da_cursors.py @@ -431,6 +431,80 @@ def source(self, descriptor: Any) -> Any: _ = second # silence unused-var warnings +def test_search_cursor_reads_query_feature_properties() -> None: + """SearchCursor must read values from a real ``QueryFeature`` (``.properties`` + / ``.id``), not just the eval stub's ``.attributes`` shape. + + ``Source.iter_features`` against a live honua-server yields + ``honua_sdk.models.QueryFeature`` -- GeoJSON-shaped ``.properties`` plus a + protocol-neutral ``.id``, with no ``.attributes`` at all. Before the fix, + ``_values_for_row`` only recognized ``.attributes``, so every field + (including ``OID@``) silently came back ``None`` against a real server; + the eval stub's ``_StubFeature`` (which does have ``.attributes``) masked + this in stub-mode CI. + """ + + from honua_sdk import QueryFeature + + class _QueryFeatureSource: + def iter_features(self, **_: Any) -> Any: + return iter([ + QueryFeature(id=1, properties={"STATUS": "CLOSED", "name": "Side Rd"}), + QueryFeature(id=2, properties={"STATUS": "OPEN", "name": "Main St"}), + ]) + + class _QueryFeatureClient: + def source(self, descriptor: Any) -> Any: + return _QueryFeatureSource() + + honua_gp.configure(client=_QueryFeatureClient()) + + with honua_gp.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + rows = list(cursor) + + assert rows == [(1, "CLOSED", "Side Rd"), (2, "OPEN", "Main St")] + + +def test_update_cursor_extracts_oid_from_query_feature_id() -> None: + """UpdateCursor.updateRow/deleteRow must resolve the OID from ``QueryFeature.id``, + not just an ``OBJECTID``-keyed ``attributes`` mapping. + + Before the fix, a real ``QueryFeature`` (no ``.attributes``, and whose + object-id field may not even be named ``OBJECTID`` server-side -- the + client-compat seed's is the lower-case ``objectid``) always resolved to + ``_extract_oid() is None``, so ``updateRow``/``deleteRow`` raised + ``HonuaGpConfigurationError`` for every real feature. + """ + + from honua_sdk import QueryFeature + + edits: dict[str, Any] = {} + + class _QueryFeatureSource: + def iter_features(self, **_: Any) -> Any: + return iter([QueryFeature(id=16, properties={"STATUS": "CLOSED"})]) + + def apply_edits(self, **kwargs: Any) -> Any: + edits.update(kwargs) + return {"ok": True} + + class _QueryFeatureClient: + def source(self, descriptor: Any) -> Any: + return _QueryFeatureSource() + + honua_gp.configure(client=_QueryFeatureClient()) + + with honua_gp.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + row = next(cursor) + assert row[0] == 16 + row[1] = "ARCHIVED" + cursor.updateRow(row) + cursor.deleteRow() + + assert edits["updates"][0]["attributes"]["OBJECTID"] == 16 + assert edits["deletes"] == [16] + + def test_cursor_open_failure_reports_real_error_kind(tmp_path) -> None: # Unconfigured session: _open() raises HonuaGpConfigurationError before # the caller's `with` block starts. The audit should record that real From 3b46f68a3bfe257090aa94426b20cc7e3e7f646d Mon Sep 17 00:00:00 2001 From: Mike McDougall Date: Sun, 13 Sep 2026 23:53:25 -1000 Subject: [PATCH 3/3] fix(gp-eval): grade dataset contents for write cursors and the table view The write-cursor oracles recorded applyEdits counts only, and update_cursor_delete_closed was blessed to a vacuous delete_count 0: an earlier script had already archived every CLOSED row, so a broken deleteRow passed. make_table_view still passed as an allow-listed "unblessed" script. - insert_cursor_append_rows reads the inserted object ids back and records the persisted rows. - update_cursor_close_status / update_cursor_delete_closed insert their own fixture row, scope the edit to it by name, and record the value left behind / that the deleted row is gone. They no longer depend on script order or on a previous run's rows. - make_table_view applies a where clause and records the rows the server returns through the view (the five 'active' seed rows). - The allow-list is gone: a live supported script without a response oracle fails, and a unit test fails the stub lane when a supported golden has none. - The generator now owns these scripts' oracles, so regenerating no longer drops them. --- packages/honua-gp/docs/golden-eval.md | 48 ++++++------ .../honua-gp/eval/UNBLESSED_ALLOWLIST.json | 3 - packages/honua-gp/eval/_emit.py | 23 ++++++ packages/honua-gp/eval/_generate_scripts.py | 74 +++++++++++++++---- .../golden/insert_cursor_append_rows.json | 16 +++- .../honua-gp/eval/golden/make_table_view.json | 27 ++++++- .../golden/update_cursor_close_status.json | 11 ++- .../golden/update_cursor_delete_closed.json | 6 +- packages/honua-gp/eval/run_eval.py | 72 +++--------------- .../eval/scripts/insert_cursor_append_rows.py | 16 ++-- .../honua-gp/eval/scripts/make_table_view.py | 13 +++- .../scripts/update_cursor_close_status.py | 24 ++++-- .../scripts/update_cursor_delete_closed.py | 26 ++++--- packages/honua-gp/tests/test_eval_harness.py | 58 +++++++++++++++ 14 files changed, 285 insertions(+), 132 deletions(-) delete mode 100644 packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json diff --git a/packages/honua-gp/docs/golden-eval.md b/packages/honua-gp/docs/golden-eval.md index 2706495..3b3f96d 100644 --- a/packages/honua-gp/docs/golden-eval.md +++ b/packages/honua-gp/docs/golden-eval.md @@ -85,27 +85,33 @@ Request fingerprints are written in any mode; response fingerprints only in live mode (the stub's canned `href` carries no real values and must never be frozen as an oracle). -## Unblessed supported scripts must be explicit (`UNBLESSED_ALLOWLIST.json`) - -A live-mode run grades every non-`expected_failure` script's response layer -against its golden `response` block. If a supported script has no `response` -block at all, the harness does **not** silently pass it: it fails unless the -script's stem is listed in `eval/UNBLESSED_ALLOWLIST.json` with a documented -reason. This closes the gap issue #202 found -- an un-oracled supported -script used to grade as an "unblessed" pass, so a response-parsing regression -in an unblessed script's operation could sail through the live smoke lane -silently. - -Two ways a supported script ends up here: - -* **Not yet blessed** -- bless it (see above) and commit the oracle. This is - the expected outcome for almost every case; a newly-added supported script - must be blessed before merge, or the live smoke lane fails on it. -* **Genuinely not capturable** -- the operation makes no honua-server HTTP - round trip to fingerprint (e.g. `MakeTableView`/`MakeFeatureLayer`, which - only register a client-side session alias). Add an entry to - `UNBLESSED_ALLOWLIST.json` explaining why, instead of leaving the gap - implicit. +## Every supported script has a response oracle + +A supported (non-`expected_failure`) script with no golden `response` block +used to grade as an "unblessed" pass in live mode, so a regression in its +operation could pass the live smoke lane silently (issue #202). There is no +unblessed set any more, and two checks keep it that way: + +* In live mode, `run_eval.py` fails a supported script that has no `response` + block. +* `tests/test_eval_harness.py::test_every_supported_script_has_a_response_oracle` + fails the unit tests (which run in the stub lane too) as soon as a supported + golden lacks one. + +The oracle must be an observable result, not the submitted payload: + +* **Schema introspection** (`Describe` / `ListFields`) records the field + names, types, OID field, shape type and SRID the server reported for the + seeded layer. +* **Write cursors** (`InsertCursor` / `UpdateCursor`) record the applyEdits + counts **and** the rows read back with a `SearchCursor` afterwards: the rows + an insert persisted, the value an update left behind, and that a deleted row + is gone. `update_cursor_close_status` and `update_cursor_delete_closed` + insert their own fixture row and scope their edit to it by name, so they do + not depend on script order or leftovers from an earlier run. +* **Session aliases** (`MakeTableView`) make no request by themselves, so the + script reads through the view (`GetCount` + `SearchCursor`) and records the + rows the server returned under the view's where clause. ## Determinism note (live mode) diff --git a/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json b/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json deleted file mode 100644 index 18cc9cd..0000000 --- a/packages/honua-gp/eval/UNBLESSED_ALLOWLIST.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "make_table_view": "management.MakeTableView (and MakeFeatureLayer) resolve to honua_gp.management._make_layer_handler, which only registers a client-side LayerAlias in the session -- it makes no honua-server HTTP round trip. There is no server response for the harness to fingerprint; the request/plumbing layers (already graded) are this script's full coverage. See issue #202 and docs/golden-eval.md." -} diff --git a/packages/honua-gp/eval/_emit.py b/packages/honua-gp/eval/_emit.py index e6ccacc..ef73adb 100644 --- a/packages/honua-gp/eval/_emit.py +++ b/packages/honua-gp/eval/_emit.py @@ -185,8 +185,31 @@ def apply_edits_fingerprint(result: Any) -> dict[str, Any]: } +def edited_object_ids(result: Any, operation: str) -> set[str]: + """Return the server-assigned object ids of the successful ``operation`` edits. + + ``operation`` is ``"add"``, ``"update"`` or ``"delete"``. The ids are + never frozen into a golden (they differ across seeds); scripts use them + to read back exactly the rows their own edit touched, so the response + oracle records what the dataset holds afterwards rather than what the + script submitted. Ids are compared as strings because + ``QueryFeature.id`` may be a ``str`` or an ``int``. The stub's plain-dict + result carries no server ids, so it yields an empty set. + """ + + if result is None or isinstance(result, Mapping): + return set() + results = getattr(result, f"{operation}_results", ()) + return { + str(entry.object_id) + for entry in results + if getattr(entry, "success", False) and getattr(entry, "object_id", None) is not None + } + + __all__ = [ "apply_edits_fingerprint", + "edited_object_ids", "emit_response", "feature_layer_fingerprint", "schema_fingerprint", diff --git a/packages/honua-gp/eval/_generate_scripts.py b/packages/honua-gp/eval/_generate_scripts.py index d86b4f2..b643f3b 100644 --- a/packages/honua-gp/eval/_generate_scripts.py +++ b/packages/honua-gp/eval/_generate_scripts.py @@ -166,12 +166,18 @@ def _supported( _supported( "make_table_view", "transport", - "Make a table view for inspection.", - """arcpy.management.MakeTableView("segments_attrs", "segments_view") -print("make_table_view ok") + "Make a filtered table view, then read the server rows through it.", + """arcpy.management.MakeTableView("segments_attrs", "segments_view", "status = 'active'") +# MakeTableView itself only registers a session alias; the observable result is +# what the server returns when the view (and its where clause) is read. +view_count = int(arcpy.management.GetCount("segments_view")) +with arcpy.da.SearchCursor("segments_view", ["name", "status"]) as cursor: + view_rows = sorted([row[0], row[1]] for row in cursor) +print(f"make_table_view ok count={view_count}") """, - 1, + 3, "make_table_view ok", + response_emit=_val_emit("make_table_view", "{'view_count': view_count, 'view_rows': view_rows}"), ) _supported( @@ -260,44 +266,86 @@ def _supported( _supported( "update_cursor_close_status", "transport", - "UpdateCursor: flip CLOSED rows to ARCHIVED.", - """with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + "UpdateCursor: flip CLOSED rows to ARCHIVED, then read the rows back.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so the oracle does not +# depend on which other scripts ran first or on a previous run's leftovers. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Close Status Rd"]) +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Close Status Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": row[1] = "ARCHIVED" cursor.updateRow(row) + edits = cursor.flush() +updated = edited_object_ids(edits, "update") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"], "name = 'Close Status Rd'") as cursor: + rows = list(cursor) +updated_rows = sorted([row[1], row[2]] for row in rows if str(row[0]) in updated) +closed_remaining = sum(1 for row in rows if row[1] == "CLOSED") print("update_cursor_close_status ok") """, - 1, + 3, "update_cursor_close_status ok", + response_emit=_val_emit( + "update_cursor_close_status", + "{**apply_edits_fingerprint(edits), 'updated_rows': updated_rows, 'closed_remaining': closed_remaining}", + ), ) _supported( "update_cursor_delete_closed", "transport", - "UpdateCursor: delete CLOSED rows.", - """with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + "UpdateCursor: delete CLOSED rows, then confirm they are gone.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so there is always a CLOSED +# row to delete -- a zero-delete run is a failure, not a vacuous pass. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Delete Closed Rd"]) + inserted = edited_object_ids(cursor.flush(), "add") +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": cursor.deleteRow() + edits = cursor.flush() +deleted = edited_object_ids(edits, "delete") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: + rows = list(cursor) print("update_cursor_delete_closed ok") """, - 1, + 3, "update_cursor_delete_closed ok", + response_emit=_val_emit( + "update_cursor_delete_closed", + "{**apply_edits_fingerprint(edits), 'deleted_inserted_row': bool(inserted) and deleted == inserted, " + "'rows_remaining': len(rows)}", + ), ) _supported( "insert_cursor_append_rows", "transport", - "InsertCursor: append three rows.", - """with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + "InsertCursor: append three rows, then read the persisted rows back.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: cursor.insertRow(["OPEN", "Main St"]) cursor.insertRow(["OPEN", "Elm Ave"]) cursor.insertRow(["CLOSED", "Side Rd"]) + edits = cursor.flush() +added = edited_object_ids(edits, "add") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + persisted_rows = sorted([row[1], row[2]] for row in cursor if str(row[0]) in added) print("insert_cursor_append_rows ok") """, - 1, + 2, "insert_cursor_append_rows ok", + response_emit=_val_emit( + "insert_cursor_append_rows", + "{**apply_edits_fingerprint(edits), 'persisted_rows': persisted_rows}", + ), ) _supported( diff --git a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json index 8cca5b1..3b14b55 100644 --- a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json +++ b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json @@ -1,13 +1,27 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 2, "stdout_contains": "insert_cursor_append_rows ok" }, "response": { "add_count": 3, "all_succeeded": true, "delete_count": 0, + "persisted_rows": [ + [ + "CLOSED", + "Side Rd" + ], + [ + "OPEN", + "Elm Ave" + ], + [ + "OPEN", + "Main St" + ] + ], "update_count": 0 }, "schema_version": 2 diff --git a/packages/honua-gp/eval/golden/make_table_view.json b/packages/honua-gp/eval/golden/make_table_view.json index a0dcfe0..f064096 100644 --- a/packages/honua-gp/eval/golden/make_table_view.json +++ b/packages/honua-gp/eval/golden/make_table_view.json @@ -1,8 +1,33 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "make_table_view ok" }, + "response": { + "view_count": 5, + "view_rows": [ + [ + "alpha", + "active" + ], + [ + "epsilon", + "active" + ], + [ + "eta", + "active" + ], + [ + "gamma", + "active" + ], + [ + "iota", + "active" + ] + ] + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/update_cursor_close_status.json b/packages/honua-gp/eval/golden/update_cursor_close_status.json index a6475fe..719a2b7 100644 --- a/packages/honua-gp/eval/golden/update_cursor_close_status.json +++ b/packages/honua-gp/eval/golden/update_cursor_close_status.json @@ -1,14 +1,21 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "update_cursor_close_status ok" }, "response": { "add_count": 0, "all_succeeded": true, + "closed_remaining": 0, "delete_count": 0, - "update_count": 1 + "update_count": 1, + "updated_rows": [ + [ + "ARCHIVED", + "Close Status Rd" + ] + ] }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json index 17e9844..24c6832 100644 --- a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json +++ b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json @@ -1,13 +1,15 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "update_cursor_delete_closed ok" }, "response": { "add_count": 0, "all_succeeded": true, - "delete_count": 0, + "delete_count": 1, + "deleted_inserted_row": true, + "rows_remaining": 0, "update_count": 0 }, "schema_version": 2 diff --git a/packages/honua-gp/eval/run_eval.py b/packages/honua-gp/eval/run_eval.py index a8f4a4b..c5fd619 100644 --- a/packages/honua-gp/eval/run_eval.py +++ b/packages/honua-gp/eval/run_eval.py @@ -62,7 +62,6 @@ PACKAGE_ROOT = Path(__file__).resolve().parents[1] DEFAULT_SCRIPT_DIR = PACKAGE_ROOT / "eval" / "scripts" DEFAULT_GOLDEN_DIR = PACKAGE_ROOT / "eval" / "golden" -DEFAULT_UNBLESSED_ALLOWLIST = PACKAGE_ROOT / "eval" / "UNBLESSED_ALLOWLIST.json" DEFAULT_PASS_RATE = 0.70 @@ -178,29 +177,6 @@ def _golden_is_expected_failure(script: Path, golden: dict[str, Any] | None) -> return "expected_failure" in script.stem -def load_unblessed_allowlist(path: Path) -> dict[str, str]: - """Load the {script_stem: reason} map of supported scripts with no capturable response oracle. - - A live-mode supported script with no golden ``response`` block is either - a gap that has never been blessed (a regression risk -- see issue #202) - or a script whose operation genuinely has no server round trip to - fingerprint (e.g. ``MakeTableView``, a session-local alias registration). - Only the second case belongs here, with the reason documented; anything - else must fail so a newly-added supported script cannot silently regrow - the unblessed set. - """ - - if not path.exists(): - return {} - try: - data = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - if not isinstance(data, dict): - return {} - return {str(k): str(v) for k, v in data.items()} - - # --------------------------------------------------------------------------- # Captured value oracles # --------------------------------------------------------------------------- @@ -337,7 +313,6 @@ def _grade( request_actual: list[dict[str, Any]], response_actual: dict[str, Any] | None, live_mode: bool, - unblessed_allowlist: dict[str, str] | None = None, ) -> tuple[str, bool, str | None, dict[str, str]]: """Grade one script across plumbing + request + response layers.""" @@ -415,27 +390,18 @@ def _grade( ) else: checks["response"] = "pass" - elif live_mode and golden is not None and not expected_failure: - # Live run, supported script, but no response oracle recorded. Pass - # ONLY when the script is explicitly documented as non-capturable in - # the unblessed allow-list (see UNBLESSED_ALLOWLIST.json) -- otherwise - # this is exactly the silent-pass gap issue #202 closed: a new - # supported script (or a regression that dropped an oracle) must fail - # loudly, not disappear into an "unblessed" pass. - allowlist = unblessed_allowlist or {} - if script.stem in allowlist: - checks["response"] = "unblessed(allowed)" - else: - checks["response"] = "fail" - return ( - "fail", - False, - f"supported script {script.stem!r} has no response oracle and is not in " - "eval/UNBLESSED_ALLOWLIST.json -- bless it " - "(HONUA_GP_EVAL_USE_STUB=0 ... run_eval.py --update-golden) or document why it " - "has no capturable response in the allow-list", - checks, - ) + elif live_mode and not expected_failure: + # Live run, supported script, but no response oracle recorded. An + # unblessed pass is not evidence (issue #202): a supported script + # must record what the server actually returned, so it fails. + checks["response"] = "fail" + return ( + "fail", + False, + f"supported script {script.stem!r} has no response oracle -- make it emit one and bless it " + "(HONUA_GP_EVAL_USE_STUB=0 ... run_eval.py --update-golden)", + checks, + ) return "pass", False, None, checks @@ -474,11 +440,9 @@ def run( timeout: float, pass_threshold: float, update_golden: bool = False, - unblessed_allowlist_path: Path = DEFAULT_UNBLESSED_ALLOWLIST, ) -> EvalSummary: scripts = sorted(p for p in script_dir.glob("*.py") if not p.name.startswith("_")) live_mode = live_values_available() - unblessed_allowlist = load_unblessed_allowlist(unblessed_allowlist_path) summary = EvalSummary(total=len(scripts), pass_threshold=pass_threshold, live_mode=live_mode) result_root = Path(tempfile.mkdtemp(prefix="honua-gp-eval-results-")) for script in scripts: @@ -532,7 +496,6 @@ def run( request_actual=request_actual, response_actual=response_actual, live_mode=live_mode, - unblessed_allowlist=unblessed_allowlist, ) result = ScriptResult( name=script.name, @@ -632,16 +595,6 @@ def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run the honua-gp compatibility eval suite.") parser.add_argument("--scripts", type=Path, default=DEFAULT_SCRIPT_DIR, help="Directory of eval scripts.") parser.add_argument("--golden", type=Path, default=DEFAULT_GOLDEN_DIR, help="Directory of golden reference values.") - parser.add_argument( - "--unblessed-allowlist", - type=Path, - default=DEFAULT_UNBLESSED_ALLOWLIST, - help=( - "JSON {script_stem: reason} map of supported scripts documented as having no " - "capturable live response oracle. A live-mode supported script with no response " - "oracle and no allow-list entry fails the run (see issue #202)." - ), - ) parser.add_argument( "--audit-root", type=Path, @@ -701,7 +654,6 @@ def main(argv: Sequence[str] | None = None) -> int: timeout=args.timeout, pass_threshold=args.pass_threshold, update_golden=args.update_golden, - unblessed_allowlist_path=args.unblessed_allowlist, ) write_json(summary, args.output_json) write_junit(summary, args.output_junit) diff --git a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py index fd57830..93e35e3 100644 --- a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py +++ b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py @@ -1,4 +1,4 @@ -"""InsertCursor: append three rows.""" +"""InsertCursor: append three rows, then read the persisted rows back.""" import sys from pathlib import Path @@ -9,7 +9,6 @@ if candidate not in sys.path: sys.path.insert(0, candidate) -from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -24,13 +23,16 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True +from eval._emit import apply_edits_fingerprint, edited_object_ids + with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: cursor.insertRow(["OPEN", "Main St"]) cursor.insertRow(["OPEN", "Elm Ave"]) cursor.insertRow(["CLOSED", "Side Rd"]) - # Flush explicitly (rather than relying on the implicit __exit__ flush) so - # the applyEdits result is available here to fingerprint. - result = cursor.flush() - -emit_response("insert_cursor_append_rows", apply_edits_fingerprint(result)) + edits = cursor.flush() +added = edited_object_ids(edits, "add") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + persisted_rows = sorted([row[1], row[2]] for row in cursor if str(row[0]) in added) print("insert_cursor_append_rows ok") +from eval._emit import emit_response +emit_response('insert_cursor_append_rows', {**apply_edits_fingerprint(edits), 'persisted_rows': persisted_rows}) diff --git a/packages/honua-gp/eval/scripts/make_table_view.py b/packages/honua-gp/eval/scripts/make_table_view.py index 9ffa4f7..2b23e8d 100644 --- a/packages/honua-gp/eval/scripts/make_table_view.py +++ b/packages/honua-gp/eval/scripts/make_table_view.py @@ -1,4 +1,4 @@ -"""Make a table view for inspection.""" +"""Make a filtered table view, then read the server rows through it.""" import sys from pathlib import Path @@ -23,5 +23,12 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -arcpy.management.MakeTableView("segments_attrs", "segments_view") -print("make_table_view ok") +arcpy.management.MakeTableView("segments_attrs", "segments_view", "status = 'active'") +# MakeTableView itself only registers a session alias; the observable result is +# what the server returns when the view (and its where clause) is read. +view_count = int(arcpy.management.GetCount("segments_view")) +with arcpy.da.SearchCursor("segments_view", ["name", "status"]) as cursor: + view_rows = sorted([row[0], row[1]] for row in cursor) +print(f"make_table_view ok count={view_count}") +from eval._emit import emit_response +emit_response('make_table_view', {'view_count': view_count, 'view_rows': view_rows}) diff --git a/packages/honua-gp/eval/scripts/update_cursor_close_status.py b/packages/honua-gp/eval/scripts/update_cursor_close_status.py index 1e5790e..201f16b 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_close_status.py +++ b/packages/honua-gp/eval/scripts/update_cursor_close_status.py @@ -1,4 +1,4 @@ -"""UpdateCursor: flip CLOSED rows to ARCHIVED.""" +"""UpdateCursor: flip CLOSED rows to ARCHIVED, then read the rows back.""" import sys from pathlib import Path @@ -9,7 +9,6 @@ if candidate not in sys.path: sys.path.insert(0, candidate) -from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -24,14 +23,23 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: +from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so the oracle does not +# depend on which other scripts ran first or on a previous run's leftovers. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Close Status Rd"]) +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Close Status Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": row[1] = "ARCHIVED" cursor.updateRow(row) - # Flush explicitly (rather than relying on the implicit __exit__ flush) so - # the applyEdits result is available here to fingerprint. - result = cursor.flush() - -emit_response("update_cursor_close_status", apply_edits_fingerprint(result)) + edits = cursor.flush() +updated = edited_object_ids(edits, "update") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"], "name = 'Close Status Rd'") as cursor: + rows = list(cursor) +updated_rows = sorted([row[1], row[2]] for row in rows if str(row[0]) in updated) +closed_remaining = sum(1 for row in rows if row[1] == "CLOSED") print("update_cursor_close_status ok") +from eval._emit import emit_response +emit_response('update_cursor_close_status', {**apply_edits_fingerprint(edits), 'updated_rows': updated_rows, 'closed_remaining': closed_remaining}) diff --git a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py index c1e8582..a0dcb25 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py +++ b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py @@ -1,4 +1,4 @@ -"""UpdateCursor: delete CLOSED rows.""" +"""UpdateCursor: delete CLOSED rows, then confirm they are gone.""" import sys from pathlib import Path @@ -9,7 +9,6 @@ if candidate not in sys.path: sys.path.insert(0, candidate) -from eval._emit import apply_edits_fingerprint, emit_response from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -24,16 +23,21 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: +from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so there is always a CLOSED +# row to delete -- a zero-delete run is a failure, not a vacuous pass. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Delete Closed Rd"]) + inserted = edited_object_ids(cursor.flush(), "add") +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": cursor.deleteRow() - # Flush explicitly (rather than relying on the implicit __exit__ flush) so - # the applyEdits result is available here to fingerprint. update_cursor_ - # close_status runs alphabetically first and archives every CLOSED row on - # the same seeded layer, so this deterministically finds zero rows to - # delete -- that is itself the stable oracle, not an absent one. - result = cursor.flush() - -emit_response("update_cursor_delete_closed", apply_edits_fingerprint(result)) + edits = cursor.flush() +deleted = edited_object_ids(edits, "delete") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: + rows = list(cursor) print("update_cursor_delete_closed ok") +from eval._emit import emit_response +emit_response('update_cursor_delete_closed', {**apply_edits_fingerprint(edits), 'deleted_inserted_row': bool(inserted) and deleted == inserted, 'rows_remaining': len(rows)}) diff --git a/packages/honua-gp/tests/test_eval_harness.py b/packages/honua-gp/tests/test_eval_harness.py index ad6d168..71b6f84 100644 --- a/packages/honua-gp/tests/test_eval_harness.py +++ b/packages/honua-gp/tests/test_eval_harness.py @@ -30,6 +30,23 @@ def test_eval_scripts_pair_with_golden_records() -> None: assert golden.exists(), f"Missing golden file for {script.name}" +def test_every_supported_script_has_a_response_oracle() -> None: + """A supported script must record what the server returned (#202). + + Stub CI never grades the response layer, so without this check a new + supported script with no ``response`` block would only surface in the live + smoke lane. The unblessed set is empty and must stay empty. + """ + + unblessed = [] + for golden_path in sorted(_PACKAGE_ROOT.glob("eval/golden/*.json")): + golden = json.loads(golden_path.read_text(encoding="utf-8")) + expected_failure = golden.get("expected_failure", "expected_failure" in golden_path.stem) + if not expected_failure and not golden.get("response"): + unblessed.append(golden_path.stem) + assert unblessed == [], f"supported eval scripts with no response oracle: {unblessed}" + + def test_committed_matrix_matches_generated_output() -> None: committed = (_PACKAGE_ROOT / "docs" / "compatibility-matrix.md").read_text(encoding="utf-8") generated = render_compat_matrix() @@ -205,6 +222,47 @@ def test_grade_response_value_diff_is_live_only(tmp_path: Path) -> None: assert reason is not None and "response value mismatch" in reason +def test_grade_live_supported_script_without_response_oracle_fails(tmp_path: Path) -> None: + """Live mode never passes a supported script as "unblessed" (#202).""" + + script = tmp_path / "make_widget_view.py" + script.write_text("", encoding="utf-8") + golden = { + "schema_version": 2, + "expected_failure": False, + "plumbing": {"audit_lines": 1, "stdout_contains": "make_widget_view ok"}, + } + common = dict(exit_code=0, audit_lines=1, golden=golden, stdout="make_widget_view ok\n", stderr="") + + status, _, reason, checks = _grade(script, response_actual={"view_count": 5}, live_mode=True, **common) + assert status == "fail" + assert checks["response"] == "fail" + assert reason is not None and "has no response oracle" in reason + + # Stub mode grades no response layer, so the same golden still passes there. + status, _, _, checks = _grade(script, live_mode=False, **common) + assert status == "pass" + assert "response" not in checks + + +def test_edited_object_ids_reads_successful_server_ids_only() -> None: + from _emit import edited_object_ids + from honua_sdk.models import ApplyEditsResult + + result = ApplyEditsResult.from_dict( + { + "addResults": [{"objectId": 11, "success": True}, {"objectId": 12, "success": False}], + "deleteResults": [{"objectId": 7, "success": True}], + } + ) + assert edited_object_ids(result, "add") == {"11"} + assert edited_object_ids(result, "delete") == {"7"} + assert edited_object_ids(result, "update") == set() + # The stub's dict result and an empty flush carry no server ids. + assert edited_object_ids({"adds": [{"attributes": {}}]}, "add") == set() + assert edited_object_ids(None, "add") == set() + + def test_run_script_always_rebuilds_pythonpath_when_host_sets_it( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: