diff --git a/packages/honua-gp/docs/golden-eval.md b/packages/honua-gp/docs/golden-eval.md index cbff390..3b3f96d 100644 --- a/packages/honua-gp/docs/golden-eval.md +++ b/packages/honua-gp/docs/golden-eval.md @@ -85,6 +85,34 @@ Request fingerprints are written in any mode; response fingerprints only in live mode (the stub's canned `href` carries no real values and must never be frozen as an oracle). +## Every supported script has a response oracle + +A supported (non-`expected_failure`) script with no golden `response` block +used to grade as an "unblessed" pass in live mode, so a regression in its +operation could pass the live smoke lane silently (issue #202). There is no +unblessed set any more, and two checks keep it that way: + +* In live mode, `run_eval.py` fails a supported script that has no `response` + block. +* `tests/test_eval_harness.py::test_every_supported_script_has_a_response_oracle` + fails the unit tests (which run in the stub lane too) as soon as a supported + golden lacks one. + +The oracle must be an observable result, not the submitted payload: + +* **Schema introspection** (`Describe` / `ListFields`) records the field + names, types, OID field, shape type and SRID the server reported for the + seeded layer. +* **Write cursors** (`InsertCursor` / `UpdateCursor`) record the applyEdits + counts **and** the rows read back with a `SearchCursor` afterwards: the rows + an insert persisted, the value an update left behind, and that a deleted row + is gone. `update_cursor_close_status` and `update_cursor_delete_closed` + insert their own fixture row and scope their edit to it by name, so they do + not depend on script order or leftovers from an earlier run. +* **Session aliases** (`MakeTableView`) make no request by themselves, so the + script reads through the view (`GetCount` + `SearchCursor`) and records the + rows the server returned under the view's where clause. + ## Determinism note (live mode) The response oracles for count/row scripts record exact values pinned to the diff --git a/packages/honua-gp/eval/_emit.py b/packages/honua-gp/eval/_emit.py index 4df92f5..ef73adb 100644 --- a/packages/honua-gp/eval/_emit.py +++ b/packages/honua-gp/eval/_emit.py @@ -123,4 +123,94 @@ def feature_layer_fingerprint(result: Any) -> dict[str, Any]: return fingerprint -__all__ = ["emit_response", "feature_layer_fingerprint"] +def schema_fingerprint(fields: Any, *, shape_type: Any = None, oid_field: Any = None, srid: Any = None) -> dict[str, Any]: + """Normalize a Describe/ListFields result into a seed-stable fingerprint. + + ``fields`` is a sequence of ``FieldDescribe`` (or anything exposing + ``.name`` / ``.type``). The seeded ``segments`` / ``roads`` schemas are + fixed by ``tests/seed/client-compat-v1.sql`` and do not change between + live runs, so field names + types are a stable oracle -- unlike geometry + coordinates or generated object ids. + """ + + return { + "field_count": len(fields), + "field_names": [str(getattr(f, "name", "")) for f in fields], + "field_types": {str(getattr(f, "name", "")): getattr(f, "type", None) for f in fields}, + "shape_type": shape_type, + "oid_field": oid_field, + "srid": srid, + } + + +def apply_edits_fingerprint(result: Any) -> dict[str, Any]: + """Normalize an ``InsertCursor``/``UpdateCursor`` ``flush()`` return value. + + ``flush()`` returns different shapes depending on transport: the stub's + ``_StubApplyEditsResult.to_dict()`` (a plain dict with ``adds`` / + ``updates`` / ``deletes`` lists) versus the live SDK's + ``honua_sdk.models.ApplyEditsResult`` dataclass (``add_results`` / + ``update_results`` / ``delete_results`` sequences of typed + ``EditOperationResult``, each carrying a server-assigned ``object_id``). + Only success *counts* are captured, never object ids -- those are not + stable oracles across repeated seed runs. ``result`` is ``None`` when + ``flush()`` had nothing buffered (e.g. an UpdateCursor predicate matched + zero rows against the current seed state) -- that is itself a valid, + deterministic oracle (all counts zero, vacuously succeeded), not an + absence of one. + """ + + if result is None: + return {"add_count": 0, "update_count": 0, "delete_count": 0, "all_succeeded": True} + if isinstance(result, Mapping): + adds, updates, deletes = result.get("adds", []), result.get("updates", []), result.get("deletes", []) + return { + "add_count": len(adds), + "update_count": len(updates), + "delete_count": len(deletes), + "all_succeeded": True, + } + add_results = getattr(result, "add_results", ()) + update_results = getattr(result, "update_results", ()) + delete_results = getattr(result, "delete_results", ()) + all_succeeded = getattr(result, "all_succeeded", None) + if all_succeeded is None: + combined = [*add_results, *update_results, *delete_results] + all_succeeded = bool(combined) and all(getattr(r, "success", False) for r in combined) + return { + "add_count": len(add_results), + "update_count": len(update_results), + "delete_count": len(delete_results), + "all_succeeded": bool(all_succeeded), + } + + +def edited_object_ids(result: Any, operation: str) -> set[str]: + """Return the server-assigned object ids of the successful ``operation`` edits. + + ``operation`` is ``"add"``, ``"update"`` or ``"delete"``. The ids are + never frozen into a golden (they differ across seeds); scripts use them + to read back exactly the rows their own edit touched, so the response + oracle records what the dataset holds afterwards rather than what the + script submitted. Ids are compared as strings because + ``QueryFeature.id`` may be a ``str`` or an ``int``. The stub's plain-dict + result carries no server ids, so it yields an empty set. + """ + + if result is None or isinstance(result, Mapping): + return set() + results = getattr(result, f"{operation}_results", ()) + return { + str(entry.object_id) + for entry in results + if getattr(entry, "success", False) and getattr(entry, "object_id", None) is not None + } + + +__all__ = [ + "apply_edits_fingerprint", + "edited_object_ids", + "emit_response", + "feature_layer_fingerprint", + "schema_fingerprint", +] diff --git a/packages/honua-gp/eval/_generate_scripts.py b/packages/honua-gp/eval/_generate_scripts.py index d86b4f2..b643f3b 100644 --- a/packages/honua-gp/eval/_generate_scripts.py +++ b/packages/honua-gp/eval/_generate_scripts.py @@ -166,12 +166,18 @@ def _supported( _supported( "make_table_view", "transport", - "Make a table view for inspection.", - """arcpy.management.MakeTableView("segments_attrs", "segments_view") -print("make_table_view ok") + "Make a filtered table view, then read the server rows through it.", + """arcpy.management.MakeTableView("segments_attrs", "segments_view", "status = 'active'") +# MakeTableView itself only registers a session alias; the observable result is +# what the server returns when the view (and its where clause) is read. +view_count = int(arcpy.management.GetCount("segments_view")) +with arcpy.da.SearchCursor("segments_view", ["name", "status"]) as cursor: + view_rows = sorted([row[0], row[1]] for row in cursor) +print(f"make_table_view ok count={view_count}") """, - 1, + 3, "make_table_view ok", + response_emit=_val_emit("make_table_view", "{'view_count': view_count, 'view_rows': view_rows}"), ) _supported( @@ -260,44 +266,86 @@ def _supported( _supported( "update_cursor_close_status", "transport", - "UpdateCursor: flip CLOSED rows to ARCHIVED.", - """with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + "UpdateCursor: flip CLOSED rows to ARCHIVED, then read the rows back.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so the oracle does not +# depend on which other scripts ran first or on a previous run's leftovers. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Close Status Rd"]) +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Close Status Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": row[1] = "ARCHIVED" cursor.updateRow(row) + edits = cursor.flush() +updated = edited_object_ids(edits, "update") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"], "name = 'Close Status Rd'") as cursor: + rows = list(cursor) +updated_rows = sorted([row[1], row[2]] for row in rows if str(row[0]) in updated) +closed_remaining = sum(1 for row in rows if row[1] == "CLOSED") print("update_cursor_close_status ok") """, - 1, + 3, "update_cursor_close_status ok", + response_emit=_val_emit( + "update_cursor_close_status", + "{**apply_edits_fingerprint(edits), 'updated_rows': updated_rows, 'closed_remaining': closed_remaining}", + ), ) _supported( "update_cursor_delete_closed", "transport", - "UpdateCursor: delete CLOSED rows.", - """with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + "UpdateCursor: delete CLOSED rows, then confirm they are gone.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so there is always a CLOSED +# row to delete -- a zero-delete run is a failure, not a vacuous pass. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Delete Closed Rd"]) + inserted = edited_object_ids(cursor.flush(), "add") +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": cursor.deleteRow() + edits = cursor.flush() +deleted = edited_object_ids(edits, "delete") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: + rows = list(cursor) print("update_cursor_delete_closed ok") """, - 1, + 3, "update_cursor_delete_closed ok", + response_emit=_val_emit( + "update_cursor_delete_closed", + "{**apply_edits_fingerprint(edits), 'deleted_inserted_row': bool(inserted) and deleted == inserted, " + "'rows_remaining': len(rows)}", + ), ) _supported( "insert_cursor_append_rows", "transport", - "InsertCursor: append three rows.", - """with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + "InsertCursor: append three rows, then read the persisted rows back.", + """from eval._emit import apply_edits_fingerprint, edited_object_ids + +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: cursor.insertRow(["OPEN", "Main St"]) cursor.insertRow(["OPEN", "Elm Ave"]) cursor.insertRow(["CLOSED", "Side Rd"]) + edits = cursor.flush() +added = edited_object_ids(edits, "add") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + persisted_rows = sorted([row[1], row[2]] for row in cursor if str(row[0]) in added) print("insert_cursor_append_rows ok") """, - 1, + 2, "insert_cursor_append_rows ok", + response_emit=_val_emit( + "insert_cursor_append_rows", + "{**apply_edits_fingerprint(edits), 'persisted_rows': persisted_rows}", + ), ) _supported( diff --git a/packages/honua-gp/eval/golden/describe_segments.json b/packages/honua-gp/eval/golden/describe_segments.json index 03dfe97..53efea4 100644 --- a/packages/honua-gp/eval/golden/describe_segments.json +++ b/packages/honua-gp/eval/golden/describe_segments.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": "objectid", + "shape_type": "Point", + "srid": 4326 + }, + "schema_version": 2, "stdout_contains": "describe_segments ok" } diff --git a/packages/honua-gp/eval/golden/describe_segments_fields.json b/packages/honua-gp/eval/golden/describe_segments_fields.json index f15e202..6ce5f8f 100644 --- a/packages/honua-gp/eval/golden/describe_segments_fields.json +++ b/packages/honua-gp/eval/golden/describe_segments_fields.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": "objectid", + "shape_type": "Point", + "srid": 4326 + }, + "schema_version": 2, "stdout_contains": "describe_segments_fields ok" } diff --git a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json index 2c607bb..3b14b55 100644 --- a/packages/honua-gp/eval/golden/insert_cursor_append_rows.json +++ b/packages/honua-gp/eval/golden/insert_cursor_append_rows.json @@ -1,8 +1,28 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 2, "stdout_contains": "insert_cursor_append_rows ok" }, + "response": { + "add_count": 3, + "all_succeeded": true, + "delete_count": 0, + "persisted_rows": [ + [ + "CLOSED", + "Side Rd" + ], + [ + "OPEN", + "Elm Ave" + ], + [ + "OPEN", + "Main St" + ] + ], + "update_count": 0 + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/list_fields_segments.json b/packages/honua-gp/eval/golden/list_fields_segments.json index e3247d9..7050282 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments.json +++ b/packages/honua-gp/eval/golden/list_fields_segments.json @@ -1,5 +1,46 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 15, + "field_names": [ + "objectid", + "name", + "description", + "shape", + "status", + "count", + "ratio", + "active", + "created_at", + "event_date", + "event_time", + "uid", + "tags", + "numbers", + "eo:cloud_cover" + ], + "field_types": { + "active": "SmallInteger", + "count": "Integer", + "created_at": "Date", + "description": "String", + "eo:cloud_cover": "Double", + "event_date": "Date", + "event_time": "String", + "name": "String", + "numbers": "String", + "objectid": "OID", + "ratio": "Double", + "shape": "Geometry", + "status": "String", + "tags": "String", + "uid": "GUID" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments ok" } diff --git a/packages/honua-gp/eval/golden/list_fields_segments_filtered.json b/packages/honua-gp/eval/golden/list_fields_segments_filtered.json index 37b7dcf..39d2f1c 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments_filtered.json +++ b/packages/honua-gp/eval/golden/list_fields_segments_filtered.json @@ -1,5 +1,28 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 6, + "field_names": [ + "name", + "description", + "status", + "event_time", + "tags", + "numbers" + ], + "field_types": { + "description": "String", + "event_time": "String", + "name": "String", + "numbers": "String", + "status": "String", + "tags": "String" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments_filtered ok" } diff --git a/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json b/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json index 549b5a6..f58f74d 100644 --- a/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json +++ b/packages/honua-gp/eval/golden/list_fields_segments_wildcard.json @@ -1,5 +1,18 @@ { "audit_lines": 1, "expected_failure": false, + "response": { + "field_count": 1, + "field_names": [ + "status" + ], + "field_types": { + "status": "String" + }, + "oid_field": null, + "shape_type": null, + "srid": null + }, + "schema_version": 2, "stdout_contains": "list_fields_segments_wildcard ok" } diff --git a/packages/honua-gp/eval/golden/make_table_view.json b/packages/honua-gp/eval/golden/make_table_view.json index a0dcfe0..f064096 100644 --- a/packages/honua-gp/eval/golden/make_table_view.json +++ b/packages/honua-gp/eval/golden/make_table_view.json @@ -1,8 +1,33 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "make_table_view ok" }, + "response": { + "view_count": 5, + "view_rows": [ + [ + "alpha", + "active" + ], + [ + "epsilon", + "active" + ], + [ + "eta", + "active" + ], + [ + "gamma", + "active" + ], + [ + "iota", + "active" + ] + ] + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/update_cursor_close_status.json b/packages/honua-gp/eval/golden/update_cursor_close_status.json index 085ffae..719a2b7 100644 --- a/packages/honua-gp/eval/golden/update_cursor_close_status.json +++ b/packages/honua-gp/eval/golden/update_cursor_close_status.json @@ -1,8 +1,21 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "update_cursor_close_status ok" }, + "response": { + "add_count": 0, + "all_succeeded": true, + "closed_remaining": 0, + "delete_count": 0, + "update_count": 1, + "updated_rows": [ + [ + "ARCHIVED", + "Close Status Rd" + ] + ] + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json index 7725630..24c6832 100644 --- a/packages/honua-gp/eval/golden/update_cursor_delete_closed.json +++ b/packages/honua-gp/eval/golden/update_cursor_delete_closed.json @@ -1,8 +1,16 @@ { "expected_failure": false, "plumbing": { - "audit_lines": 1, + "audit_lines": 3, "stdout_contains": "update_cursor_delete_closed ok" }, + "response": { + "add_count": 0, + "all_succeeded": true, + "delete_count": 1, + "deleted_inserted_row": true, + "rows_remaining": 0, + "update_count": 0 + }, "schema_version": 2 } diff --git a/packages/honua-gp/eval/run_eval.py b/packages/honua-gp/eval/run_eval.py index e170307..c5fd619 100644 --- a/packages/honua-gp/eval/run_eval.py +++ b/packages/honua-gp/eval/run_eval.py @@ -390,9 +390,18 @@ def _grade( ) else: checks["response"] = "pass" - elif live_mode and golden is not None and not expected_failure: - # Live run, supported script, but no response oracle recorded yet. - checks["response"] = "unblessed" + elif live_mode and not expected_failure: + # Live run, supported script, but no response oracle recorded. An + # unblessed pass is not evidence (issue #202): a supported script + # must record what the server actually returned, so it fails. + checks["response"] = "fail" + return ( + "fail", + False, + f"supported script {script.stem!r} has no response oracle -- make it emit one and bless it " + "(HONUA_GP_EVAL_USE_STUB=0 ... run_eval.py --update-golden)", + checks, + ) return "pass", False, None, checks diff --git a/packages/honua-gp/eval/scripts/describe_segments.py b/packages/honua-gp/eval/scripts/describe_segments.py index 301e61d..d09135d 100644 --- a/packages/honua-gp/eval/scripts/describe_segments.py +++ b/packages/honua-gp/eval/scripts/describe_segments.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,6 +26,10 @@ desc = arcpy.Describe("segments") srid = desc.spatialReference.factoryCode if desc.spatialReference else None +emit_response( + "describe_segments", + schema_fingerprint(desc.fields, shape_type=desc.shapeType, oid_field=desc.OIDFieldName, srid=srid), +) print( f"describe_segments ok shapeType={desc.shapeType} " f"fields={len(desc.fields)} oidField={desc.OIDFieldName} srid={srid}" diff --git a/packages/honua-gp/eval/scripts/describe_segments_fields.py b/packages/honua-gp/eval/scripts/describe_segments_fields.py index 6b9bfd6..9a24b64 100644 --- a/packages/honua-gp/eval/scripts/describe_segments_fields.py +++ b/packages/honua-gp/eval/scripts/describe_segments_fields.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,9 @@ desc = arcpy.Describe("segments") names = [field.name for field in desc.fields] +srid = desc.spatialReference.factoryCode if desc.spatialReference else None +emit_response( + "describe_segments_fields", + schema_fingerprint(desc.fields, shape_type=desc.shapeType, oid_field=desc.OIDFieldName, srid=srid), +) print(f"describe_segments_fields ok fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py index e11b487..93e35e3 100644 --- a/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py +++ b/packages/honua-gp/eval/scripts/insert_cursor_append_rows.py @@ -1,4 +1,4 @@ -"""InsertCursor: append three rows.""" +"""InsertCursor: append three rows, then read the persisted rows back.""" import sys from pathlib import Path @@ -23,8 +23,16 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True +from eval._emit import apply_edits_fingerprint, edited_object_ids + with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: cursor.insertRow(["OPEN", "Main St"]) cursor.insertRow(["OPEN", "Elm Ave"]) cursor.insertRow(["CLOSED", "Side Rd"]) + edits = cursor.flush() +added = edited_object_ids(edits, "add") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + persisted_rows = sorted([row[1], row[2]] for row in cursor if str(row[0]) in added) print("insert_cursor_append_rows ok") +from eval._emit import emit_response +emit_response('insert_cursor_append_rows', {**apply_edits_fingerprint(edits), 'persisted_rows': persisted_rows}) diff --git a/packages/honua-gp/eval/scripts/list_fields_segments.py b/packages/honua-gp/eval/scripts/list_fields_segments.py index a73bbc6..c386acf 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments") names = [f.name for f in fields] +emit_response("list_fields_segments", schema_fingerprint(fields)) print(f"list_fields_segments ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py b/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py index 6a87715..7490284 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments_filtered.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments", field_type="String") names = [f.name for f in fields] +emit_response("list_fields_segments_filtered", schema_fingerprint(fields)) print(f"list_fields_segments_filtered ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py b/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py index b657679..3d3cfdc 100644 --- a/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py +++ b/packages/honua-gp/eval/scripts/list_fields_segments_wildcard.py @@ -9,6 +9,7 @@ if candidate not in sys.path: sys.path.insert(0, candidate) +from eval._emit import emit_response, schema_fingerprint from eval._stub import install_stub, stub_active import honua_gp as arcpy @@ -25,4 +26,5 @@ fields = arcpy.management.ListFields("segments", wild_card="STAT*") names = [f.name for f in fields] +emit_response("list_fields_segments_wildcard", schema_fingerprint(fields)) print(f"list_fields_segments_wildcard ok count={len(fields)} fields={','.join(names)}") diff --git a/packages/honua-gp/eval/scripts/make_table_view.py b/packages/honua-gp/eval/scripts/make_table_view.py index 9ffa4f7..2b23e8d 100644 --- a/packages/honua-gp/eval/scripts/make_table_view.py +++ b/packages/honua-gp/eval/scripts/make_table_view.py @@ -1,4 +1,4 @@ -"""Make a table view for inspection.""" +"""Make a filtered table view, then read the server rows through it.""" import sys from pathlib import Path @@ -23,5 +23,12 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -arcpy.management.MakeTableView("segments_attrs", "segments_view") -print("make_table_view ok") +arcpy.management.MakeTableView("segments_attrs", "segments_view", "status = 'active'") +# MakeTableView itself only registers a session alias; the observable result is +# what the server returns when the view (and its where clause) is read. +view_count = int(arcpy.management.GetCount("segments_view")) +with arcpy.da.SearchCursor("segments_view", ["name", "status"]) as cursor: + view_rows = sorted([row[0], row[1]] for row in cursor) +print(f"make_table_view ok count={view_count}") +from eval._emit import emit_response +emit_response('make_table_view', {'view_count': view_count, 'view_rows': view_rows}) diff --git a/packages/honua-gp/eval/scripts/update_cursor_close_status.py b/packages/honua-gp/eval/scripts/update_cursor_close_status.py index 1a18207..201f16b 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_close_status.py +++ b/packages/honua-gp/eval/scripts/update_cursor_close_status.py @@ -1,4 +1,4 @@ -"""UpdateCursor: flip CLOSED rows to ARCHIVED.""" +"""UpdateCursor: flip CLOSED rows to ARCHIVED, then read the rows back.""" import sys from pathlib import Path @@ -23,9 +23,23 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: +from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so the oracle does not +# depend on which other scripts ran first or on a previous run's leftovers. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Close Status Rd"]) +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Close Status Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": row[1] = "ARCHIVED" cursor.updateRow(row) + edits = cursor.flush() +updated = edited_object_ids(edits, "update") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS", "name"], "name = 'Close Status Rd'") as cursor: + rows = list(cursor) +updated_rows = sorted([row[1], row[2]] for row in rows if str(row[0]) in updated) +closed_remaining = sum(1 for row in rows if row[1] == "CLOSED") print("update_cursor_close_status ok") +from eval._emit import emit_response +emit_response('update_cursor_close_status', {**apply_edits_fingerprint(edits), 'updated_rows': updated_rows, 'closed_remaining': closed_remaining}) diff --git a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py index 8fa0f13..a0dcb25 100644 --- a/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py +++ b/packages/honua-gp/eval/scripts/update_cursor_delete_closed.py @@ -1,4 +1,4 @@ -"""UpdateCursor: delete CLOSED rows.""" +"""UpdateCursor: delete CLOSED rows, then confirm they are gone.""" import sys from pathlib import Path @@ -23,8 +23,21 @@ arcpy.env.workspace = "honua://services/transport" arcpy.env.overwriteOutput = True -with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: +from eval._emit import apply_edits_fingerprint, edited_object_ids + +# The script owns its fixture row (scoped by name), so there is always a CLOSED +# row to delete -- a zero-delete run is a failure, not a vacuous pass. +with arcpy.da.InsertCursor("roads", ["STATUS", "name"]) as cursor: + cursor.insertRow(["CLOSED", "Delete Closed Rd"]) + inserted = edited_object_ids(cursor.flush(), "add") +with arcpy.da.UpdateCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: for row in cursor: if row[1] == "CLOSED": cursor.deleteRow() + edits = cursor.flush() +deleted = edited_object_ids(edits, "delete") +with arcpy.da.SearchCursor("roads", ["OID@", "STATUS"], "name = 'Delete Closed Rd'") as cursor: + rows = list(cursor) print("update_cursor_delete_closed ok") +from eval._emit import emit_response +emit_response('update_cursor_delete_closed', {**apply_edits_fingerprint(edits), 'deleted_inserted_row': bool(inserted) and deleted == inserted, 'rows_remaining': len(rows)}) diff --git a/packages/honua-gp/honua_gp/da/__init__.py b/packages/honua-gp/honua_gp/da/__init__.py index c76cf9c..6d165d8 100644 --- a/packages/honua-gp/honua_gp/da/__init__.py +++ b/packages/honua-gp/honua_gp/da/__init__.py @@ -102,13 +102,45 @@ def _oid_from_attrs(attrs: Mapping[str, Any]) -> Any: return None -def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: +def _attrs_for_feature(feature: Any) -> dict[str, Any]: + """Extract the attribute/property mapping from any feature shape a Source yields. + + The eval stub's ``_StubFeature`` exposes ``.attributes``; the real + ``honua_sdk.models.QueryFeature`` (what ``Source.iter_features`` actually + yields against a live server) exposes GeoJSON-shaped ``.properties`` + instead -- there is no ``.attributes``. Without the ``.properties`` + fallback every live cursor read silently degraded to an empty mapping + (every field value ``None``), which stub-mode CI could never catch since + the stub always has ``.attributes``. + """ + if hasattr(feature, "attributes"): - attrs = dict(feature.attributes or {}) - elif isinstance(feature, dict): - attrs = dict(feature.get("attributes") or feature.get("properties") or {}) - else: - attrs = {} + return dict(feature.attributes or {}) + if hasattr(feature, "properties"): + return dict(feature.properties or {}) + if isinstance(feature, dict): + return dict(feature.get("attributes") or feature.get("properties") or {}) + return {} + + +def _oid_from_feature(feature: Any, attrs: Mapping[str, Any]) -> Any: + """Resolve a feature's OID, preferring the SDK's protocol-neutral ``.id``. + + ``QueryFeature.id`` is the SDK's already-resolved stable identifier -- + it does not depend on the server's object-id field name matching one of + ``_OID_KEYS`` (the client-compat seed's object-id field is the lower-case + ``objectid``, which ``_OID_KEYS`` never matched). Fall back to scanning + ``attrs`` for the stub's legacy shape, which has no ``.id``. + """ + + feature_id = getattr(feature, "id", None) + if feature_id is not None: + return feature_id + return _oid_from_attrs(attrs) + + +def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: + attrs = _attrs_for_feature(feature) geometry = getattr(feature, "geometry", None) if geometry is None and isinstance(feature, dict): @@ -119,7 +151,7 @@ def _values_for_row(feature: Any, fields: Sequence[str]) -> tuple[Any, ...]: if field.upper() in {"SHAPE@", "SHAPE@JSON"}: out.append(_shape_value(geometry, field.upper())) elif field.upper() == "OID@": - out.append(_oid_from_attrs(attrs)) + out.append(_oid_from_feature(feature, attrs)) else: out.append(attrs.get(field)) return tuple(out) @@ -554,8 +586,7 @@ def _close(self, exc_type, exc, tb) -> None: }) def _extract_oid(self, feature: Any) -> Any: - attrs = getattr(feature, "attributes", None) or (feature.get("attributes") if isinstance(feature, dict) else None) or {} - return _oid_from_attrs(attrs) + return _oid_from_feature(feature, _attrs_for_feature(feature)) class InsertCursor(_BaseCursor): diff --git a/packages/honua-gp/tests/test_da_cursors.py b/packages/honua-gp/tests/test_da_cursors.py index 9d52a9f..3947c17 100644 --- a/packages/honua-gp/tests/test_da_cursors.py +++ b/packages/honua-gp/tests/test_da_cursors.py @@ -431,6 +431,80 @@ def source(self, descriptor: Any) -> Any: _ = second # silence unused-var warnings +def test_search_cursor_reads_query_feature_properties() -> None: + """SearchCursor must read values from a real ``QueryFeature`` (``.properties`` + / ``.id``), not just the eval stub's ``.attributes`` shape. + + ``Source.iter_features`` against a live honua-server yields + ``honua_sdk.models.QueryFeature`` -- GeoJSON-shaped ``.properties`` plus a + protocol-neutral ``.id``, with no ``.attributes`` at all. Before the fix, + ``_values_for_row`` only recognized ``.attributes``, so every field + (including ``OID@``) silently came back ``None`` against a real server; + the eval stub's ``_StubFeature`` (which does have ``.attributes``) masked + this in stub-mode CI. + """ + + from honua_sdk import QueryFeature + + class _QueryFeatureSource: + def iter_features(self, **_: Any) -> Any: + return iter([ + QueryFeature(id=1, properties={"STATUS": "CLOSED", "name": "Side Rd"}), + QueryFeature(id=2, properties={"STATUS": "OPEN", "name": "Main St"}), + ]) + + class _QueryFeatureClient: + def source(self, descriptor: Any) -> Any: + return _QueryFeatureSource() + + honua_gp.configure(client=_QueryFeatureClient()) + + with honua_gp.da.SearchCursor("roads", ["OID@", "STATUS", "name"]) as cursor: + rows = list(cursor) + + assert rows == [(1, "CLOSED", "Side Rd"), (2, "OPEN", "Main St")] + + +def test_update_cursor_extracts_oid_from_query_feature_id() -> None: + """UpdateCursor.updateRow/deleteRow must resolve the OID from ``QueryFeature.id``, + not just an ``OBJECTID``-keyed ``attributes`` mapping. + + Before the fix, a real ``QueryFeature`` (no ``.attributes``, and whose + object-id field may not even be named ``OBJECTID`` server-side -- the + client-compat seed's is the lower-case ``objectid``) always resolved to + ``_extract_oid() is None``, so ``updateRow``/``deleteRow`` raised + ``HonuaGpConfigurationError`` for every real feature. + """ + + from honua_sdk import QueryFeature + + edits: dict[str, Any] = {} + + class _QueryFeatureSource: + def iter_features(self, **_: Any) -> Any: + return iter([QueryFeature(id=16, properties={"STATUS": "CLOSED"})]) + + def apply_edits(self, **kwargs: Any) -> Any: + edits.update(kwargs) + return {"ok": True} + + class _QueryFeatureClient: + def source(self, descriptor: Any) -> Any: + return _QueryFeatureSource() + + honua_gp.configure(client=_QueryFeatureClient()) + + with honua_gp.da.UpdateCursor("roads", ["OID@", "STATUS"]) as cursor: + row = next(cursor) + assert row[0] == 16 + row[1] = "ARCHIVED" + cursor.updateRow(row) + cursor.deleteRow() + + assert edits["updates"][0]["attributes"]["OBJECTID"] == 16 + assert edits["deletes"] == [16] + + def test_cursor_open_failure_reports_real_error_kind(tmp_path) -> None: # Unconfigured session: _open() raises HonuaGpConfigurationError before # the caller's `with` block starts. The audit should record that real diff --git a/packages/honua-gp/tests/test_eval_harness.py b/packages/honua-gp/tests/test_eval_harness.py index ad6d168..71b6f84 100644 --- a/packages/honua-gp/tests/test_eval_harness.py +++ b/packages/honua-gp/tests/test_eval_harness.py @@ -30,6 +30,23 @@ def test_eval_scripts_pair_with_golden_records() -> None: assert golden.exists(), f"Missing golden file for {script.name}" +def test_every_supported_script_has_a_response_oracle() -> None: + """A supported script must record what the server returned (#202). + + Stub CI never grades the response layer, so without this check a new + supported script with no ``response`` block would only surface in the live + smoke lane. The unblessed set is empty and must stay empty. + """ + + unblessed = [] + for golden_path in sorted(_PACKAGE_ROOT.glob("eval/golden/*.json")): + golden = json.loads(golden_path.read_text(encoding="utf-8")) + expected_failure = golden.get("expected_failure", "expected_failure" in golden_path.stem) + if not expected_failure and not golden.get("response"): + unblessed.append(golden_path.stem) + assert unblessed == [], f"supported eval scripts with no response oracle: {unblessed}" + + def test_committed_matrix_matches_generated_output() -> None: committed = (_PACKAGE_ROOT / "docs" / "compatibility-matrix.md").read_text(encoding="utf-8") generated = render_compat_matrix() @@ -205,6 +222,47 @@ def test_grade_response_value_diff_is_live_only(tmp_path: Path) -> None: assert reason is not None and "response value mismatch" in reason +def test_grade_live_supported_script_without_response_oracle_fails(tmp_path: Path) -> None: + """Live mode never passes a supported script as "unblessed" (#202).""" + + script = tmp_path / "make_widget_view.py" + script.write_text("", encoding="utf-8") + golden = { + "schema_version": 2, + "expected_failure": False, + "plumbing": {"audit_lines": 1, "stdout_contains": "make_widget_view ok"}, + } + common = dict(exit_code=0, audit_lines=1, golden=golden, stdout="make_widget_view ok\n", stderr="") + + status, _, reason, checks = _grade(script, response_actual={"view_count": 5}, live_mode=True, **common) + assert status == "fail" + assert checks["response"] == "fail" + assert reason is not None and "has no response oracle" in reason + + # Stub mode grades no response layer, so the same golden still passes there. + status, _, _, checks = _grade(script, live_mode=False, **common) + assert status == "pass" + assert "response" not in checks + + +def test_edited_object_ids_reads_successful_server_ids_only() -> None: + from _emit import edited_object_ids + from honua_sdk.models import ApplyEditsResult + + result = ApplyEditsResult.from_dict( + { + "addResults": [{"objectId": 11, "success": True}, {"objectId": 12, "success": False}], + "deleteResults": [{"objectId": 7, "success": True}], + } + ) + assert edited_object_ids(result, "add") == {"11"} + assert edited_object_ids(result, "delete") == {"7"} + assert edited_object_ids(result, "update") == set() + # The stub's dict result and an empty flush carry no server ids. + assert edited_object_ids({"adds": [{"attributes": {}}]}, "add") == set() + assert edited_object_ids(None, "add") == set() + + def test_run_script_always_rebuilds_pythonpath_when_host_sets_it( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: