From 912676b6ae1f7a0a18f3193f607e07ccc99b67c8 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 10:47:23 -0400 Subject: [PATCH 01/21] add pathfinder models --- test_harness/reporter.py | 75 ++++++++++++++++++++++---------- test_harness/result_collector.py | 6 +-- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/test_harness/reporter.py b/test_harness/reporter.py index 51b1778..762fd7e 100644 --- a/test_harness/reporter.py +++ b/test_harness/reporter.py @@ -4,9 +4,9 @@ import httpx import logging import os -from typing import List +from typing import List, Union -from translator_testing_model.datamodel.pydanticmodel import TestCase, TestAsset +from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestAsset, PathfinderTestAsset class Reporter: @@ -63,25 +63,32 @@ async def create_test_run(self, test_env, suite_name): self.test_run_id = res_json["id"] return self.test_run_id - async def create_test(self, test: TestCase, asset: TestAsset): + async def create_test(self, test: Union[TestCase, PathfinderTestCase], asset: Union[TestAsset, PathfinderTestAsset]): """Create a test in the IR.""" name = asset.name if asset.name else asset.description - res = await self.authenticated_client.post( - url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests", - json={ - "name": name, - "className": test.name, - "methodName": asset.name, - "startedAt": datetime.now().astimezone().isoformat(), - "labels": [ - { - "key": "TestCase", - "value": test.id, - }, - { - "key": "TestAsset", - "value": asset.id, - }, + test_json = { + "name": name, + "className": test.name, + "methodName": asset.name, + "startedAt": datetime.now().astimezone().isoformat(), + "labels": [ + { + "key": "TestCase", + "value": test.id, + }, + { + "key": "TestAsset", + "value": asset.id, + }, + { + "key": "ExpectedOutput", + "value": asset.expected_output, + }, + ], + } + if isinstance(test, TestCase) and isinstance(asset, TestAsset): + test_json["labels"].extend( + [ { "key": "InputCurie", "value": asset.input_id, @@ -90,12 +97,34 @@ async def create_test(self, test: TestCase, asset: TestAsset): "key": "OutputCurie", "value": asset.output_id, }, + ] + ) + elif isinstance(test, PathfinderTestCase) and isinstance(asset, PathfinderTestAsset): + test_json["labels"].extend( + [ { - "key": "ExpectedOutput", - "value": asset.expected_output, + "key": "SourceInputCurie", + "value": asset.source_input_id, }, - ], - }, + { + "key": "TargetInputCurie", + "value": asset.target_input_id, + }, + { + "key": "PathOutputCuries", + "value": [ + path_node_id + for path_node in asset.path_nodes + for path_node_id in path_node.ids + ] + } + ] + ) + else: + raise Exception + res = await self.authenticated_client.post( + url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests", + json=test_json ) res.raise_for_status() res_json = res.json() diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index d075e09..cc639d1 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -2,7 +2,7 @@ import logging from typing import Union -from translator_testing_model.datamodel.pydanticmodel import TestAsset, TestCase +from translator_testing_model.datamodel.pydanticmodel import TestAsset, PathfinderTestAsset, TestCase, PathfinderTestCase from test_harness.utils import get_tag @@ -43,8 +43,8 @@ def __init__(self, logger: logging.Logger): def collect_result( self, - test: TestCase, - asset: TestAsset, + test: Union[TestCase, PathfinderTestCase], + asset: Union[TestAsset, PathfinderTestAsset], report: dict, parent_pk: Union[str, None], url: str, From d7ae86cf13a550675b23aeb3843f5af9af8088dc Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:35:37 -0400 Subject: [PATCH 02/21] update requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7dbe719..074ac93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ pydantic==2.7.1 setproctitle==1.3.3 slack_sdk==3.27.2 tqdm==4.66.4 -translator-testing-model==0.3.2 +translator-testing-model==0.4.0 +reasoner-validator==4.2.5 From 2b48560ef2ab35ac6f14e6207b91e008ff7b1ecc Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:49:12 -0400 Subject: [PATCH 03/21] add pathfinder testing examples and download --- test_harness/download.py | 4 +- tests/helpers/example_tests.py | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/test_harness/download.py b/test_harness/download.py index e84295c..91d332e 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -10,14 +10,14 @@ from typing import List, Union, Dict import zipfile -from translator_testing_model.datamodel.pydanticmodel import TestCase, TestSuite +from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestSuite def download_tests( suite: Union[str, List[str]], url: Path, logger: logging.Logger, -) -> Dict[str, TestCase]: +) -> Dict[str, Union[TestCase, PathfinderTestCase]]: """Download tests from specified location.""" assert Path(url).suffix == ".zip" logger.info(f"Downloading tests from {url}...") diff --git a/tests/helpers/example_tests.py b/tests/helpers/example_tests.py index 672fcd1..4b41d0d 100644 --- a/tests/helpers/example_tests.py +++ b/tests/helpers/example_tests.py @@ -103,6 +103,80 @@ "test_case_predicate_id": "biolink:treats", "test_case_input_id": "MONDO:0010794", "test_runner_settings": ["inferred"], + }, + "TestCase_2": { + "id": "TestCase_0", + "name": "imatinib to asthma", + "description": "imatinib to asthma", + "tags": [], + "test_runner_settings": [ + "pathfinder" + ], + "query_type": None, + "test_assets": [ + { + "id": "PTFQ_1", + "name": "Imatinib to Asthma", + "description": "Imatinib to Asthma", + "tags": [], + "test_runner_settings": [ + "pathfinder" + ], + "source_input_id": "CHEBI:31690", + "source_input_name": "Imatinib", + "source_input_category": "biolink:Drug", + "target_input_id": "MONDO:0004979", + "target_input_name": "Asthma", + "target_input_category": "biolink:Disease", + "predicate_id": "biolink:related_to", + "predicate_name": "related to", + "path_nodes": [ + { + "ids": ["NCBIGene:3815"], + "name": "KIT" + }, + { + "ids": ["CHEBI:18295", "PR:000049994"], + "name": "Histamine" + }, + { + "ids": ["NCBIGene:4254"], + "name": "SCF-1" + }, + { + "ids": ["CL:0000097"], + "name": "Mast Cell" + } + ], + "association": None, + "qualifiers": None, + "expected_output": "TopAnswer", + "test_issue": None, + "semantic_severity": None, + "in_v1": None, + "well_known": False, + "test_reference": None, + "test_metadata": { + "id": "1", + "name": None, + "description": None, + "tags": [], + "test_runner_settings": [], + "test_source": "SMURF", + "test_reference": None, + "test_objective": "AcceptanceTest", + "test_annotations": [] + } + } + ], + "preconditions": [], + "trapi_template": None, + "test_case_objective": "AcceptanceTest", + "test_case_source": None, + "components": [ + "ars" + ], + "test_env": "ci" } }, } From 1827b5233c66865194d337c79dffbbab93a80082 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:49:36 -0400 Subject: [PATCH 04/21] pathfinder query generation --- test_harness/runner/generate_query.py | 45 +++++++++++++++++++++++++-- test_harness/utils.py | 40 ++++++++++++++++++------ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/test_harness/runner/generate_query.py b/test_harness/runner/generate_query.py index 2a585c4..8a24700 100644 --- a/test_harness/runner/generate_query.py +++ b/test_harness/runner/generate_query.py @@ -1,7 +1,8 @@ """Given a Test Asset, generate a TRAPI query.""" import copy -from translator_testing_model.datamodel.pydanticmodel import TestAsset +from typing import Union +from translator_testing_model.datamodel.pydanticmodel import TestAsset, PathfinderTestAsset from test_harness.utils import get_qualifier_constraints @@ -56,11 +57,48 @@ } } +PATHFINDER = { + "message": { + "query_graph": { + "nodes": { + "SN": { + "set_interpretation": "BATCH", + "constraints": [], + "member_ids": [] + }, + "ON": { + "set_interpretation": "BATCH", + "constraints": [], + "member_ids": [] + } + }, + "paths": { + "p0": { + "subject": "SN", + "object": "ON" + } + } + } + } +} + -def generate_query(test_asset: TestAsset) -> dict: +def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: """Generate a TRAPI query.""" query = {} - if test_asset.predicate_id == "biolink:treats": + if isinstance(test_asset, PathfinderTestAsset): + source_id = test_asset.source_input_id + target_id = test_asset.target_input_id + query = copy.deepcopy(PATHFINDER) + query["message"]["query_graph"]["nodes"]["SN"] = { + "ids": [source_id], + "categories": [test_asset.source_input_category] + } + query["message"]["query_graph"]["nodes"]["ON"] = { + "ids": [target_id], + "categories": [test_asset.target_input_category] + } + elif test_asset.predicate_id == "biolink:treats": # MVP1 query = copy.deepcopy(MVP1) # add id to node @@ -108,3 +146,4 @@ def generate_query(test_asset: TestAsset) -> dict: raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") return query + diff --git a/test_harness/utils.py b/test_harness/utils.py index a50b2c1..0f2dc25 100644 --- a/test_harness/utils.py +++ b/test_harness/utils.py @@ -4,7 +4,7 @@ import logging from typing import Dict, Union, List, Tuple -from translator_testing_model.datamodel.pydanticmodel import TestCase, TestAsset +from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestAsset, PathfinderTestAsset NODE_NORM_URL = { "dev": "https://nodenormalization-sri.renci.org/1.4", @@ -15,15 +15,27 @@ async def normalize_curies( - test: TestCase, + test: Union[TestCase, PathfinderTestCase], logger: logging.Logger = logging.getLogger(__name__), ) -> Dict[str, Dict[str, Union[Dict[str, str], List[str]]]]: """Normalize a list of curies.""" node_norm = NODE_NORM_URL.get(test.test_env) # collect all curies from test - curies = set([asset.output_id for asset in test.test_assets]) - curies.update([asset.input_id for asset in test.test_assets]) - curies.add(test.test_case_input_id) + if isinstance(test, PathfinderTestCase): + curies = set([asset.source_input_id for asset in test.test_assets]) + curies.update([asset.target_input_id for asset in test.test_assets]) + curies.update( + [ + path_node_id + for asset in test.test_assets + for path_node in asset.path_nodes + for path_node_id in path_node.ids + ] + ) + else: + curies = set([asset.output_id for asset in test.test_assets]) + curies.update([asset.input_id for asset in test.test_assets]) + curies.add(test.test_case_input_id) normalized_curies = {} async with httpx.AsyncClient() as client: @@ -63,15 +75,25 @@ def get_tag(result): return tag -def hash_test_asset(test_asset: TestAsset) -> int: +def hash_test_asset(test_asset: Union[TestAsset, PathfinderTestAsset]) -> int: """Given a test asset, return its unique hash.""" - asset_hash = hash( + if isinstance(test_asset, PathfinderTestAsset): + asset_hash = hash( ( - test_asset.input_id, + test_asset.source_input_id, + test_asset.target_input_id, test_asset.predicate_id, - *[qualifier.value for qualifier in test_asset.qualifiers], + *[qualifier.value for qualifier in (test_asset.qualifiers or [])], ) ) + else: + asset_hash = hash( + ( + test_asset.input_id, + test_asset.predicate_id, + *[qualifier.value for qualifier in test_asset.qualifiers], + ) + ) return asset_hash From d57a20551b38551c17fe0dd188791f6408887487 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:49:57 -0400 Subject: [PATCH 05/21] test runer --- test_harness/pathfinder_test_runner.py | 26 ++++ test_harness/run.py | 39 +++-- test_harness/runner/query_runner.py | 193 +++++++++++++++++-------- 3 files changed, 182 insertions(+), 76 deletions(-) create mode 100644 test_harness/pathfinder_test_runner.py diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py new file mode 100644 index 0000000..e2655c2 --- /dev/null +++ b/test_harness/pathfinder_test_runner.py @@ -0,0 +1,26 @@ +from typing import Dict, Union, List + +async def pass_fail_analysis( + report: Dict[str, any], + agent: str, + message: Dict[str, any], + path_curies: List[str] +) -> Dict[str, any]: + found_path_nodes = set() + for analysis in message["results"][0]["analyses"]: + for path_bindings in analysis["path_bindings"].values(): + for path_binding in path_bindings: + path_id = path_binding["id"] + for edge_id in message["auxiliary_graphs"][path_id]["edges"]: + edge = message["knowledge_graph"]["edges"][edge_id] + if edge["subject"] in path_curies: + found_path_nodes.add(edge["subject"]) + elif edge["object"] in path_curies: + found_path_nodes.add(edge["object"]) + if len(found_path_nodes) > 0: + report[agent]["status"] = "PASSED" + report[agent]["expected_path_nodes"] = found_path_nodes + else: + report[agent]["status"] = "FAILED" + + return report \ No newline at end of file diff --git a/test_harness/run.py b/test_harness/run.py index fb85d8d..d0480df 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -5,26 +5,27 @@ import time from tqdm import tqdm import traceback -from typing import Dict +from typing import Dict, Union from ARS_Test_Runner.semantic_test import pass_fail_analysis from standards_validation_test_runner import StandardsValidationTest # from benchmarks_runner import run_benchmarks -from translator_testing_model.datamodel.pydanticmodel import TestCase +from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase from test_harness.runner.query_runner import QueryRunner from test_harness.reporter import Reporter from test_harness.slacker import Slacker from test_harness.result_collector import ResultCollector from test_harness.utils import get_tag, hash_test_asset - +from test_harness.pathfinder_test_runner import pass_fail_analysis as pathfinder_pass_fail_analysis + async def run_tests( reporter: Reporter, slacker: Slacker, - tests: Dict[str, TestCase], + tests: Dict[str, Union[TestCase, PathfinderTestCase]], logger: logging.Logger = logging.getLogger(__name__), args: Dict[str, any] = {}, ) -> Dict: @@ -71,7 +72,7 @@ async def run_tests( try: test_id = await reporter.create_test(test, asset) test_ids.append(test_id) - except Exception: + except Exception as e: logger.error(f"Failed to create test: {test.id}") continue @@ -147,13 +148,25 @@ async def run_tests( agent_report["status"] = "DONE" agent_report["message"] = "No results" continue - await pass_fail_analysis( - report["result"], - agent, - response["response"]["message"]["results"], - normalized_curies[asset.output_id], - asset.expected_output, - ) + if isinstance(test, PathfinderTestCase): + await pathfinder_pass_fail_analysis( + report["result"], + agent, + response["response"]["message"], + [ + normalized_curies[path_node_id] + for path_node in asset.path_nodes + for path_node_id in path_node.ids + ] + ) + else: + await pass_fail_analysis( + report["result"], + agent, + response["response"]["message"]["results"], + normalized_curies[asset.output_id], + asset.expected_output, + ) except Exception as e: logger.error( f"Failed to run acceptance test analysis on {agent}: {e}" @@ -271,4 +284,4 @@ async def run_tests( ) await slacker.upload_test_results_file(reporter.test_name, "json", collector.stats) await slacker.upload_test_results_file(reporter.test_name, "csv", collector.csv) - return full_report + return full_report \ No newline at end of file diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 07c2882..cb721c0 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -4,9 +4,12 @@ import httpx import logging import time -from typing import Tuple, Dict +from typing import Tuple, Dict, Union -from translator_testing_model.datamodel.pydanticmodel import TestCase +from translator_testing_model.datamodel.pydanticmodel import ( + TestCase, + PathfinderTestCase, +) from test_harness.runner.smart_api_registry import retrieve_registry_from_smartapi from test_harness.runner.generate_query import generate_query from test_harness.utils import hash_test_asset, normalize_curies @@ -22,9 +25,7 @@ } -class QueryRunner: - """Translator Test Query Runner.""" - +class BaseQueryRunner: def __init__(self, logger: logging.Logger): self.registry = {} self.logger = logger @@ -79,64 +80,6 @@ async def run_query( return query_hash, responses, pks - async def run_queries( - self, - test_case: TestCase, - concurrency: int = 1, # for performance testing - ) -> Tuple[Dict[int, dict], Dict[str, str]]: - """Run all queries specified in a Test Case.""" - # normalize all the curies in a test case - normalized_curies = await normalize_curies(test_case, self.logger) - # TODO: figure out the right way to handle input category wrt normalization - - queries: Dict[int, dict] = {} - for test_asset in test_case.test_assets: - test_asset.input_id = normalized_curies[test_asset.input_id] - # TODO: make this better - asset_hash = hash_test_asset(test_asset) - if asset_hash not in queries: - # generate query - try: - query = generate_query(test_asset) - queries[asset_hash] = { - "query": query, - "responses": {}, - "pks": {}, - } - except Exception as e: - self.logger.warning(e) - - # send queries to a single type of component at a time - for component in test_case.components: - # component = "ara" - # loop over all specified components, i.e. ars, ara, kp, utilities - semaphore = asyncio.Semaphore(concurrency) - self.logger.info( - f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" - ) - tasks = [ - asyncio.create_task( - self.run_query( - query_hash, - semaphore, - query["query"], - service["url"], - service["infores"], - ) - ) - for service in self.registry[env_map[test_case.test_env]][component] - for query_hash, query in queries.items() - ] - try: - all_responses = await asyncio.gather(*tasks, return_exceptions=True) - for query_hash, responses, pks in all_responses: - queries[query_hash]["responses"].update(responses) - queries[query_hash]["pks"].update(pks) - except Exception as e: - self.logger.error(f"Something went wrong with the queries: {e}") - - return queries, normalized_curies - async def get_ars_child_response( self, child_pk: str, @@ -306,3 +249,127 @@ async def get_ars_responses( } return responses, pks + + +class QueryRunner(BaseQueryRunner): + """Translator Test Query Runner.""" + + async def run_queries( + self, + test_case: Union[TestCase, PathfinderTestCase], + concurrency: int = 1, # for performance testing + ) -> Tuple[Dict[int, dict], Dict[str, str]]: + """Run all queries specified in a Test Case.""" + # normalize all the curies in a test case + normalized_curies = await normalize_curies(test_case, self.logger) + # TODO: figure out the right way to handle input category wrt normalization + + queries: Dict[int, dict] = {} + for test_asset in test_case.test_assets: + if isinstance(test_case, PathfinderTestCase): + test_asset.source_input_id = normalized_curies[test_asset.source_input_id] + test_asset.target_input_id = normalized_curies[test_asset.target_input_id] + else: + test_asset.input_id = normalized_curies[test_asset.input_id] + # TODO: make this better + asset_hash = hash_test_asset(test_asset) + if asset_hash not in queries: + # generate query + try: + query = generate_query(test_asset) + queries[asset_hash] = { + "query": query, + "responses": {}, + "pks": {}, + } + except Exception as e: + self.logger.warning(e) + + # send queries to a single type of component at a time + for component in test_case.components: + # component = "ara" + # loop over all specified components, i.e. ars, ara, kp, utilities + semaphore = asyncio.Semaphore(concurrency) + self.logger.info( + f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + ) + tasks = [ + asyncio.create_task( + self.run_query( + query_hash, + semaphore, + query["query"], + service["url"], + service["infores"], + ) + ) + for service in self.registry[env_map[test_case.test_env]][component] + for query_hash, query in queries.items() + ] + try: + all_responses = await asyncio.gather(*tasks, return_exceptions=True) + for query_hash, responses, pks in all_responses: + queries[query_hash]["responses"].update(responses) + queries[query_hash]["pks"].update(pks) + except Exception as e: + self.logger.error(f"Something went wrong with the queries: {e}") + + return queries, normalized_curies + + +class PathfinderQueryRunner(BaseQueryRunner): + async def run_queries( + self, + test_case: PathfinderTestCase, + concurrency: int = 1 + ) -> Tuple[Dict[int, dict], Dict[str, str]]: + # normalize all the curies in a test case + normalized_curies = await normalize_curies(test_case, self.logger) + # TODO: figure out the right way to handle input category wrt normalization + + queries: Dict[int, dict] = {} + for test_asset in test_case.test_assets: + # TODO: make this better + asset_hash = hash_test_asset(test_asset) + if asset_hash not in queries: + # generate query + try: + query = generate_query(test_asset) + queries[asset_hash] = { + "query": query, + "responses": {}, + "pks": {}, + } + except Exception as e: + self.logger.warning(e) + + # send queries to a single type of component at a time + for component in test_case.components: + # component = "ara" + # loop over all specified components, i.e. ars, ara, kp, utilities + semaphore = asyncio.Semaphore(concurrency) + self.logger.info( + f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" + ) + tasks = [ + asyncio.create_task( + self.run_query( + query_hash, + semaphore, + query["query"], + service["url"], + service["infores"], + ) + ) + for service in self.registry[env_map[test_case.test_env]][component] + for query_hash, query in queries.items() + ] + try: + all_responses = await asyncio.gather(*tasks, return_exceptions=True) + for query_hash, responses, pks in all_responses: + queries[query_hash]["responses"].update(responses) + queries[query_hash]["pks"].update(pks) + except Exception as e: + self.logger.error(f"Something went wrong with the queries: {e}") + + return queries, normalized_curies From 89fad4255f7dee6b7ce446debaeff37607d918ae Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:52:30 -0400 Subject: [PATCH 06/21] fid test case id --- tests/helpers/example_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/example_tests.py b/tests/helpers/example_tests.py index 4b41d0d..73307c6 100644 --- a/tests/helpers/example_tests.py +++ b/tests/helpers/example_tests.py @@ -105,7 +105,7 @@ "test_runner_settings": ["inferred"], }, "TestCase_2": { - "id": "TestCase_0", + "id": "TestCase_2", "name": "imatinib to asthma", "description": "imatinib to asthma", "tags": [], From fef4572a21e431bd2d242df1adf84ec74af0f4eb Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 11:56:50 -0400 Subject: [PATCH 07/21] black --- test_harness/download.py | 6 +++++- test_harness/pathfinder_test_runner.py | 10 ++++------ test_harness/reporter.py | 25 ++++++++++++++++++------- test_harness/result_collector.py | 7 ++++++- test_harness/run.py | 21 +++++++++++++-------- test_harness/runner/generate_query.py | 23 ++++++++++------------- test_harness/runner/query_runner.py | 12 +++++++----- test_harness/utils.py | 19 ++++++++++++------- 8 files changed, 75 insertions(+), 48 deletions(-) diff --git a/test_harness/download.py b/test_harness/download.py index 91d332e..58d2dd8 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -10,7 +10,11 @@ from typing import List, Union, Dict import zipfile -from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestSuite +from translator_testing_model.datamodel.pydanticmodel import ( + TestCase, + PathfinderTestCase, + TestSuite, +) def download_tests( diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index e2655c2..319304f 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -1,10 +1,8 @@ from typing import Dict, Union, List + async def pass_fail_analysis( - report: Dict[str, any], - agent: str, - message: Dict[str, any], - path_curies: List[str] + report: Dict[str, any], agent: str, message: Dict[str, any], path_curies: List[str] ) -> Dict[str, any]: found_path_nodes = set() for analysis in message["results"][0]["analyses"]: @@ -22,5 +20,5 @@ async def pass_fail_analysis( report[agent]["expected_path_nodes"] = found_path_nodes else: report[agent]["status"] = "FAILED" - - return report \ No newline at end of file + + return report diff --git a/test_harness/reporter.py b/test_harness/reporter.py index 762fd7e..dd05344 100644 --- a/test_harness/reporter.py +++ b/test_harness/reporter.py @@ -6,7 +6,12 @@ import os from typing import List, Union -from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestAsset, PathfinderTestAsset +from translator_testing_model.datamodel.pydanticmodel import ( + TestCase, + PathfinderTestCase, + TestAsset, + PathfinderTestAsset, +) class Reporter: @@ -63,7 +68,11 @@ async def create_test_run(self, test_env, suite_name): self.test_run_id = res_json["id"] return self.test_run_id - async def create_test(self, test: Union[TestCase, PathfinderTestCase], asset: Union[TestAsset, PathfinderTestAsset]): + async def create_test( + self, + test: Union[TestCase, PathfinderTestCase], + asset: Union[TestAsset, PathfinderTestAsset], + ): """Create a test in the IR.""" name = asset.name if asset.name else asset.description test_json = { @@ -99,7 +108,9 @@ async def create_test(self, test: Union[TestCase, PathfinderTestCase], asset: Un }, ] ) - elif isinstance(test, PathfinderTestCase) and isinstance(asset, PathfinderTestAsset): + elif isinstance(test, PathfinderTestCase) and isinstance( + asset, PathfinderTestAsset + ): test_json["labels"].extend( [ { @@ -113,18 +124,18 @@ async def create_test(self, test: Union[TestCase, PathfinderTestCase], asset: Un { "key": "PathOutputCuries", "value": [ - path_node_id + path_node_id for path_node in asset.path_nodes for path_node_id in path_node.ids - ] - } + ], + }, ] ) else: raise Exception res = await self.authenticated_client.post( url=f"{self.base_path}/api/reporting/v1/test-runs/{self.test_run_id}/tests", - json=test_json + json=test_json, ) res.raise_for_status() res_json = res.json() diff --git a/test_harness/result_collector.py b/test_harness/result_collector.py index cc639d1..622ba29 100644 --- a/test_harness/result_collector.py +++ b/test_harness/result_collector.py @@ -2,7 +2,12 @@ import logging from typing import Union -from translator_testing_model.datamodel.pydanticmodel import TestAsset, PathfinderTestAsset, TestCase, PathfinderTestCase +from translator_testing_model.datamodel.pydanticmodel import ( + TestAsset, + PathfinderTestAsset, + TestCase, + PathfinderTestCase, +) from test_harness.utils import get_tag diff --git a/test_harness/run.py b/test_harness/run.py index d0480df..09dcf6c 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -12,15 +12,20 @@ # from benchmarks_runner import run_benchmarks -from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase +from translator_testing_model.datamodel.pydanticmodel import ( + TestCase, + PathfinderTestCase, +) from test_harness.runner.query_runner import QueryRunner from test_harness.reporter import Reporter from test_harness.slacker import Slacker from test_harness.result_collector import ResultCollector from test_harness.utils import get_tag, hash_test_asset -from test_harness.pathfinder_test_runner import pass_fail_analysis as pathfinder_pass_fail_analysis - +from test_harness.pathfinder_test_runner import ( + pass_fail_analysis as pathfinder_pass_fail_analysis, +) + async def run_tests( reporter: Reporter, @@ -103,9 +108,9 @@ async def run_tests( if response["status_code"] == "598": agent_report["message"] = "Timed out" else: - agent_report["message"] = ( - f"Status code: {response['status_code']}" - ) + agent_report[ + "message" + ] = f"Status code: {response['status_code']}" continue elif ( "response" not in response @@ -157,7 +162,7 @@ async def run_tests( normalized_curies[path_node_id] for path_node in asset.path_nodes for path_node_id in path_node.ids - ] + ], ) else: await pass_fail_analysis( @@ -284,4 +289,4 @@ async def run_tests( ) await slacker.upload_test_results_file(reporter.test_name, "json", collector.stats) await slacker.upload_test_results_file(reporter.test_name, "csv", collector.csv) - return full_report \ No newline at end of file + return full_report diff --git a/test_harness/runner/generate_query.py b/test_harness/runner/generate_query.py index 8a24700..8da5d5d 100644 --- a/test_harness/runner/generate_query.py +++ b/test_harness/runner/generate_query.py @@ -2,7 +2,10 @@ import copy from typing import Union -from translator_testing_model.datamodel.pydanticmodel import TestAsset, PathfinderTestAsset +from translator_testing_model.datamodel.pydanticmodel import ( + TestAsset, + PathfinderTestAsset, +) from test_harness.utils import get_qualifier_constraints @@ -64,20 +67,15 @@ "SN": { "set_interpretation": "BATCH", "constraints": [], - "member_ids": [] + "member_ids": [], }, "ON": { "set_interpretation": "BATCH", "constraints": [], - "member_ids": [] - } + "member_ids": [], + }, }, - "paths": { - "p0": { - "subject": "SN", - "object": "ON" - } - } + "paths": {"p0": {"subject": "SN", "object": "ON"}}, } } } @@ -92,11 +90,11 @@ def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: query = copy.deepcopy(PATHFINDER) query["message"]["query_graph"]["nodes"]["SN"] = { "ids": [source_id], - "categories": [test_asset.source_input_category] + "categories": [test_asset.source_input_category], } query["message"]["query_graph"]["nodes"]["ON"] = { "ids": [target_id], - "categories": [test_asset.target_input_category] + "categories": [test_asset.target_input_category], } elif test_asset.predicate_id == "biolink:treats": # MVP1 @@ -146,4 +144,3 @@ def generate_query(test_asset: Union[TestAsset, PathfinderTestAsset]) -> dict: raise Exception(f"Unsupported predicate: {test_asset.predicate_id}") return query - diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index cb721c0..390e157 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -267,8 +267,12 @@ async def run_queries( queries: Dict[int, dict] = {} for test_asset in test_case.test_assets: if isinstance(test_case, PathfinderTestCase): - test_asset.source_input_id = normalized_curies[test_asset.source_input_id] - test_asset.target_input_id = normalized_curies[test_asset.target_input_id] + test_asset.source_input_id = normalized_curies[ + test_asset.source_input_id + ] + test_asset.target_input_id = normalized_curies[ + test_asset.target_input_id + ] else: test_asset.input_id = normalized_curies[test_asset.input_id] # TODO: make this better @@ -319,9 +323,7 @@ async def run_queries( class PathfinderQueryRunner(BaseQueryRunner): async def run_queries( - self, - test_case: PathfinderTestCase, - concurrency: int = 1 + self, test_case: PathfinderTestCase, concurrency: int = 1 ) -> Tuple[Dict[int, dict], Dict[str, str]]: # normalize all the curies in a test case normalized_curies = await normalize_curies(test_case, self.logger) diff --git a/test_harness/utils.py b/test_harness/utils.py index 0f2dc25..e78041a 100644 --- a/test_harness/utils.py +++ b/test_harness/utils.py @@ -4,7 +4,12 @@ import logging from typing import Dict, Union, List, Tuple -from translator_testing_model.datamodel.pydanticmodel import TestCase, PathfinderTestCase, TestAsset, PathfinderTestAsset +from translator_testing_model.datamodel.pydanticmodel import ( + TestCase, + PathfinderTestCase, + TestAsset, + PathfinderTestAsset, +) NODE_NORM_URL = { "dev": "https://nodenormalization-sri.renci.org/1.4", @@ -79,13 +84,13 @@ def hash_test_asset(test_asset: Union[TestAsset, PathfinderTestAsset]) -> int: """Given a test asset, return its unique hash.""" if isinstance(test_asset, PathfinderTestAsset): asset_hash = hash( - ( - test_asset.source_input_id, - test_asset.target_input_id, - test_asset.predicate_id, - *[qualifier.value for qualifier in (test_asset.qualifiers or [])], + ( + test_asset.source_input_id, + test_asset.target_input_id, + test_asset.predicate_id, + *[qualifier.value for qualifier in (test_asset.qualifiers or [])], + ) ) - ) else: asset_hash = hash( ( From 29e5f93e6222353ca5120ef59e5bb5e7a7454b9f Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 13:06:44 -0400 Subject: [PATCH 08/21] remove graph validation test runner --- test_harness/run.py | 52 +++++++++++++++++----------------- tests/helpers/example_tests.py | 37 +++++++----------------- 2 files changed, 37 insertions(+), 52 deletions(-) diff --git a/test_harness/run.py b/test_harness/run.py index 09dcf6c..427c870 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -8,7 +8,7 @@ from typing import Dict, Union from ARS_Test_Runner.semantic_test import pass_fail_analysis -from standards_validation_test_runner import StandardsValidationTest +# from standards_validation_test_runner import StandardsValidationTest # from benchmarks_runner import run_benchmarks @@ -108,9 +108,9 @@ async def run_tests( if response["status_code"] == "598": agent_report["message"] = "Timed out" else: - agent_report[ - "message" - ] = f"Status code: {response['status_code']}" + agent_report["message"] = ( + f"Status code: {response['status_code']}" + ) continue elif ( "response" not in response @@ -123,28 +123,28 @@ async def run_tests( logger.warning( f"Failed to parse basic response fields from {agent}: {e}" ) - try: - svt = StandardsValidationTest( - test_asset=asset, - environment=test.test_env, - component=agent, - trapi_version=args["trapi_version"], - biolink_version="suppress", - runner_settings="Inferred", - ) - results = svt.test_case_processor( - trapi_response=response["response"] - ) - agent_report["trapi_validation"] = results[ - next(iter(results.keys())) - ][agent]["status"] - if agent_report["trapi_validation"] == "FAILED": - agent_report["status"] = "FAILED" - agent_report["message"] = "TRAPI Validation Error" - continue - except Exception as e: - logger.warning(f"Failed to run TRAPI validation with {e}") - agent_report["trapi_validation"] = "ERROR" + # try: + # svt = StandardsValidationTest( + # test_asset=asset, + # environment=test.test_env, + # component=agent, + # trapi_version=args["trapi_version"], + # biolink_version="suppress", + # runner_settings="Inferred", + # ) + # results = svt.test_case_processor( + # trapi_response=response["response"] + # ) + # agent_report["trapi_validation"] = results[ + # next(iter(results.keys())) + # ][agent]["status"] + # if agent_report["trapi_validation"] == "FAILED": + # agent_report["status"] = "FAILED" + # agent_report["message"] = "TRAPI Validation Error" + # continue + # except Exception as e: + # logger.warning(f"Failed to run TRAPI validation with {e}") + # agent_report["trapi_validation"] = "ERROR" try: if ( response["response"]["message"].get("results") is None diff --git a/tests/helpers/example_tests.py b/tests/helpers/example_tests.py index 73307c6..40eb3b7 100644 --- a/tests/helpers/example_tests.py +++ b/tests/helpers/example_tests.py @@ -109,9 +109,7 @@ "name": "imatinib to asthma", "description": "imatinib to asthma", "tags": [], - "test_runner_settings": [ - "pathfinder" - ], + "test_runner_settings": ["pathfinder"], "query_type": None, "test_assets": [ { @@ -119,9 +117,7 @@ "name": "Imatinib to Asthma", "description": "Imatinib to Asthma", "tags": [], - "test_runner_settings": [ - "pathfinder" - ], + "test_runner_settings": ["pathfinder"], "source_input_id": "CHEBI:31690", "source_input_name": "Imatinib", "source_input_category": "biolink:Drug", @@ -131,22 +127,13 @@ "predicate_id": "biolink:related_to", "predicate_name": "related to", "path_nodes": [ - { - "ids": ["NCBIGene:3815"], - "name": "KIT" - }, + {"ids": ["NCBIGene:3815"], "name": "KIT"}, { "ids": ["CHEBI:18295", "PR:000049994"], - "name": "Histamine" + "name": "Histamine", }, - { - "ids": ["NCBIGene:4254"], - "name": "SCF-1" - }, - { - "ids": ["CL:0000097"], - "name": "Mast Cell" - } + {"ids": ["NCBIGene:4254"], "name": "SCF-1"}, + {"ids": ["CL:0000097"], "name": "Mast Cell"}, ], "association": None, "qualifiers": None, @@ -165,19 +152,17 @@ "test_source": "SMURF", "test_reference": None, "test_objective": "AcceptanceTest", - "test_annotations": [] - } + "test_annotations": [], + }, } ], "preconditions": [], "trapi_template": None, "test_case_objective": "AcceptanceTest", "test_case_source": None, - "components": [ - "ars" - ], - "test_env": "ci" - } + "components": ["ars"], + "test_env": "ci", + }, }, } ).test_cases From 9a545060722b33781f3adcd739d763dfc8cf2d5f Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 13:09:46 -0400 Subject: [PATCH 09/21] black and remove import --- requirements-runners.txt | 4 +--- test_harness/run.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements-runners.txt b/requirements-runners.txt index 7923650..d1c31a8 100644 --- a/requirements-runners.txt +++ b/requirements-runners.txt @@ -1,6 +1,4 @@ ARS_Test_Runner==0.2.4 # benchmarks-runner==0.1.3 # ui-test-runner==0.0.2 -graph-validation-test-runners==0.1.5 -# we need to manually pin the right reasoner-validator version -reasoner-validator==4.2.5 +# graph-validation-test-runners==0.1.5 diff --git a/test_harness/run.py b/test_harness/run.py index 427c870..d038ec0 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -8,6 +8,7 @@ from typing import Dict, Union from ARS_Test_Runner.semantic_test import pass_fail_analysis + # from standards_validation_test_runner import StandardsValidationTest # from benchmarks_runner import run_benchmarks From 644edf8495c3097c4eca1dbb6394f14aa7c5a32e Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 15:10:34 -0400 Subject: [PATCH 10/21] fix tests --- tests/test_run.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index d4c4ae1..c94b569 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -35,6 +35,13 @@ async def test_run_tests(mocker, httpx_mock: HTTPXMock): "MONDO:0010794": None, "DRUGBANK:DB00313": None, "MESH:D001463": None, + "CHEBI:18295": None, + "CHEBI:31690": None, + "CL:0000097": None, + "MONDO:0004979": None, + "NCBIGene:3815": None, + "NCBIGene:4254": None, + "PR:000049994": None }, ) full_report = await run_tests( @@ -46,7 +53,7 @@ async def test_run_tests(mocker, httpx_mock: HTTPXMock): logger=logger, args={ "suite": "testing", - "trapi_version": "1.5.0", + "trapi_version": "1.6.0", }, ) - assert full_report["SKIPPED"] == 2 + assert full_report["SKIPPED"] == 3 From 0c90263b9298372b640d2a012e6b8d21383eca16 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Tue, 23 Sep 2025 15:11:28 -0400 Subject: [PATCH 11/21] black --- tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run.py b/tests/test_run.py index c94b569..517eaa9 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -41,7 +41,7 @@ async def test_run_tests(mocker, httpx_mock: HTTPXMock): "MONDO:0004979": None, "NCBIGene:3815": None, "NCBIGene:4254": None, - "PR:000049994": None + "PR:000049994": None, }, ) full_report = await run_tests( From 262ff1c802947ea50f99a42198455b6c20600efa Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 25 Sep 2025 14:08:52 -0400 Subject: [PATCH 12/21] fix test reporting --- test_harness/reporter.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test_harness/reporter.py b/test_harness/reporter.py index dd05344..5a953fb 100644 --- a/test_harness/reporter.py +++ b/test_harness/reporter.py @@ -121,14 +121,6 @@ async def create_test( "key": "TargetInputCurie", "value": asset.target_input_id, }, - { - "key": "PathOutputCuries", - "value": [ - path_node_id - for path_node in asset.path_nodes - for path_node_id in path_node.ids - ], - }, ] ) else: From 0bb363da5a434a00b6dafb4b80b99bcbafa698e8 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 2 Oct 2025 15:13:00 -0400 Subject: [PATCH 13/21] update pathfinder test runner --- test_harness/pathfinder_test_runner.py | 27 ++++++++++++++++++-------- test_harness/run.py | 11 ++++++----- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index 319304f..e484657 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -1,24 +1,35 @@ from typing import Dict, Union, List -async def pass_fail_analysis( - report: Dict[str, any], agent: str, message: Dict[str, any], path_curies: List[str] +async def pathfinder_pass_fail_analysis( + report: Dict[str, any], agent: str, message: Dict[str, any], path_nodes: List[List[str]], minimum_expected_path_nodes: int ) -> Dict[str, any]: found_path_nodes = set() for analysis in message["results"][0]["analyses"]: for path_bindings in analysis["path_bindings"].values(): for path_binding in path_bindings: path_id = path_binding["id"] + matching_path_nodes = set() for edge_id in message["auxiliary_graphs"][path_id]["edges"]: edge = message["knowledge_graph"]["edges"][edge_id] - if edge["subject"] in path_curies: - found_path_nodes.add(edge["subject"]) - elif edge["object"] in path_curies: - found_path_nodes.add(edge["object"]) + for node_curies in path_nodes: + unhit_node = True + for curie in node_curies: + if curie in matching_path_nodes: + unhit_node = False + if unhit_node: + if edge["subject"] in node_curies: + matching_path_nodes.add(edge["subject"]) + if edge["object"] in node_curies: + matching_path_nodes.add(edge["object"]) + if len(matching_path_nodes) >= minimum_expected_path_nodes: + found_path_nodes.add(",".join(matching_path_nodes)) if len(found_path_nodes) > 0: report[agent]["status"] = "PASSED" - report[agent]["expected_path_nodes"] = found_path_nodes + report[agent]["expected_path_nodes"] = "; ".join(found_path_nodes) else: report[agent]["status"] = "FAILED" - + report[agent]["expected_path_nodes"] = "" + + print(f"ran test, {agent}, {path_nodes}, {report[agent]['expected_path_nodes']}, {report[agent]['status']}") return report diff --git a/test_harness/run.py b/test_harness/run.py index d038ec0..3904b86 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -23,9 +23,7 @@ from test_harness.slacker import Slacker from test_harness.result_collector import ResultCollector from test_harness.utils import get_tag, hash_test_asset -from test_harness.pathfinder_test_runner import ( - pass_fail_analysis as pathfinder_pass_fail_analysis, -) +from test_harness.pathfinder_test_runner import pathfinder_pass_fail_analysis async def run_tests( @@ -160,10 +158,13 @@ async def run_tests( agent, response["response"]["message"], [ - normalized_curies[path_node_id] + [ + normalized_curies[path_node_id] + for path_node_id in path_node.ids + ] for path_node in asset.path_nodes - for path_node_id in path_node.ids ], + asset.minimum_expected_path_nodes ) else: await pass_fail_analysis( From 24c7c7f74712694d26288e023e45edeef0d0c2a7 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 2 Oct 2025 15:13:19 -0400 Subject: [PATCH 14/21] bump model version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 074ac93..59f0bac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,5 @@ pydantic==2.7.1 setproctitle==1.3.3 slack_sdk==3.27.2 tqdm==4.66.4 -translator-testing-model==0.4.0 +translator-testing-model==0.4.1 reasoner-validator==4.2.5 From 553b1a617a2ad5adb005c90bbca5a81fe13864be Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 2 Oct 2025 15:54:08 -0400 Subject: [PATCH 15/21] update field name --- test_harness/pathfinder_test_runner.py | 4 ++-- test_harness/run.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index e484657..e7c7215 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -2,7 +2,7 @@ async def pathfinder_pass_fail_analysis( - report: Dict[str, any], agent: str, message: Dict[str, any], path_nodes: List[List[str]], minimum_expected_path_nodes: int + report: Dict[str, any], agent: str, message: Dict[str, any], path_nodes: List[List[str]], minimum_required_path_nodes: int ) -> Dict[str, any]: found_path_nodes = set() for analysis in message["results"][0]["analyses"]: @@ -22,7 +22,7 @@ async def pathfinder_pass_fail_analysis( matching_path_nodes.add(edge["subject"]) if edge["object"] in node_curies: matching_path_nodes.add(edge["object"]) - if len(matching_path_nodes) >= minimum_expected_path_nodes: + if len(matching_path_nodes) >= minimum_required_path_nodes: found_path_nodes.add(",".join(matching_path_nodes)) if len(found_path_nodes) > 0: report[agent]["status"] = "PASSED" diff --git a/test_harness/run.py b/test_harness/run.py index 3904b86..1dcc7a6 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -164,7 +164,7 @@ async def run_tests( ] for path_node in asset.path_nodes ], - asset.minimum_expected_path_nodes + asset.minimum_required_path_nodes ) else: await pass_fail_analysis( From bcb57f58e22099965b99f1f2cc87e57ef8c80a8f Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 2 Oct 2025 15:56:21 -0400 Subject: [PATCH 16/21] update test --- tests/helpers/example_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/example_tests.py b/tests/helpers/example_tests.py index 40eb3b7..40eb295 100644 --- a/tests/helpers/example_tests.py +++ b/tests/helpers/example_tests.py @@ -126,6 +126,7 @@ "target_input_category": "biolink:Disease", "predicate_id": "biolink:related_to", "predicate_name": "related to", + "minimum_required_path_nodes": 2, "path_nodes": [ {"ids": ["NCBIGene:3815"], "name": "KIT"}, { From 331c6a3da4d4914a9d03ec07091f703a19b8d118 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Thu, 2 Oct 2025 15:58:36 -0400 Subject: [PATCH 17/21] black --- test_harness/pathfinder_test_runner.py | 10 ++++++---- test_harness/run.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index e7c7215..35d8be6 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -2,7 +2,11 @@ async def pathfinder_pass_fail_analysis( - report: Dict[str, any], agent: str, message: Dict[str, any], path_nodes: List[List[str]], minimum_required_path_nodes: int + report: Dict[str, any], + agent: str, + message: Dict[str, any], + path_nodes: List[List[str]], + minimum_required_path_nodes: int, ) -> Dict[str, any]: found_path_nodes = set() for analysis in message["results"][0]["analyses"]: @@ -29,7 +33,5 @@ async def pathfinder_pass_fail_analysis( report[agent]["expected_path_nodes"] = "; ".join(found_path_nodes) else: report[agent]["status"] = "FAILED" - report[agent]["expected_path_nodes"] = "" - - print(f"ran test, {agent}, {path_nodes}, {report[agent]['expected_path_nodes']}, {report[agent]['status']}") + return report diff --git a/test_harness/run.py b/test_harness/run.py index 1dcc7a6..6474cb2 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -164,7 +164,7 @@ async def run_tests( ] for path_node in asset.path_nodes ], - asset.minimum_required_path_nodes + asset.minimum_required_path_nodes, ) else: await pass_fail_analysis( From f4812c140a9ec11e99b371130d07524773fc96ec Mon Sep 17 00:00:00 2001 From: uhbrar Date: Fri, 3 Oct 2025 15:16:27 -0400 Subject: [PATCH 18/21] remove base query runner --- test_harness/runner/query_runner.py | 66 ++--------------------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 390e157..1a9a7ed 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -25,7 +25,9 @@ } -class BaseQueryRunner: +class QueryRunner(BaseQueryRunner): + """Translator Test Query Runner.""" + def __init__(self, logger: logging.Logger): self.registry = {} self.logger = logger @@ -250,10 +252,6 @@ async def get_ars_responses( return responses, pks - -class QueryRunner(BaseQueryRunner): - """Translator Test Query Runner.""" - async def run_queries( self, test_case: Union[TestCase, PathfinderTestCase], @@ -318,60 +316,4 @@ async def run_queries( except Exception as e: self.logger.error(f"Something went wrong with the queries: {e}") - return queries, normalized_curies - - -class PathfinderQueryRunner(BaseQueryRunner): - async def run_queries( - self, test_case: PathfinderTestCase, concurrency: int = 1 - ) -> Tuple[Dict[int, dict], Dict[str, str]]: - # normalize all the curies in a test case - normalized_curies = await normalize_curies(test_case, self.logger) - # TODO: figure out the right way to handle input category wrt normalization - - queries: Dict[int, dict] = {} - for test_asset in test_case.test_assets: - # TODO: make this better - asset_hash = hash_test_asset(test_asset) - if asset_hash not in queries: - # generate query - try: - query = generate_query(test_asset) - queries[asset_hash] = { - "query": query, - "responses": {}, - "pks": {}, - } - except Exception as e: - self.logger.warning(e) - - # send queries to a single type of component at a time - for component in test_case.components: - # component = "ara" - # loop over all specified components, i.e. ars, ara, kp, utilities - semaphore = asyncio.Semaphore(concurrency) - self.logger.info( - f"Sending queries to {self.registry[env_map[test_case.test_env]][component]}" - ) - tasks = [ - asyncio.create_task( - self.run_query( - query_hash, - semaphore, - query["query"], - service["url"], - service["infores"], - ) - ) - for service in self.registry[env_map[test_case.test_env]][component] - for query_hash, query in queries.items() - ] - try: - all_responses = await asyncio.gather(*tasks, return_exceptions=True) - for query_hash, responses, pks in all_responses: - queries[query_hash]["responses"].update(responses) - queries[query_hash]["pks"].update(pks) - except Exception as e: - self.logger.error(f"Something went wrong with the queries: {e}") - - return queries, normalized_curies + return queries, normalized_curies \ No newline at end of file From d127101a1fdaa6409db0dfbaab8b2bb8c2170d93 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Fri, 3 Oct 2025 15:25:17 -0400 Subject: [PATCH 19/21] change report --- test_harness/pathfinder_test_runner.py | 9 ++++++++- test_harness/run.py | 21 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/test_harness/pathfinder_test_runner.py b/test_harness/pathfinder_test_runner.py index 35d8be6..0799d31 100644 --- a/test_harness/pathfinder_test_runner.py +++ b/test_harness/pathfinder_test_runner.py @@ -9,6 +9,7 @@ async def pathfinder_pass_fail_analysis( minimum_required_path_nodes: int, ) -> Dict[str, any]: found_path_nodes = set() + unmatched_paths = set() for analysis in message["results"][0]["analyses"]: for path_bindings in analysis["path_bindings"].values(): for path_binding in path_bindings: @@ -28,9 +29,15 @@ async def pathfinder_pass_fail_analysis( matching_path_nodes.add(edge["object"]) if len(matching_path_nodes) >= minimum_required_path_nodes: found_path_nodes.add(",".join(matching_path_nodes)) + elif len(matching_path_nodes) > 0: + unmatched_paths.add(",".join(matching_path_nodes)) + if len(found_path_nodes) > 0: report[agent]["status"] = "PASSED" - report[agent]["expected_path_nodes"] = "; ".join(found_path_nodes) + report[agent]["expected_nodes_found"] = "; ".join(found_path_nodes) + elif len(unmatched_paths) > 0: + report[agent]["status"] = "FAILED" + report[agent]["expected_nodes_found"] = "; ".join(unmatched_paths) else: report[agent]["status"] = "FAILED" diff --git a/test_harness/run.py b/test_harness/run.py index 6474cb2..82cf256 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -96,10 +96,25 @@ async def run_tests( "pks": test_query["pks"], "result": {}, } - for agent, response in test_query["responses"].items(): - report["result"][agent] = { - "trapi_validation": "NA", + if isinstance(test, PathfinderTestCase): + report["test_details"] = { + "minimum_required_path_nodes": asset.minimum_required_path_nodes, + "expected_path_nodes": "; ".join( + [ + ",".join( + [ + normalized_curies[path_node_id] + for path_node_id in path_node.ids + ] + ) + for path_node in asset.path_nodes + ] + ) } + for agent, response in test_query["responses"].items(): + # report["result"][agent] = { + # "trapi_validation": "NA", + # } agent_report = report["result"][agent] try: if response["status_code"] > 299: From 052277a26815b996be49b75f39eae39009468972 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Fri, 3 Oct 2025 15:27:21 -0400 Subject: [PATCH 20/21] fix queryrunner --- test_harness/runner/query_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 1a9a7ed..9db6fca 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -25,7 +25,7 @@ } -class QueryRunner(BaseQueryRunner): +class QueryRunner: """Translator Test Query Runner.""" def __init__(self, logger: logging.Logger): From 9a8f2b85f57d289c93ecc93e5f8e9fd0fbb59867 Mon Sep 17 00:00:00 2001 From: uhbrar Date: Fri, 3 Oct 2025 15:31:01 -0400 Subject: [PATCH 21/21] black --- test_harness/run.py | 8 ++++---- test_harness/runner/query_runner.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test_harness/run.py b/test_harness/run.py index 82cf256..0384931 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -109,12 +109,12 @@ async def run_tests( ) for path_node in asset.path_nodes ] - ) + ), } for agent, response in test_query["responses"].items(): - # report["result"][agent] = { - # "trapi_validation": "NA", - # } + report["result"][agent] = { + # "trapi_validation": "NA", + } agent_report = report["result"][agent] try: if response["status_code"] > 299: diff --git a/test_harness/runner/query_runner.py b/test_harness/runner/query_runner.py index 9db6fca..6f9470c 100644 --- a/test_harness/runner/query_runner.py +++ b/test_harness/runner/query_runner.py @@ -316,4 +316,4 @@ async def run_queries( except Exception as e: self.logger.error(f"Something went wrong with the queries: {e}") - return queries, normalized_curies \ No newline at end of file + return queries, normalized_curies