From 58a950a6e08a4ece6227b15acbdb7eba12e3cea5 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Wed, 30 Sep 2026 16:28:39 -0700 Subject: [PATCH 1/5] Sign msa-search nim Signed-off-by: Ohad Mosafi --- nim-skills/msa-search-nim/SKILL.md | 29 +++- .../config/skillspector-baseline.yml | 11 ++ nim-skills/msa-search-nim/evals/evals.json | 150 +++--------------- .../msa-search-nim/references/examples.md | 3 + .../msa-search-nim/scripts/hosted_search.py | 135 ++++++++++++++++ .../tests/test_hosted_search.py | 109 +++++++++++++ .../skills/msa-search-nim/SKILL.md | 29 +++- .../config/skillspector-baseline.yml | 11 ++ .../skills/msa-search-nim/evals/evals.json | 150 +++--------------- .../msa-search-nim/references/examples.md | 3 + .../msa-search-nim/scripts/hosted_search.py | 135 ++++++++++++++++ .../tests/test_hosted_search.py | 109 +++++++++++++ 12 files changed, 622 insertions(+), 252 deletions(-) create mode 100644 nim-skills/msa-search-nim/scripts/hosted_search.py create mode 100644 nim-skills/msa-search-nim/tests/test_hosted_search.py create mode 100644 skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py create mode 100644 skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py diff --git a/nim-skills/msa-search-nim/SKILL.md b/nim-skills/msa-search-nim/SKILL.md index 393f60c..efa9ef4 100644 --- a/nim-skills/msa-search-nim/SKILL.md +++ b/nim-skills/msa-search-nim/SKILL.md @@ -5,11 +5,14 @@ description: > license: Apache-2.0 AND CC-BY-4.0 compatibility: "requests>=2.28" allowed-tools: Bash, Read, Write, AskUserQuestion +permissions: + - env # reads NGC_API_KEY/NVIDIA_API_KEY and local NIM setup variables + - network # hosted MSA requests and documented NGC/local NIM setup --- # MSA-Search NIM -Generate protein MSAs with GPU-accelerated MMSeqs2. Use this `SKILL.md` for +Generate protein MSAs with GPU-accelerated MMSeqs2. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: - `references/api.md`: exact endpoints, schemas, Docker flags, response fields. @@ -280,6 +283,24 @@ Notes: Use exact case-sensitive database names and response keys. +For a hosted standard search, run the bundled client from this skill's directory. +It submits the real request, validates both database results, and saves the raw +JSON and A3M files. Choose a new output directory for each run: + +```bash +python scripts/hosted_search.py \ + --sequence SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR \ + --output-dir msa-output +``` + +The client reads `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment. It permits +at most two requests, each with a 10-second connection timeout and a 300-second +read timeout, with five seconds between attempts. If it exits nonzero, report the +service failure and stop. Do not restart it repeatedly, extend timeouts beyond the +task budget, or replace the missing response with synthetic alignments. + +The underlying request format, also usable with a running local NIM, is: + ```python import os import requests @@ -371,3 +392,9 @@ template, and sequence sanity checks, read `references/validation.md`. - Paired MSA requires at least two sequences. - Local URL 404 usually means an accidental `/v1/` prefix. - First local run can take hours while databases populate `LOCAL_NIM_CACHE`. +- Hosted HTTP 502/503/504 or repeated read timeouts indicate that the hosted + request did not complete. Check service availability after the bounded retry; + a longer client timeout cannot fix a server-generated HTTP 504. +- Do not invent a polling URL for `health.api.nvidia.com`. The published standard + MSA example uses synchronous POST; a pending response needs a documented + service-specific completion mechanism before it can count as a result. diff --git a/nim-skills/msa-search-nim/config/skillspector-baseline.yml b/nim-skills/msa-search-nim/config/skillspector-baseline.yml index 0246c5d..c575fb8 100644 --- a/nim-skills/msa-search-nim/config/skillspector-baseline.yml +++ b/nim-skills/msa-search-nim/config/skillspector-baseline.yml @@ -6,6 +6,17 @@ version: 1 rules: + - id: "PE3" + path: "evals/evals.json" + message: ".env " + reason: >- + Reviewed JSON-only false positive (2026-09-16). The two matches are + expected-output/assertion text in deferred local-setup case 3. They + describe the same optional repo-root dotenv setup documented in this + skill; the JSON does not read files or execute that setup. This rule + matches only the literal .env finding in this eval file. Credential + access in executable scripts and references to other secret stores + remain subject to scanning. - id: "PE3" path: "*SKILL.md" reason: >- diff --git a/nim-skills/msa-search-nim/evals/evals.json b/nim-skills/msa-search-nim/evals/evals.json index c04f9af..442b83c 100644 --- a/nim-skills/msa-search-nim/evals/evals.json +++ b/nim-skills/msa-search-nim/evals/evals.json @@ -7,41 +7,13 @@ "expected_output": "A successfully executed hosted MSA-Search request with Bearer auth, case-correct database names, and A3M output format, plus the actual returned alignment saved to a file and summarized from the response.", "files": [], "assertions": [ - { - "id": "hosted-request-executed", - "description": "Executes the hosted request instead of only writing code", - "check": "Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path" - }, - { - "id": "hosted-endpoint-url", - "description": "Uses the correct hosted MSA-Search endpoint URL", - "check": "Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'" - }, - { - "id": "bearer-auth-header", - "description": "Sets Authorization header with Bearer token from NGC_API_KEY", - "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'" - }, - { - "id": "sequence-field", - "description": "Request uses 'sequence' (singular) field with the provided sequence", - "check": "Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'" - }, - { - "id": "databases-field", - "description": "databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases", - "check": "Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'" - }, - { - "id": "a3m-output-format", - "description": "Requests A3M alignment output format", - "check": "Script contains 'output_alignment_formats' and 'a3m'" - }, - { - "id": "saves-alignment-output", - "description": "Saves the returned alignment to a file", - "check": "Script writes alignment content from 'alignments' in the response to a file" - } + "[hosted-request-executed] Executes the hosted request instead of only writing code: Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path", + "[hosted-endpoint-url] Uses the correct hosted MSA-Search endpoint URL: Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'", + "[bearer-auth-header] Sets Authorization header with Bearer token from NGC_API_KEY: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'", + "[sequence-field] Request uses 'sequence' (singular) field with the provided sequence: Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'", + "[databases-field] databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases: Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'", + "[a3m-output-format] Requests A3M alignment output format: Script contains 'output_alignment_formats' and 'a3m'", + "[saves-alignment-output] Saves the returned alignment to a file: Script writes alignment content from 'alignments' in the response to a file" ] } ], @@ -52,36 +24,12 @@ "expected_output": "A Python script that calls the hosted /paired/predict endpoint with a 'sequences' list containing both chains, extracts per-chain alignments from alignments_by_chain, and saves each chain's alignment to a separate file.", "files": [], "assertions": [ - { - "id": "paired-endpoint-url", - "description": "Uses the correct hosted paired MSA endpoint URL", - "check": "Script contains 'msa-search/paired/predict'" - }, - { - "id": "sequences-plural-field", - "description": "Uses 'sequences' (plural, list) field — not 'sequence' (singular)", - "check": "Script payload contains 'sequences' as a list/array, not 'sequence'" - }, - { - "id": "both-chains-present", - "description": "Both protein sequences are included in the request", - "check": "Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'" - }, - { - "id": "bearer-auth-header", - "description": "Sets Authorization header with Bearer token", - "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'" - }, - { - "id": "parses-alignments-by-chain", - "description": "Response parsed by 'alignments_by_chain' (not 'alignments')", - "check": "Script references 'alignments_by_chain' from the response" - }, - { - "id": "saves-per-chain-alignments", - "description": "Saves alignment for each chain to separate files", - "check": "Script saves at least two alignment files, one per chain" - } + "[paired-endpoint-url] Uses the correct hosted paired MSA endpoint URL: Script contains 'msa-search/paired/predict'", + "[sequences-plural-field] Uses 'sequences' (plural, list) field — not 'sequence' (singular): Script payload contains 'sequences' as a list/array, not 'sequence'", + "[both-chains-present] Both protein sequences are included in the request: Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'", + "[bearer-auth-header] Sets Authorization header with Bearer token: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'", + "[parses-alignments-by-chain] Response parsed by 'alignments_by_chain' (not 'alignments'): Script references 'alignments_by_chain' from the response", + "[saves-per-chain-alignments] Saves alignment for each chain to separate files: Script saves at least two alignment files, one per chain" ] }, { @@ -91,36 +39,12 @@ "expected_output": "Docker setup instructions using shell env first and optional repo-root .env overrides, requiring NGC_API_KEY or NVIDIA_API_KEY fallback plus LOCAL_NIM_CACHE, warning about the 1.4 TB database cache, health-checking the service, then sending a no-auth request to localhost:8000 without a /v1/ prefix.", "files": [], "assertions": [ - { - "id": "docker-image-tag", - "description": "References the correct MSA-Search container image with :2 tag", - "check": "Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'" - }, - { - "id": "env-contract-and-cache", - "description": "Local setup uses the repo env contract and LOCAL_NIM_CACHE", - "check": "Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache" - }, - { - "id": "storage-warning", - "description": "Mentions the large storage requirement for databases", - "check": "Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases" - }, - { - "id": "nim-cache-mount", - "description": "Mounts cache directory to /opt/nim/.cache", - "check": "Output contains '/opt/nim/.cache' in the volume mount" - }, - { - "id": "health-check", - "description": "Includes health check before submitting request", - "check": "Output contains health check against localhost:8000/v1/health/ready" - }, - { - "id": "local-endpoint-no-v1", - "description": "Local prediction request uses path without /v1/ prefix", - "check": "Script contains 'localhost:8000/biology/colabfold/msa-search/predict'" - } + "[docker-image-tag] References the correct MSA-Search container image with :2 tag: Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'", + "[env-contract-and-cache] Local setup uses the repo env contract and LOCAL_NIM_CACHE: Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache", + "[storage-warning] Mentions the large storage requirement for databases: Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases", + "[nim-cache-mount] Mounts cache directory to /opt/nim/.cache: Output contains '/opt/nim/.cache' in the volume mount", + "[health-check] Includes health check before submitting request: Output contains health check against localhost:8000/v1/health/ready", + "[local-endpoint-no-v1] Local prediction request uses path without /v1/ prefix: Script contains 'localhost:8000/biology/colabfold/msa-search/predict'" ] }, { @@ -130,36 +54,12 @@ "expected_output": "A Python script targeting the local /biology/colabfold/msa-search/structure-templates/predict endpoint because the hosted health.api template path returned HTTP 404 in validation, with structural_template_databases=['pdb70_220313'], the sequence, max_structures=20, max_msa_sequences=500 to match NIM_GLOBAL_MAX_MSA_DEPTH, no Authorization header for localhost inference, and parsing/saving both returned mmCIF template structures and search_hits M8 tables.", "files": [], "assertions": [ - { - "id": "structure-templates-endpoint", - "description": "Uses the structure-templates endpoint, not the standard predict endpoint", - "check": "Script contains 'structure-templates/predict'" - }, - { - "id": "sequence-field", - "description": "Request uses 'sequence' (singular) field", - "check": "Script contains 'sequence' field with the provided sequence" - }, - { - "id": "max-structures-param", - "description": "max_structures is set to 20 and max_msa_sequences matches the default GPU server depth", - "check": "Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH" - }, - { - "id": "pdb70-canonical-database", - "description": "Uses the canonical pdb70_220313 template database name", - "check": "Script sets structural_template_databases to include 'pdb70_220313'" - }, - { - "id": "local-no-auth", - "description": "Uses no Authorization header for local template inference", - "check": "Script does not send Authorization/Bearer headers to localhost" - }, - { - "id": "parses-structures-and-search-hits", - "description": "Parses template structures and search_hits M8 output", - "check": "Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table" - } + "[structure-templates-endpoint] Uses the structure-templates endpoint, not the standard predict endpoint: Script contains 'structure-templates/predict'", + "[sequence-field] Request uses 'sequence' (singular) field: Script contains 'sequence' field with the provided sequence", + "[max-structures-param] max_structures is set to 20 and max_msa_sequences matches the default GPU server depth: Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH", + "[pdb70-canonical-database] Uses the canonical pdb70_220313 template database name: Script sets structural_template_databases to include 'pdb70_220313'", + "[local-no-auth] Uses no Authorization header for local template inference: Script does not send Authorization/Bearer headers to localhost", + "[parses-structures-and-search-hits] Parses template structures and search_hits M8 output: Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table" ] } ] diff --git a/nim-skills/msa-search-nim/references/examples.md b/nim-skills/msa-search-nim/references/examples.md index 4f5c176..18a2702 100644 --- a/nim-skills/msa-search-nim/references/examples.md +++ b/nim-skills/msa-search-nim/references/examples.md @@ -21,6 +21,9 @@ exact aria2c + `NIM_MODEL_NAME` commands. ## Hosted Standard MSA +Run `scripts/hosted_search.py` from the skill root to save the response and A3M +files. The request payload is: + ```python payload = { "sequence": sequence, diff --git a/nim-skills/msa-search-nim/scripts/hosted_search.py b/nim-skills/msa-search-nim/scripts/hosted_search.py new file mode 100644 index 0000000..1d76e77 --- /dev/null +++ b/nim-skills/msa-search-nim/scripts/hosted_search.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Run hosted MSA search with bounded retries and save actual A3M results. + +Requires requests>=2.28 and NGC_API_KEY (or NVIDIA_API_KEY). The two attempts +use 10-second connect and 300-second read timeouts. A failed hosted service +must remain a failure; this client never substitutes a fabricated alignment. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys +import time + +import requests + + +URL = "https://health.api.nvidia.com/v1/biology/colabfold/msa-search/predict" +DATABASES = ("Uniref30_2302", "colabfold_envdb_202108") +REQUEST_TIMEOUT = (10, 300) +MAX_ATTEMPTS = 2 +RETRY_DELAY = 5 +RETRYABLE_STATUS = {429, 502, 503, 504} + + +class SearchError(RuntimeError): + """The hosted service did not produce the requested alignments.""" + + +def search(sequence: str, databases: list[str], api_key: str) -> dict: + """Return a validated response, or fail after at most two requests. + + Do not forward server error bodies or requests exceptions to stderr: + those can contain sensitive request information. No redirects are followed + with the Authorization header. Database names are also used as filenames, + so only the documented database names are accepted. + """ + if not api_key: + raise SearchError("Set NGC_API_KEY or NVIDIA_API_KEY before running hosted search.") + if not 1 <= len(sequence) <= 4096 or any(c not in "ACDEFGHIKLMNPQRSTVWYX" for c in sequence): + raise SearchError("Sequence must contain 1–4096 uppercase amino-acid letters (including X).") + if not databases or any(db not in DATABASES for db in databases): + raise SearchError("Select one or both documented databases.") + payload = { + "sequence": sequence, + "databases": list(dict.fromkeys(databases)), + "e_value": 0.0001, + "output_alignment_formats": ["a3m"], + } + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} + last_error = "No response received." + for attempt in range(MAX_ATTEMPTS): + response = None + try: + response = requests.post( + URL, headers=headers, json=payload, + timeout=REQUEST_TIMEOUT, allow_redirects=False, + ) + except (requests.Timeout, requests.ConnectionError): + last_error = "Hosted MSA request timed out or could not connect." + except requests.RequestException: + raise SearchError("Hosted MSA request failed before a usable response arrived.") from None + else: + try: + if response.status_code == 200: + try: + result = response.json() + except ValueError: + raise SearchError("Hosted MSA returned invalid JSON.") from None + validate_alignments(result, payload["databases"]) + return result + last_error = f"Hosted MSA returned HTTP {response.status_code}." + if response.status_code not in RETRYABLE_STATUS: + raise SearchError(last_error) + finally: + response.close() + if attempt + 1 < MAX_ATTEMPTS: + time.sleep(RETRY_DELAY) + raise SearchError( + f"{last_error} Stopped after {MAX_ATTEMPTS} attempts. " + "No alignment was produced. Check hosted-service availability before retrying." + ) + + +def validate_alignments(result: object, databases: list[str]) -> None: + """Require an actual, nonempty A3M result for every requested database.""" + alignments = result.get("alignments") if isinstance(result, dict) else None + if not isinstance(alignments, dict): + raise SearchError("Hosted MSA response has no alignments object.") + for database in databases: + formats = alignments.get(database) + a3m = formats.get("a3m") if isinstance(formats, dict) else None + alignment = a3m.get("alignment") if isinstance(a3m, dict) else None + if not isinstance(alignment, str) or not alignment.lstrip().startswith(">"): + raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.") + if not any(line.strip() and not line.startswith((">", "#")) for line in alignment.splitlines()): + raise SearchError(f"Hosted MSA returned an empty A3M alignment for {database}.") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sequence", required=True) + parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES)) + parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request") + args = parser.parse_args() + if args.output_dir.exists(): + parser.error("--output-dir must not already exist; use a new path for each request") + try: + result = search( + args.sequence, args.databases, + os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""), + ) + # Create output only after all requested alignments have been validated. + args.output_dir.mkdir(parents=True, exist_ok=False) + (args.output_dir / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + for database in dict.fromkeys(args.databases): + alignment = result["alignments"][database]["a3m"]["alignment"] + path = args.output_dir / f"{database}.a3m" + path.write_text(alignment, encoding="utf-8") + records = sum(line.startswith(">") for line in alignment.splitlines()) + print(f"{path}: {records} sequences") + except SearchError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + except OSError: + print("ERROR: Could not save the hosted response to the output directory.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/nim-skills/msa-search-nim/tests/test_hosted_search.py b/nim-skills/msa-search-nim/tests/test_hosted_search.py new file mode 100644 index 0000000..87a925c --- /dev/null +++ b/nim-skills/msa-search-nim/tests/test_hosted_search.py @@ -0,0 +1,109 @@ +"""Offline transport and artifact checks; no hosted-service calls are made.""" + +from contextlib import redirect_stderr, redirect_stdout +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + + +SPEC = importlib.util.spec_from_file_location( + "hosted_search", Path(__file__).resolve().parents[1] / "scripts" / "hosted_search.py" +) +client = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(client) + + +def response(status=200, data=None): + result = Mock(status_code=status) + result.json.return_value = data + return result + + +def alignments(): + return {"alignments": { + db: {"a3m": {"alignment": ">query\nACDE\n>hit\nAC-E\n", "format": "a3m"}} + for db in client.DATABASES + }} + + +class HostedSearchTests(unittest.TestCase): + def test_gateway_timeout_retries_once_then_stops(self): + failures = [response(504), response(504), response(data=alignments())] + with patch.object(client.requests, "post", side_effect=failures) as post, \ + patch.object(client.time, "sleep"): + with self.assertRaisesRegex(client.SearchError, "HTTP 504.*Stopped after 2"): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 2) + for call in post.call_args_list: + self.assertEqual(call.kwargs["timeout"], (10, 300)) + self.assertFalse(call.kwargs["allow_redirects"]) + + def test_transient_error_can_recover_without_changing_the_request(self): + expected = alignments() + with patch.object(client.requests, "post", side_effect=[response(503), response(data=expected)]) as post, \ + patch.object(client.time, "sleep"): + actual = client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(actual, expected) + self.assertEqual(post.call_args_list[0], post.call_args_list[1]) + + def test_auth_error_and_redirect_are_not_retried(self): + for status in [401, 403, 302]: + with self.subTest(status=status), patch.object(client.requests, "post", return_value=response(status)) as post: + with self.assertRaisesRegex(client.SearchError, f"HTTP {status}"): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + + def test_timeout_does_not_expose_exception_text(self): + with patch.object(client.requests, "post", side_effect=client.requests.ReadTimeout("sensitive request details")) as post, \ + patch.object(client.time, "sleep"): + with self.assertRaises(client.SearchError) as failure: + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertNotIn("sensitive", str(failure.exception)) + self.assertEqual(post.call_count, 2) + + def test_partial_or_empty_alignment_is_not_success(self): + partial = alignments() + del partial["alignments"][client.DATABASES[1]] + empty = alignments() + empty["alignments"][client.DATABASES[0]]["a3m"]["alignment"] = ">query\n" + for result in [None, {}, partial, empty]: + with self.subTest(result=result), patch.object(client.requests, "post", return_value=response(data=result)): + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + + def test_missing_key_and_unknown_database_fail_before_network(self): + for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]: + with self.subTest(databases=databases), patch.object(client.requests, "post") as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", databases, key) + post.assert_not_called() + + def test_cli_writes_actual_response_and_alignments(self): + expected = alignments() + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=expected)), redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual(json.loads((output / "response.json").read_text()), expected) + for database in client.DATABASES: + self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"]) + + def test_cli_failure_leaves_no_success_artifacts(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(504)), \ + patch.object(client.time, "sleep"), redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md index 393f60c..efa9ef4 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md @@ -5,11 +5,14 @@ description: > license: Apache-2.0 AND CC-BY-4.0 compatibility: "requests>=2.28" allowed-tools: Bash, Read, Write, AskUserQuestion +permissions: + - env # reads NGC_API_KEY/NVIDIA_API_KEY and local NIM setup variables + - network # hosted MSA requests and documented NGC/local NIM setup --- # MSA-Search NIM -Generate protein MSAs with GPU-accelerated MMSeqs2. Use this `SKILL.md` for +Generate protein MSAs with GPU-accelerated MMSeqs2. Use this guide for first-pass hosted/local usage; load supplemental files only when needed: - `references/api.md`: exact endpoints, schemas, Docker flags, response fields. @@ -280,6 +283,24 @@ Notes: Use exact case-sensitive database names and response keys. +For a hosted standard search, run the bundled client from this skill's directory. +It submits the real request, validates both database results, and saves the raw +JSON and A3M files. Choose a new output directory for each run: + +```bash +python scripts/hosted_search.py \ + --sequence SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR \ + --output-dir msa-output +``` + +The client reads `NGC_API_KEY` or `NVIDIA_API_KEY` from the environment. It permits +at most two requests, each with a 10-second connection timeout and a 300-second +read timeout, with five seconds between attempts. If it exits nonzero, report the +service failure and stop. Do not restart it repeatedly, extend timeouts beyond the +task budget, or replace the missing response with synthetic alignments. + +The underlying request format, also usable with a running local NIM, is: + ```python import os import requests @@ -371,3 +392,9 @@ template, and sequence sanity checks, read `references/validation.md`. - Paired MSA requires at least two sequences. - Local URL 404 usually means an accidental `/v1/` prefix. - First local run can take hours while databases populate `LOCAL_NIM_CACHE`. +- Hosted HTTP 502/503/504 or repeated read timeouts indicate that the hosted + request did not complete. Check service availability after the bounded retry; + a longer client timeout cannot fix a server-generated HTTP 504. +- Do not invent a polling URL for `health.api.nvidia.com`. The published standard + MSA example uses synchronous POST; a pending response needs a documented + service-specific completion mechanism before it can count as a result. diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml b/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml index 0246c5d..c575fb8 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/config/skillspector-baseline.yml @@ -6,6 +6,17 @@ version: 1 rules: + - id: "PE3" + path: "evals/evals.json" + message: ".env " + reason: >- + Reviewed JSON-only false positive (2026-09-16). The two matches are + expected-output/assertion text in deferred local-setup case 3. They + describe the same optional repo-root dotenv setup documented in this + skill; the JSON does not read files or execute that setup. This rule + matches only the literal .env finding in this eval file. Credential + access in executable scripts and references to other secret stores + remain subject to scanning. - id: "PE3" path: "*SKILL.md" reason: >- diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json b/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json index c04f9af..442b83c 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/evals/evals.json @@ -7,41 +7,13 @@ "expected_output": "A successfully executed hosted MSA-Search request with Bearer auth, case-correct database names, and A3M output format, plus the actual returned alignment saved to a file and summarized from the response.", "files": [], "assertions": [ - { - "id": "hosted-request-executed", - "description": "Executes the hosted request instead of only writing code", - "check": "Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path" - }, - { - "id": "hosted-endpoint-url", - "description": "Uses the correct hosted MSA-Search endpoint URL", - "check": "Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'" - }, - { - "id": "bearer-auth-header", - "description": "Sets Authorization header with Bearer token from NGC_API_KEY", - "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'" - }, - { - "id": "sequence-field", - "description": "Request uses 'sequence' (singular) field with the provided sequence", - "check": "Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'" - }, - { - "id": "databases-field", - "description": "databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases", - "check": "Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'" - }, - { - "id": "a3m-output-format", - "description": "Requests A3M alignment output format", - "check": "Script contains 'output_alignment_formats' and 'a3m'" - }, - { - "id": "saves-alignment-output", - "description": "Saves the returned alignment to a file", - "check": "Script writes alignment content from 'alignments' in the response to a file" - } + "[hosted-request-executed] Executes the hosted request instead of only writing code: Trajectory shows successful execution of the hosted request, and the final response reports actual response-derived alignment information and the saved A3M path", + "[hosted-endpoint-url] Uses the correct hosted MSA-Search endpoint URL: Script contains 'health.api.nvidia.com/v1/biology/colabfold/msa-search/predict'", + "[bearer-auth-header] Sets Authorization header with Bearer token from NGC_API_KEY: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'", + "[sequence-field] Request uses 'sequence' (singular) field with the provided sequence: Script contains 'sequence' field and 'SGSMKTAISLPDETFDRVSRRASELGMSRSEFFTKAAQR'", + "[databases-field] databases field specifies case-correct hosted database names, not the lowercase or unversioned aliases: Script contains 'databases' and 'Uniref30_2302' and 'colabfold_envdb_202108'", + "[a3m-output-format] Requests A3M alignment output format: Script contains 'output_alignment_formats' and 'a3m'", + "[saves-alignment-output] Saves the returned alignment to a file: Script writes alignment content from 'alignments' in the response to a file" ] } ], @@ -52,36 +24,12 @@ "expected_output": "A Python script that calls the hosted /paired/predict endpoint with a 'sequences' list containing both chains, extracts per-chain alignments from alignments_by_chain, and saves each chain's alignment to a separate file.", "files": [], "assertions": [ - { - "id": "paired-endpoint-url", - "description": "Uses the correct hosted paired MSA endpoint URL", - "check": "Script contains 'msa-search/paired/predict'" - }, - { - "id": "sequences-plural-field", - "description": "Uses 'sequences' (plural, list) field — not 'sequence' (singular)", - "check": "Script payload contains 'sequences' as a list/array, not 'sequence'" - }, - { - "id": "both-chains-present", - "description": "Both protein sequences are included in the request", - "check": "Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'" - }, - { - "id": "bearer-auth-header", - "description": "Sets Authorization header with Bearer token", - "check": "Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'" - }, - { - "id": "parses-alignments-by-chain", - "description": "Response parsed by 'alignments_by_chain' (not 'alignments')", - "check": "Script references 'alignments_by_chain' from the response" - }, - { - "id": "saves-per-chain-alignments", - "description": "Saves alignment for each chain to separate files", - "check": "Script saves at least two alignment files, one per chain" - } + "[paired-endpoint-url] Uses the correct hosted paired MSA endpoint URL: Script contains 'msa-search/paired/predict'", + "[sequences-plural-field] Uses 'sequences' (plural, list) field — not 'sequence' (singular): Script payload contains 'sequences' as a list/array, not 'sequence'", + "[both-chains-present] Both protein sequences are included in the request: Script contains 'VLSPADKTNVKAAWGKVGAHAG' and 'MHLTPEEKSAVTALWGKVNVD'", + "[bearer-auth-header] Sets Authorization header with Bearer token: Script contains 'Authorization' and 'Bearer' and 'NGC_API_KEY'", + "[parses-alignments-by-chain] Response parsed by 'alignments_by_chain' (not 'alignments'): Script references 'alignments_by_chain' from the response", + "[saves-per-chain-alignments] Saves alignment for each chain to separate files: Script saves at least two alignment files, one per chain" ] }, { @@ -91,36 +39,12 @@ "expected_output": "Docker setup instructions using shell env first and optional repo-root .env overrides, requiring NGC_API_KEY or NVIDIA_API_KEY fallback plus LOCAL_NIM_CACHE, warning about the 1.4 TB database cache, health-checking the service, then sending a no-auth request to localhost:8000 without a /v1/ prefix.", "files": [], "assertions": [ - { - "id": "docker-image-tag", - "description": "References the correct MSA-Search container image with :2 tag", - "check": "Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'" - }, - { - "id": "env-contract-and-cache", - "description": "Local setup uses the repo env contract and LOCAL_NIM_CACHE", - "check": "Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache" - }, - { - "id": "storage-warning", - "description": "Mentions the large storage requirement for databases", - "check": "Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases" - }, - { - "id": "nim-cache-mount", - "description": "Mounts cache directory to /opt/nim/.cache", - "check": "Output contains '/opt/nim/.cache' in the volume mount" - }, - { - "id": "health-check", - "description": "Includes health check before submitting request", - "check": "Output contains health check against localhost:8000/v1/health/ready" - }, - { - "id": "local-endpoint-no-v1", - "description": "Local prediction request uses path without /v1/ prefix", - "check": "Script contains 'localhost:8000/biology/colabfold/msa-search/predict'" - } + "[docker-image-tag] References the correct MSA-Search container image with :2 tag: Output contains 'nvcr.io/nim/colabfold/msa-search' and ':2'", + "[env-contract-and-cache] Local setup uses the repo env contract and LOCAL_NIM_CACHE: Output sources repo-root .env only if present, supports NVIDIA_API_KEY fallback to NGC_API_KEY, requires LOCAL_NIM_CACHE, and mounts LOCAL_NIM_CACHE to /opt/nim/.cache", + "[storage-warning] Mentions the large storage requirement for databases: Output mentions at least 1 TB or 1.4 TB or 1660 GB of storage needed for databases", + "[nim-cache-mount] Mounts cache directory to /opt/nim/.cache: Output contains '/opt/nim/.cache' in the volume mount", + "[health-check] Includes health check before submitting request: Output contains health check against localhost:8000/v1/health/ready", + "[local-endpoint-no-v1] Local prediction request uses path without /v1/ prefix: Script contains 'localhost:8000/biology/colabfold/msa-search/predict'" ] }, { @@ -130,36 +54,12 @@ "expected_output": "A Python script targeting the local /biology/colabfold/msa-search/structure-templates/predict endpoint because the hosted health.api template path returned HTTP 404 in validation, with structural_template_databases=['pdb70_220313'], the sequence, max_structures=20, max_msa_sequences=500 to match NIM_GLOBAL_MAX_MSA_DEPTH, no Authorization header for localhost inference, and parsing/saving both returned mmCIF template structures and search_hits M8 tables.", "files": [], "assertions": [ - { - "id": "structure-templates-endpoint", - "description": "Uses the structure-templates endpoint, not the standard predict endpoint", - "check": "Script contains 'structure-templates/predict'" - }, - { - "id": "sequence-field", - "description": "Request uses 'sequence' (singular) field", - "check": "Script contains 'sequence' field with the provided sequence" - }, - { - "id": "max-structures-param", - "description": "max_structures is set to 20 and max_msa_sequences matches the default GPU server depth", - "check": "Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH" - }, - { - "id": "pdb70-canonical-database", - "description": "Uses the canonical pdb70_220313 template database name", - "check": "Script sets structural_template_databases to include 'pdb70_220313'" - }, - { - "id": "local-no-auth", - "description": "Uses no Authorization header for local template inference", - "check": "Script does not send Authorization/Bearer headers to localhost" - }, - { - "id": "parses-structures-and-search-hits", - "description": "Parses template structures and search_hits M8 output", - "check": "Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table" - } + "[structure-templates-endpoint] Uses the structure-templates endpoint, not the standard predict endpoint: Script contains 'structure-templates/predict'", + "[sequence-field] Request uses 'sequence' (singular) field: Script contains 'sequence' field with the provided sequence", + "[max-structures-param] max_structures is set to 20 and max_msa_sequences matches the default GPU server depth: Script contains 'max_structures' and '20', plus 'max_msa_sequences' and '500' or explains it must match NIM_GLOBAL_MAX_MSA_DEPTH", + "[pdb70-canonical-database] Uses the canonical pdb70_220313 template database name: Script sets structural_template_databases to include 'pdb70_220313'", + "[local-no-auth] Uses no Authorization header for local template inference: Script does not send Authorization/Bearer headers to localhost", + "[parses-structures-and-search-hits] Parses template structures and search_hits M8 output: Script references both 'structures' for mmCIF content and 'search_hits'/'m8' for the hit table" ] } ] diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md index 4f5c176..18a2702 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/examples.md @@ -21,6 +21,9 @@ exact aria2c + `NIM_MODEL_NAME` commands. ## Hosted Standard MSA +Run `scripts/hosted_search.py` from the skill root to save the response and A3M +files. The request payload is: + ```python payload = { "sequence": sequence, diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py new file mode 100644 index 0000000..1d76e77 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Run hosted MSA search with bounded retries and save actual A3M results. + +Requires requests>=2.28 and NGC_API_KEY (or NVIDIA_API_KEY). The two attempts +use 10-second connect and 300-second read timeouts. A failed hosted service +must remain a failure; this client never substitutes a fabricated alignment. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys +import time + +import requests + + +URL = "https://health.api.nvidia.com/v1/biology/colabfold/msa-search/predict" +DATABASES = ("Uniref30_2302", "colabfold_envdb_202108") +REQUEST_TIMEOUT = (10, 300) +MAX_ATTEMPTS = 2 +RETRY_DELAY = 5 +RETRYABLE_STATUS = {429, 502, 503, 504} + + +class SearchError(RuntimeError): + """The hosted service did not produce the requested alignments.""" + + +def search(sequence: str, databases: list[str], api_key: str) -> dict: + """Return a validated response, or fail after at most two requests. + + Do not forward server error bodies or requests exceptions to stderr: + those can contain sensitive request information. No redirects are followed + with the Authorization header. Database names are also used as filenames, + so only the documented database names are accepted. + """ + if not api_key: + raise SearchError("Set NGC_API_KEY or NVIDIA_API_KEY before running hosted search.") + if not 1 <= len(sequence) <= 4096 or any(c not in "ACDEFGHIKLMNPQRSTVWYX" for c in sequence): + raise SearchError("Sequence must contain 1–4096 uppercase amino-acid letters (including X).") + if not databases or any(db not in DATABASES for db in databases): + raise SearchError("Select one or both documented databases.") + payload = { + "sequence": sequence, + "databases": list(dict.fromkeys(databases)), + "e_value": 0.0001, + "output_alignment_formats": ["a3m"], + } + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"} + last_error = "No response received." + for attempt in range(MAX_ATTEMPTS): + response = None + try: + response = requests.post( + URL, headers=headers, json=payload, + timeout=REQUEST_TIMEOUT, allow_redirects=False, + ) + except (requests.Timeout, requests.ConnectionError): + last_error = "Hosted MSA request timed out or could not connect." + except requests.RequestException: + raise SearchError("Hosted MSA request failed before a usable response arrived.") from None + else: + try: + if response.status_code == 200: + try: + result = response.json() + except ValueError: + raise SearchError("Hosted MSA returned invalid JSON.") from None + validate_alignments(result, payload["databases"]) + return result + last_error = f"Hosted MSA returned HTTP {response.status_code}." + if response.status_code not in RETRYABLE_STATUS: + raise SearchError(last_error) + finally: + response.close() + if attempt + 1 < MAX_ATTEMPTS: + time.sleep(RETRY_DELAY) + raise SearchError( + f"{last_error} Stopped after {MAX_ATTEMPTS} attempts. " + "No alignment was produced. Check hosted-service availability before retrying." + ) + + +def validate_alignments(result: object, databases: list[str]) -> None: + """Require an actual, nonempty A3M result for every requested database.""" + alignments = result.get("alignments") if isinstance(result, dict) else None + if not isinstance(alignments, dict): + raise SearchError("Hosted MSA response has no alignments object.") + for database in databases: + formats = alignments.get(database) + a3m = formats.get("a3m") if isinstance(formats, dict) else None + alignment = a3m.get("alignment") if isinstance(a3m, dict) else None + if not isinstance(alignment, str) or not alignment.lstrip().startswith(">"): + raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.") + if not any(line.strip() and not line.startswith((">", "#")) for line in alignment.splitlines()): + raise SearchError(f"Hosted MSA returned an empty A3M alignment for {database}.") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sequence", required=True) + parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES)) + parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request") + args = parser.parse_args() + if args.output_dir.exists(): + parser.error("--output-dir must not already exist; use a new path for each request") + try: + result = search( + args.sequence, args.databases, + os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""), + ) + # Create output only after all requested alignments have been validated. + args.output_dir.mkdir(parents=True, exist_ok=False) + (args.output_dir / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + for database in dict.fromkeys(args.databases): + alignment = result["alignments"][database]["a3m"]["alignment"] + path = args.output_dir / f"{database}.a3m" + path.write_text(alignment, encoding="utf-8") + records = sum(line.startswith(">") for line in alignment.splitlines()) + print(f"{path}: {records} sequences") + except SearchError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + except OSError: + print("ERROR: Could not save the hosted response to the output directory.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py new file mode 100644 index 0000000..87a925c --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py @@ -0,0 +1,109 @@ +"""Offline transport and artifact checks; no hosted-service calls are made.""" + +from contextlib import redirect_stderr, redirect_stdout +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + + +SPEC = importlib.util.spec_from_file_location( + "hosted_search", Path(__file__).resolve().parents[1] / "scripts" / "hosted_search.py" +) +client = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(client) + + +def response(status=200, data=None): + result = Mock(status_code=status) + result.json.return_value = data + return result + + +def alignments(): + return {"alignments": { + db: {"a3m": {"alignment": ">query\nACDE\n>hit\nAC-E\n", "format": "a3m"}} + for db in client.DATABASES + }} + + +class HostedSearchTests(unittest.TestCase): + def test_gateway_timeout_retries_once_then_stops(self): + failures = [response(504), response(504), response(data=alignments())] + with patch.object(client.requests, "post", side_effect=failures) as post, \ + patch.object(client.time, "sleep"): + with self.assertRaisesRegex(client.SearchError, "HTTP 504.*Stopped after 2"): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 2) + for call in post.call_args_list: + self.assertEqual(call.kwargs["timeout"], (10, 300)) + self.assertFalse(call.kwargs["allow_redirects"]) + + def test_transient_error_can_recover_without_changing_the_request(self): + expected = alignments() + with patch.object(client.requests, "post", side_effect=[response(503), response(data=expected)]) as post, \ + patch.object(client.time, "sleep"): + actual = client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(actual, expected) + self.assertEqual(post.call_args_list[0], post.call_args_list[1]) + + def test_auth_error_and_redirect_are_not_retried(self): + for status in [401, 403, 302]: + with self.subTest(status=status), patch.object(client.requests, "post", return_value=response(status)) as post: + with self.assertRaisesRegex(client.SearchError, f"HTTP {status}"): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + + def test_timeout_does_not_expose_exception_text(self): + with patch.object(client.requests, "post", side_effect=client.requests.ReadTimeout("sensitive request details")) as post, \ + patch.object(client.time, "sleep"): + with self.assertRaises(client.SearchError) as failure: + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertNotIn("sensitive", str(failure.exception)) + self.assertEqual(post.call_count, 2) + + def test_partial_or_empty_alignment_is_not_success(self): + partial = alignments() + del partial["alignments"][client.DATABASES[1]] + empty = alignments() + empty["alignments"][client.DATABASES[0]]["a3m"]["alignment"] = ">query\n" + for result in [None, {}, partial, empty]: + with self.subTest(result=result), patch.object(client.requests, "post", return_value=response(data=result)): + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + + def test_missing_key_and_unknown_database_fail_before_network(self): + for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]: + with self.subTest(databases=databases), patch.object(client.requests, "post") as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", databases, key) + post.assert_not_called() + + def test_cli_writes_actual_response_and_alignments(self): + expected = alignments() + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=expected)), redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual(json.loads((output / "response.json").read_text()), expected) + for database in client.DATABASES: + self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"]) + + def test_cli_failure_leaves_no_success_artifacts(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(504)), \ + patch.object(client.time, "sleep"), redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() From b47fcf5d6d6ab72b83e776f60f71b3b1db2b7015 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 30 Sep 2026 23:51:24 +0000 Subject: [PATCH 2/5] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../skills/msa-search-nim/BENCHMARK.md | 119 ++++++++++++++++++ .../skills/msa-search-nim/skill-card.md | 86 +++++++++++++ .../skills/msa-search-nim/skill.oms.sig | 1 + 3 files changed, 206 insertions(+) create mode 100644 skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md create mode 100644 skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md create mode 100644 skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md new file mode 100644 index 0000000..ffb2f26 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md @@ -0,0 +1,119 @@ +# Skill Benchmark: msa-search-nim + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `msa-search-nim` +- Evaluation date: 2026-09-30 +- Evaluator version: `1.5.6` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 1 evaluation tasks (1 positive) +- Dataset digest: `sha256:da6512955763a8e542da5599461202e0804188cd0eb5cd50377924e40f71d403` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 3 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 73.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 100.0% (+50.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 0.0% (-100.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 67.9% → 100.0% (+32.1 points) | 100.0% → 95.0% (-5.0 points) | +| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 486,740 | 1,135,884 | -649,144 | -57.15% | skill 1/1; base 1/1 | +| claude-code | 1 | 486,740 | 1,135,884 | -649,144 | -57.15% | skill 1/1; base 1/1 | +| codex | All cases | 205,561 | 232,346 | -26,785 | -11.53% | skill 1/1; base 1/1 | +| codex | 1 | 205,561 | 232,346 | -26,785 | -11.53% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 692,301 | 1,368,230 | -675,929 | -49.40% | skill 2/2; base 2/2 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 38 finding(s) | +| Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 1 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** QUALITY/quality_correctness: No documented scripts in table format (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: Instructions don't mention 'run_script' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) +- **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) +- **MEDIUM** SCHEMA/folder_hierarchy: Unexpected nesting depth for general skill (`skills/bionemo-agent-toolkit/skills/msa-search-nim`) +- 33 additional finding(s) are available in the full evaluation artifacts. + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md new file mode 100644 index 0000000..d17398c --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md @@ -0,0 +1,86 @@ +## Description:
+Generate multiple sequence alignments (MSAs) for protein sequences using the ColabFold MSA-Search NIM.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0 AND CC-BY-4.0
+## Use Case:
+Developers and engineers use this skill to generate multiple sequence alignments for protein sequences via GPU-accelerated MMSeqs2, supporting hosted NVIDIA API or local Docker NIM deployment for homolog search, paired MSA for complexes, and structural template retrieval.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [API key]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [API Reference](references/api.md)
+- [Science Background](references/science.md)
+- [Parameters Guide](references/parameters.md)
+- [Validation Guide](references/validation.md)
+- [Examples](references/examples.md)
+ + +## Skill Output:
+**Output Type(s):** [API Calls, Shell commands, Files]
+**Output Format:** [JSON responses, A3M alignment files, mmCIF structure files]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+Evaluated against 1 task (1 positive case, 3 attempts per task) in isolated k8s-sandbox pods.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Checks final-answer correctness against the reference answer.
+- Discoverability: Checks whether the expected skill was selected and the workflow executed.
+- Effectiveness: Checks goal completion (50%) and expected workflow adherence (50%).
+- Efficiency: Checks tool-call productivity (50%) and token efficiency (50%).
+ +Underlying evaluation signals used in this run:
+- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+- `skill_efficiency`: Tool-call productivity.
+- `token_efficiency`: Actual uncached prompt plus completion token usage.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 73.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 100.0% (+50.0 points) | 100.0% → 100.0% (±0.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 0.0% (-100.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 67.9% → 100.0% (+32.1 points) | 100.0% → 95.0% (-5.0 points) | +| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | + +## Skill Version(s):
+0.1.0 (source: pyproject.toml)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig new file mode 100644 index 0000000..95e7ac5 --- /dev/null +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibXNhLXNlYXJjaC1uaW0iLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMDNmODJkYWMxYWY2M2RlMThjNTY4MDdjYjc3MDM1NjhiMTk5YzQ3YTk5Zjg5ZGE4MDZkM2Y4ZGZmN2Q5OTFkMSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2VjNjczMWEyODNlN2FkYzcxNGVjNGJkMWNlYjE1OTgxYWMzNWMzNTBkYTU1M2Y2NDhlNTdlZWIyY2Y4ZDZjMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIzZDQyYzUxMzZjNWRhMWJjZTg4NjJlNjdmZWEyNTc0MDk1OGM3ZGFiYTFmMjdhMWM2MTI2OTVhYzRkZDAzMDYyIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImNvbmZpZy9za2lsbHNwZWN0b3ItYmFzZWxpbmUueW1sIiwKICAgICAgICAiZGlnZXN0IjogImRhMmQxMjMzNzU1YzZhNzZkNjNhNmFkZDY3ZjBjYmNiZGI2NjY0NzBjNzc4MjE0ZGIxMWNhODdiZThiYmQ0MWYiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICJiYTJiOGNmMGVhZDEzYmZiNjVkODEzMDllMTYzMTYyZGJjMmM2YzUwNTg5NjRjNGFlYTQxNGUwMDcwODk4YThhIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiOTQwMzlkNmU0ODY3MTljYWFiZjU2YjNkM2I3ZWRjYzk1ZTcyNDhlZjI4NjQwYTM5M2IzZTg2ODA1NTQxYjBhNyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy90cmlnZ2VyX2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiMWY3YzUyMWY2OThhYTBiMDA3NmIxNDc2MjI2NWFiZjU3MDAzYWJkN2ZjNzdhM2Y5ZGUyZWY5NjExMWU1M2Y4OCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FwaS5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0ZDIwYTMwMDEwNDdjZDY3NjQ1YTYxNDViMTI4YmYwZTFkODJkZmViZDA0ZmM0YzU0ZmY4NmRmOWQ0NWQ0MTg3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZXhhbXBsZXMubWQiLAogICAgICAgICJkaWdlc3QiOiAiMzgxNmEyNDcyZmU4ODI4OGM2M2I1NzgxMmQ0N2ZmOGIxNDNmZTI1MWQxZmYyM2MwNjVhODhkOTkyM2JhMzYzYSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BhcmFtZXRlcnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiMDI1ZDQwMGY3ZWM5Nzg4YWE0YTFiMjk4N2QwOWRkM2I0ZTdkN2RkMTVjMmU4MWM2MTQyYjI3NjNjYmZlYTViNCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3NjaWVuY2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTA2MjkzZTRhZWU0MWZhNGQ4NjQxZmM4NTE5NTUwM2Q5NjZhYzQwMmE0N2JhODViZmVmNDM2ZDIzYTdmNzVhZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3ZhbGlkYXRpb24ubWQiLAogICAgICAgICJkaWdlc3QiOiAiZjhkMDVjYjI2MzViYmZmMjY1YWE5MTIyZGNmMWQ1MTU2YmRlOTY2NGIyMDNmMmU2OWUwOTZkMmRiMTE3MmQ3MSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL2hvc3RlZF9zZWFyY2gucHkiLAogICAgICAgICJkaWdlc3QiOiAiYmU5ZDg2MmYxOTkyODg3ZDc2NmZlZjkxOGIzNzJiZTQ4YjIxZTMwYmM1N2Y2YjMyNTg1ZjYzMDYwNWNlMDQ4NSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjJiNjIzNDA4NDUwNjliZDVlODU2YWVjNTdjNTE0M2JlNDg3NzMwZWFjMmY4NTQ4ZjMxNDY2MzlmYjVkZDc2YzQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAidGVzdHMvdGVzdF9ob3N0ZWRfc2VhcmNoLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjBhZjdlNzJmYThhNzM0NDYxZjYwOGYwOTAxMDE5YzVmMzAyZTk4NDBhZGNhZTUzZjBmYjg3ZmNkM2I4NzEwNzUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDA1d1Dqn5+MMZGMIBovLfBRbvPE3aymwq6US0fzjN/gJdmTSH3DV2ad/hQEV2yL3cCMQD9rppSkerLdNWSgbUze1nde+7vu/jrZfVC/36Igr8a4aZLI2jIIQZrIy8FcVKxC8o=","keyid":""}]}} \ No newline at end of file From 9cab4d7a8c39c35612edbf1e08b256c75dfc5a25 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Thu, 1 Oct 2026 10:51:51 -0700 Subject: [PATCH 3/5] Fix MSA validation, output cleanup, and template support docs Signed-off-by: Ohad Mosafi --- .../msa-search-nim/references/validation.md | 11 +- .../msa-search-nim/scripts/hosted_search.py | 58 ++++++-- .../tests/test_hosted_search.py | 136 ++++++++++++++++++ .../msa-search-nim/references/validation.md | 11 +- .../msa-search-nim/scripts/hosted_search.py | 58 ++++++-- .../skills/msa-search-nim/skill-card.md | 2 +- .../tests/test_hosted_search.py | 136 ++++++++++++++++++ 7 files changed, 389 insertions(+), 23 deletions(-) diff --git a/nim-skills/msa-search-nim/references/validation.md b/nim-skills/msa-search-nim/references/validation.md index 82ef75f..0b42467 100644 --- a/nim-skills/msa-search-nim/references/validation.md +++ b/nim-skills/msa-search-nim/references/validation.md @@ -7,10 +7,17 @@ passing them downstream. - `alignments` exists for standard search. - `alignments_by_chain` exists for paired search. -- Each returned alignment has `alignment` text and a `format`. -- A3M/FASTA text starts with FASTA-style headers. +- Each returned alignment has `alignment` text and a matching `format` (`a3m` + for the hosted client's requested output). +- Each A3M/FASTA record has a nonempty FASTA-style header followed by sequence + data. Reject missing records, empty records, and invalid sequence characters. +- A3M records have equal numbers of match columns: uppercase residues and `-` + count toward the width; lowercase insertions do not. Wrapped sequence lines, + blank lines, and `#` comments are allowed. - Saved filenames include database and format so outputs do not overwrite each other. +- The hosted client publishes the output directory after every file is written; + a write failure removes temporary files so the same output path can be retried. - Record database names and e-value used for the search. ## Template Checks diff --git a/nim-skills/msa-search-nim/scripts/hosted_search.py b/nim-skills/msa-search-nim/scripts/hosted_search.py index 1d76e77..a3cddd4 100644 --- a/nim-skills/msa-search-nim/scripts/hosted_search.py +++ b/nim-skills/msa-search-nim/scripts/hosted_search.py @@ -13,6 +13,7 @@ import os from pathlib import Path import sys +import tempfile import time import requests @@ -93,11 +94,53 @@ def validate_alignments(result: object, databases: list[str]) -> None: for database in databases: formats = alignments.get(database) a3m = formats.get("a3m") if isinstance(formats, dict) else None - alignment = a3m.get("alignment") if isinstance(a3m, dict) else None - if not isinstance(alignment, str) or not alignment.lstrip().startswith(">"): + if not isinstance(a3m, dict) or a3m.get("format") != "a3m": + raise SearchError(f"Hosted MSA returned no A3M-formatted result for {database}.") + alignment = a3m.get("alignment") + if not isinstance(alignment, str): raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.") - if not any(line.strip() and not line.startswith((">", "#")) for line in alignment.splitlines()): - raise SearchError(f"Hosted MSA returned an empty A3M alignment for {database}.") + validate_a3m(alignment, database) + + +def validate_a3m(alignment: str, database: str) -> None: + """Require named, nonempty records with consistent A3M match columns. + + Uppercase residues and '-' occupy match columns; lowercase insertions + do not. Wrapped sequences, blank lines and '#' comments are supported. + """ + error = f"Hosted MSA returned a malformed A3M alignment for {database}." + lengths: list[int] = [] + for line in alignment.splitlines(): + if not line.strip() or line.startswith("#"): + continue + if line.startswith(">"): + if not line[1:].strip(): + raise SearchError(error) + lengths.append(0) + else: + if not lengths or any(c not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" for c in line): + raise SearchError(error) + lengths[-1] += sum("A" <= c <= "Z" or c == "-" for c in line) + if not lengths or not lengths[0] or any(length != lengths[0] for length in lengths): + raise SearchError(error) + + +def save_results(result: dict, databases: list[str], output_dir: Path) -> None: + """Publish the output directory only after all result files are written.""" + output_dir.parent.mkdir(parents=True, exist_ok=True) + # Stage on the same filesystem so the final directory rename is atomic. + # The temporary wrapper cleans up failed writes and failed publication. + with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging: + staged_output = Path(staging) / "output" + staged_output.mkdir() + (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + for database in dict.fromkeys(databases): + alignment = result["alignments"][database]["a3m"]["alignment"] + (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8") + # Recheck after the request and writes in case another run used this path. + if output_dir.exists() or output_dir.is_symlink(): + raise FileExistsError("Output directory already exists.") + staged_output.rename(output_dir) def main() -> int: @@ -106,20 +149,17 @@ def main() -> int: parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES)) parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request") args = parser.parse_args() - if args.output_dir.exists(): + if args.output_dir.exists() or args.output_dir.is_symlink(): parser.error("--output-dir must not already exist; use a new path for each request") try: result = search( args.sequence, args.databases, os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""), ) - # Create output only after all requested alignments have been validated. - args.output_dir.mkdir(parents=True, exist_ok=False) - (args.output_dir / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + save_results(result, args.databases, args.output_dir) for database in dict.fromkeys(args.databases): alignment = result["alignments"][database]["a3m"]["alignment"] path = args.output_dir / f"{database}.a3m" - path.write_text(alignment, encoding="utf-8") records = sum(line.startswith(">") for line in alignment.splitlines()) print(f"{path}: {records} sequences") except SearchError as exc: diff --git a/nim-skills/msa-search-nim/tests/test_hosted_search.py b/nim-skills/msa-search-nim/tests/test_hosted_search.py index 87a925c..a854a85 100644 --- a/nim-skills/msa-search-nim/tests/test_hosted_search.py +++ b/nim-skills/msa-search-nim/tests/test_hosted_search.py @@ -75,6 +75,63 @@ def test_partial_or_empty_alignment_is_not_success(self): with self.assertRaises(client.SearchError): client.search("ACDE", list(client.DATABASES), "test-credential") + def test_alignment_format_must_be_a3m(self): + for returned_format in [None, "fasta", "A3M", 1]: + result = alignments() + a3m = result["alignments"][client.DATABASES[0]]["a3m"] + if returned_format is None: + del a3m["format"] + else: + a3m["format"] = returned_format + with self.subTest(format=returned_format), \ + patch.object(client.requests, "post", return_value=response(data=result)) as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + + def test_malformed_a3m_is_not_success(self): + malformed = [ + None, + "", + "ACDE\n", + "ACDE\n>query\nACDE\n", + ">query\n>hit\nACDE\n", + ">query\nACDE\n>hit\n", + ">query\nACDE\n>empty\n>hit\nACDE\n", + "> \nACDE\n", + " >query\nACDE\n", + ">query\nAC DE\n", + ">query\nACD1\n", + ">query\nACD*\n", + ">query\nACDÉ\n", + ">query\nACDE\n>hit\nACD\n", + ">query\nacde\n", + ">query\n# no sequence\n", + ] + for alignment in malformed: + result = alignments() + result["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = alignment + reply = response(data=result) + with self.subTest(alignment=alignment), \ + patch.object(client.requests, "post", return_value=reply) as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + reply.close.assert_called_once() + + def test_valid_a3m_supports_wrapping_insertions_and_comments(self): + for alignment in [ + ">query\nACDE", + ">query description\nAC\nDE\n>hit\nAcC-\nE\n", + "# comment\n\n>query\nACDE\n\n>hit\nacACd-Efg\n# comment\n", + ]: + expected = alignments() + for database in client.DATABASES: + expected["alignments"][database]["a3m"]["alignment"] = alignment + with self.subTest(alignment=alignment), \ + patch.object(client.requests, "post", return_value=response(data=expected)): + self.assertEqual(client.search("ACDE", list(client.DATABASES), "test-credential"), expected) + def test_missing_key_and_unknown_database_fail_before_network(self): for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]: with self.subTest(databases=databases), patch.object(client.requests, "post") as post: @@ -104,6 +161,85 @@ def test_cli_failure_leaves_no_success_artifacts(self): self.assertEqual(client.main(), 1) self.assertFalse(output.exists()) + def test_cli_write_failure_cleans_up_and_allows_retry(self): + write_text = Path.write_text + filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)] + for failing_file in filenames: + with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + expected = alignments() + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def fail_during_write(path, data, **kwargs): + if path.name == failing_file: + write_text(path, "partial", **kwargs) + raise OSError("sensitive filesystem details") + return write_text(path, data, **kwargs) + + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=expected)): + with patch.object(Path, "write_text", fail_during_write), \ + redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + self.assertNotIn("sensitive", stderr.getvalue()) + with redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual(json.loads((output / "response.json").read_text()), expected) + for database in client.DATABASES: + self.assertEqual((output / f"{database}.a3m").read_text(), + expected["alignments"][database]["a3m"]["alignment"]) + + def test_cli_publish_failure_leaves_no_artifacts(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + patch.object(Path, "rename", side_effect=OSError("cannot publish")), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + + def test_cli_does_not_overwrite_output_created_during_search(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def concurrent_output(*args, **kwargs): + output.mkdir() + (output / "keep.txt").write_text("existing data") + return response(data=alignments()) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", side_effect=concurrent_output), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), [output]) + self.assertEqual(list(output.iterdir()), [output / "keep.txt"]) + self.assertEqual((output / "keep.txt").read_text(), "existing data") + self.assertEqual(stdout.getvalue(), "") + + def test_cli_malformed_response_leaves_no_output(self): + malformed = alignments() + malformed["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = ">query\nACDE\n>hit\n" + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=malformed)), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + if __name__ == "__main__": unittest.main() diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md index 82ef75f..0b42467 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md @@ -7,10 +7,17 @@ passing them downstream. - `alignments` exists for standard search. - `alignments_by_chain` exists for paired search. -- Each returned alignment has `alignment` text and a `format`. -- A3M/FASTA text starts with FASTA-style headers. +- Each returned alignment has `alignment` text and a matching `format` (`a3m` + for the hosted client's requested output). +- Each A3M/FASTA record has a nonempty FASTA-style header followed by sequence + data. Reject missing records, empty records, and invalid sequence characters. +- A3M records have equal numbers of match columns: uppercase residues and `-` + count toward the width; lowercase insertions do not. Wrapped sequence lines, + blank lines, and `#` comments are allowed. - Saved filenames include database and format so outputs do not overwrite each other. +- The hosted client publishes the output directory after every file is written; + a write failure removes temporary files so the same output path can be retried. - Record database names and e-value used for the search. ## Template Checks diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py index 1d76e77..a3cddd4 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py @@ -13,6 +13,7 @@ import os from pathlib import Path import sys +import tempfile import time import requests @@ -93,11 +94,53 @@ def validate_alignments(result: object, databases: list[str]) -> None: for database in databases: formats = alignments.get(database) a3m = formats.get("a3m") if isinstance(formats, dict) else None - alignment = a3m.get("alignment") if isinstance(a3m, dict) else None - if not isinstance(alignment, str) or not alignment.lstrip().startswith(">"): + if not isinstance(a3m, dict) or a3m.get("format") != "a3m": + raise SearchError(f"Hosted MSA returned no A3M-formatted result for {database}.") + alignment = a3m.get("alignment") + if not isinstance(alignment, str): raise SearchError(f"Hosted MSA returned no A3M alignment for {database}.") - if not any(line.strip() and not line.startswith((">", "#")) for line in alignment.splitlines()): - raise SearchError(f"Hosted MSA returned an empty A3M alignment for {database}.") + validate_a3m(alignment, database) + + +def validate_a3m(alignment: str, database: str) -> None: + """Require named, nonempty records with consistent A3M match columns. + + Uppercase residues and '-' occupy match columns; lowercase insertions + do not. Wrapped sequences, blank lines and '#' comments are supported. + """ + error = f"Hosted MSA returned a malformed A3M alignment for {database}." + lengths: list[int] = [] + for line in alignment.splitlines(): + if not line.strip() or line.startswith("#"): + continue + if line.startswith(">"): + if not line[1:].strip(): + raise SearchError(error) + lengths.append(0) + else: + if not lengths or any(c not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-" for c in line): + raise SearchError(error) + lengths[-1] += sum("A" <= c <= "Z" or c == "-" for c in line) + if not lengths or not lengths[0] or any(length != lengths[0] for length in lengths): + raise SearchError(error) + + +def save_results(result: dict, databases: list[str], output_dir: Path) -> None: + """Publish the output directory only after all result files are written.""" + output_dir.parent.mkdir(parents=True, exist_ok=True) + # Stage on the same filesystem so the final directory rename is atomic. + # The temporary wrapper cleans up failed writes and failed publication. + with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging: + staged_output = Path(staging) / "output" + staged_output.mkdir() + (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + for database in dict.fromkeys(databases): + alignment = result["alignments"][database]["a3m"]["alignment"] + (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8") + # Recheck after the request and writes in case another run used this path. + if output_dir.exists() or output_dir.is_symlink(): + raise FileExistsError("Output directory already exists.") + staged_output.rename(output_dir) def main() -> int: @@ -106,20 +149,17 @@ def main() -> int: parser.add_argument("--databases", nargs="+", choices=DATABASES, default=list(DATABASES)) parser.add_argument("--output-dir", type=Path, required=True, help="A new directory for this request") args = parser.parse_args() - if args.output_dir.exists(): + if args.output_dir.exists() or args.output_dir.is_symlink(): parser.error("--output-dir must not already exist; use a new path for each request") try: result = search( args.sequence, args.databases, os.environ.get("NGC_API_KEY") or os.environ.get("NVIDIA_API_KEY", ""), ) - # Create output only after all requested alignments have been validated. - args.output_dir.mkdir(parents=True, exist_ok=False) - (args.output_dir / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + save_results(result, args.databases, args.output_dir) for database in dict.fromkeys(args.databases): alignment = result["alignments"][database]["a3m"]["alignment"] path = args.output_dir / f"{database}.a3m" - path.write_text(alignment, encoding="utf-8") records = sum(line.startswith(">") for line in alignment.splitlines()) print(f"{path}: {records} sequences") except SearchError as exc: diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md index d17398c..4eae1c4 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache-2.0 AND CC-BY-4.0
## Use Case:
-Developers and engineers use this skill to generate multiple sequence alignments for protein sequences via GPU-accelerated MMSeqs2, supporting hosted NVIDIA API or local Docker NIM deployment for homolog search, paired MSA for complexes, and structural template retrieval.
+Developers and engineers use this skill to generate multiple sequence alignments for protein sequences via GPU-accelerated MMSeqs2. Hosted NVIDIA API and local Docker NIM support homolog search and paired MSA for complexes; structural template retrieval requires a local Docker NIM.
### Deployment Geography for Use:
Global
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py index 87a925c..a854a85 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py @@ -75,6 +75,63 @@ def test_partial_or_empty_alignment_is_not_success(self): with self.assertRaises(client.SearchError): client.search("ACDE", list(client.DATABASES), "test-credential") + def test_alignment_format_must_be_a3m(self): + for returned_format in [None, "fasta", "A3M", 1]: + result = alignments() + a3m = result["alignments"][client.DATABASES[0]]["a3m"] + if returned_format is None: + del a3m["format"] + else: + a3m["format"] = returned_format + with self.subTest(format=returned_format), \ + patch.object(client.requests, "post", return_value=response(data=result)) as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + + def test_malformed_a3m_is_not_success(self): + malformed = [ + None, + "", + "ACDE\n", + "ACDE\n>query\nACDE\n", + ">query\n>hit\nACDE\n", + ">query\nACDE\n>hit\n", + ">query\nACDE\n>empty\n>hit\nACDE\n", + "> \nACDE\n", + " >query\nACDE\n", + ">query\nAC DE\n", + ">query\nACD1\n", + ">query\nACD*\n", + ">query\nACDÉ\n", + ">query\nACDE\n>hit\nACD\n", + ">query\nacde\n", + ">query\n# no sequence\n", + ] + for alignment in malformed: + result = alignments() + result["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = alignment + reply = response(data=result) + with self.subTest(alignment=alignment), \ + patch.object(client.requests, "post", return_value=reply) as post: + with self.assertRaises(client.SearchError): + client.search("ACDE", list(client.DATABASES), "test-credential") + self.assertEqual(post.call_count, 1) + reply.close.assert_called_once() + + def test_valid_a3m_supports_wrapping_insertions_and_comments(self): + for alignment in [ + ">query\nACDE", + ">query description\nAC\nDE\n>hit\nAcC-\nE\n", + "# comment\n\n>query\nACDE\n\n>hit\nacACd-Efg\n# comment\n", + ]: + expected = alignments() + for database in client.DATABASES: + expected["alignments"][database]["a3m"]["alignment"] = alignment + with self.subTest(alignment=alignment), \ + patch.object(client.requests, "post", return_value=response(data=expected)): + self.assertEqual(client.search("ACDE", list(client.DATABASES), "test-credential"), expected) + def test_missing_key_and_unknown_database_fail_before_network(self): for databases, key in [(list(client.DATABASES), ""), (["../result"], "test-credential")]: with self.subTest(databases=databases), patch.object(client.requests, "post") as post: @@ -104,6 +161,85 @@ def test_cli_failure_leaves_no_success_artifacts(self): self.assertEqual(client.main(), 1) self.assertFalse(output.exists()) + def test_cli_write_failure_cleans_up_and_allows_retry(self): + write_text = Path.write_text + filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)] + for failing_file in filenames: + with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + expected = alignments() + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def fail_during_write(path, data, **kwargs): + if path.name == failing_file: + write_text(path, "partial", **kwargs) + raise OSError("sensitive filesystem details") + return write_text(path, data, **kwargs) + + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=expected)): + with patch.object(Path, "write_text", fail_during_write), \ + redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + self.assertNotIn("sensitive", stderr.getvalue()) + with redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual(json.loads((output / "response.json").read_text()), expected) + for database in client.DATABASES: + self.assertEqual((output / f"{database}.a3m").read_text(), + expected["alignments"][database]["a3m"]["alignment"]) + + def test_cli_publish_failure_leaves_no_artifacts(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + patch.object(Path, "rename", side_effect=OSError("cannot publish")), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + + def test_cli_does_not_overwrite_output_created_during_search(self): + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def concurrent_output(*args, **kwargs): + output.mkdir() + (output / "keep.txt").write_text("existing data") + return response(data=alignments()) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", side_effect=concurrent_output), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), [output]) + self.assertEqual(list(output.iterdir()), [output / "keep.txt"]) + self.assertEqual((output / "keep.txt").read_text(), "existing data") + self.assertEqual(stdout.getvalue(), "") + + def test_cli_malformed_response_leaves_no_output(self): + malformed = alignments() + malformed["alignments"][client.DATABASES[1]]["a3m"]["alignment"] = ">query\nACDE\n>hit\n" + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=malformed)), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + if __name__ == "__main__": unittest.main() From c7d07bb0670e1c3ff51afc6f00b7c7e67bcc5850 Mon Sep 17 00:00:00 2001 From: Ohad Mosafi Date: Thu, 1 Oct 2026 11:21:25 -0700 Subject: [PATCH 4/5] Fix MSA output permissions and concurrent directory overwrite race Signed-off-by: Ohad Mosafi --- .../msa-search-nim/references/validation.md | 10 +- .../msa-search-nim/scripts/hosted_search.py | 26 ++++-- .../tests/test_hosted_search.py | 91 ++++++++++++++++--- .../msa-search-nim/references/validation.md | 10 +- .../msa-search-nim/scripts/hosted_search.py | 26 ++++-- .../tests/test_hosted_search.py | 91 ++++++++++++++++--- 6 files changed, 210 insertions(+), 44 deletions(-) diff --git a/nim-skills/msa-search-nim/references/validation.md b/nim-skills/msa-search-nim/references/validation.md index 0b42467..06d62b0 100644 --- a/nim-skills/msa-search-nim/references/validation.md +++ b/nim-skills/msa-search-nim/references/validation.md @@ -16,8 +16,14 @@ passing them downstream. blank lines, and `#` comments are allowed. - Saved filenames include database and format so outputs do not overwrite each other. -- The hosted client publishes the output directory after every file is written; - a write failure removes temporary files so the same output path can be retried. +- The hosted client finishes all writes in private staging, then exclusively + creates the output directory and moves the result files into it. An existing + output path, including an empty directory created by another run, is preserved. +- On POSIX, the output directory uses owner-only permissions (`0700`), and result + files use `0600`, even with a permissive umask. +- A failed write or move removes temporary files and any output directory created + by this run, so the same output path can be retried. Treat output as complete + only after the client exits successfully. - Record database names and e-value used for the search. ## Template Checks diff --git a/nim-skills/msa-search-nim/scripts/hosted_search.py b/nim-skills/msa-search-nim/scripts/hosted_search.py index a3cddd4..d4c733c 100644 --- a/nim-skills/msa-search-nim/scripts/hosted_search.py +++ b/nim-skills/msa-search-nim/scripts/hosted_search.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shutil import sys import tempfile import time @@ -126,21 +127,28 @@ def validate_a3m(alignment: str, database: str) -> None: def save_results(result: dict, databases: list[str], output_dir: Path) -> None: - """Publish the output directory only after all result files are written.""" + """Save private results in a new directory, cleaning up failed publication.""" output_dir.parent.mkdir(parents=True, exist_ok=True) - # Stage on the same filesystem so the final directory rename is atomic. - # The temporary wrapper cleans up failed writes and failed publication. + # Finish all writes inside private staging on the destination filesystem. with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging: - staged_output = Path(staging) / "output" - staged_output.mkdir() + staged_output = Path(staging) (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") for database in dict.fromkeys(databases): alignment = result["alignments"][database]["a3m"]["alignment"] (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8") - # Recheck after the request and writes in case another run used this path. - if output_dir.exists() or output_dir.is_symlink(): - raise FileExistsError("Output directory already exists.") - staged_output.rename(output_dir) + files = list(staged_output.iterdir()) + for path in files: + path.chmod(0o600) + # mkdir exclusively reserves the path, including against empty + # directories and dangling symlinks created by a concurrent run. + output_dir.mkdir(mode=0o700) + try: + for path in files: + path.rename(output_dir / path.name) + except BaseException: + # Only remove a directory that this run successfully reserved. + shutil.rmtree(output_dir) + raise def main() -> int: diff --git a/nim-skills/msa-search-nim/tests/test_hosted_search.py b/nim-skills/msa-search-nim/tests/test_hosted_search.py index a854a85..062ba75 100644 --- a/nim-skills/msa-search-nim/tests/test_hosted_search.py +++ b/nim-skills/msa-search-nim/tests/test_hosted_search.py @@ -5,6 +5,7 @@ import io import json from pathlib import Path +import stat import tempfile import unittest from unittest.mock import Mock, patch @@ -151,6 +152,62 @@ def test_cli_writes_actual_response_and_alignments(self): for database in client.DATABASES: self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"]) + @unittest.skipUnless(client.os.name == "posix", "Requires POSIX permissions") + def test_cli_output_is_private_with_permissive_umask(self): + with tempfile.TemporaryDirectory() as root: + Path(root).chmod(0o755) + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + previous_umask = client.os.umask(0) + try: + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + finally: + client.os.umask(previous_umask) + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700) + for path in output.iterdir(): + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600, path.name) + + def test_cli_does_not_replace_concurrent_empty_output_directory(self): + mkdir, rename = Path.mkdir, Path.rename + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + competing_stat = None + + def reserve_output(): + nonlocal competing_stat + mkdir(output, mode=0o700) + competing_stat = output.stat() + + # Simulate a competing reservation immediately before publication, + # after any existence check, with either directory operation. + def racing_mkdir(path, *args, **kwargs): + if path == output: + reserve_output() + return mkdir(path, *args, **kwargs) + + def racing_rename(path, target): + if Path(target) == output: + reserve_output() + return rename(path, target) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + patch.object(Path, "mkdir", racing_mkdir), \ + patch.object(Path, "rename", racing_rename), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertIsNotNone(competing_stat) + self.assertEqual(output.stat().st_ino, competing_stat.st_ino) + self.assertEqual(list(output.iterdir()), []) + self.assertEqual(list(Path(root).iterdir()), [output]) + self.assertEqual(stdout.getvalue(), "") + def test_cli_failure_leaves_no_success_artifacts(self): with tempfile.TemporaryDirectory() as root: output = Path(root) / "msa" @@ -194,17 +251,29 @@ def fail_during_write(path, data, **kwargs): expected["alignments"][database]["a3m"]["alignment"]) def test_cli_publish_failure_leaves_no_artifacts(self): - with tempfile.TemporaryDirectory() as root: - output = Path(root) / "msa" - args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] - with patch.object(client.sys, "argv", args), \ - patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ - patch.object(client.requests, "post", return_value=response(data=alignments())), \ - patch.object(Path, "rename", side_effect=OSError("cannot publish")), \ - redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): - self.assertEqual(client.main(), 1) - self.assertEqual(list(Path(root).iterdir()), []) - self.assertEqual(stdout.getvalue(), "") + rename = Path.rename + filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)] + for failing_file in filenames: + with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def fail_during_publish(path, target): + if path.name == failing_file: + raise OSError("cannot publish") + return rename(path, target) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())): + with patch.object(Path, "rename", fail_during_publish), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + with redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual({path.name for path in output.iterdir()}, set(filenames)) def test_cli_does_not_overwrite_output_created_during_search(self): with tempfile.TemporaryDirectory() as root: diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md index 0b42467..06d62b0 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/references/validation.md @@ -16,8 +16,14 @@ passing them downstream. blank lines, and `#` comments are allowed. - Saved filenames include database and format so outputs do not overwrite each other. -- The hosted client publishes the output directory after every file is written; - a write failure removes temporary files so the same output path can be retried. +- The hosted client finishes all writes in private staging, then exclusively + creates the output directory and moves the result files into it. An existing + output path, including an empty directory created by another run, is preserved. +- On POSIX, the output directory uses owner-only permissions (`0700`), and result + files use `0600`, even with a permissive umask. +- A failed write or move removes temporary files and any output directory created + by this run, so the same output path can be retried. Treat output as complete + only after the client exits successfully. - Record database names and e-value used for the search. ## Template Checks diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py index a3cddd4..d4c733c 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/scripts/hosted_search.py @@ -12,6 +12,7 @@ import json import os from pathlib import Path +import shutil import sys import tempfile import time @@ -126,21 +127,28 @@ def validate_a3m(alignment: str, database: str) -> None: def save_results(result: dict, databases: list[str], output_dir: Path) -> None: - """Publish the output directory only after all result files are written.""" + """Save private results in a new directory, cleaning up failed publication.""" output_dir.parent.mkdir(parents=True, exist_ok=True) - # Stage on the same filesystem so the final directory rename is atomic. - # The temporary wrapper cleans up failed writes and failed publication. + # Finish all writes inside private staging on the destination filesystem. with tempfile.TemporaryDirectory(prefix=f".{output_dir.name}-", dir=output_dir.parent) as staging: - staged_output = Path(staging) / "output" - staged_output.mkdir() + staged_output = Path(staging) (staged_output / "response.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") for database in dict.fromkeys(databases): alignment = result["alignments"][database]["a3m"]["alignment"] (staged_output / f"{database}.a3m").write_text(alignment, encoding="utf-8") - # Recheck after the request and writes in case another run used this path. - if output_dir.exists() or output_dir.is_symlink(): - raise FileExistsError("Output directory already exists.") - staged_output.rename(output_dir) + files = list(staged_output.iterdir()) + for path in files: + path.chmod(0o600) + # mkdir exclusively reserves the path, including against empty + # directories and dangling symlinks created by a concurrent run. + output_dir.mkdir(mode=0o700) + try: + for path in files: + path.rename(output_dir / path.name) + except BaseException: + # Only remove a directory that this run successfully reserved. + shutil.rmtree(output_dir) + raise def main() -> int: diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py index a854a85..062ba75 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/tests/test_hosted_search.py @@ -5,6 +5,7 @@ import io import json from pathlib import Path +import stat import tempfile import unittest from unittest.mock import Mock, patch @@ -151,6 +152,62 @@ def test_cli_writes_actual_response_and_alignments(self): for database in client.DATABASES: self.assertEqual((output / f"{database}.a3m").read_text(), expected["alignments"][database]["a3m"]["alignment"]) + @unittest.skipUnless(client.os.name == "posix", "Requires POSIX permissions") + def test_cli_output_is_private_with_permissive_umask(self): + with tempfile.TemporaryDirectory() as root: + Path(root).chmod(0o755) + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + previous_umask = client.os.umask(0) + try: + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + finally: + client.os.umask(previous_umask) + self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700) + for path in output.iterdir(): + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600, path.name) + + def test_cli_does_not_replace_concurrent_empty_output_directory(self): + mkdir, rename = Path.mkdir, Path.rename + with tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + competing_stat = None + + def reserve_output(): + nonlocal competing_stat + mkdir(output, mode=0o700) + competing_stat = output.stat() + + # Simulate a competing reservation immediately before publication, + # after any existence check, with either directory operation. + def racing_mkdir(path, *args, **kwargs): + if path == output: + reserve_output() + return mkdir(path, *args, **kwargs) + + def racing_rename(path, target): + if Path(target) == output: + reserve_output() + return rename(path, target) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())), \ + patch.object(Path, "mkdir", racing_mkdir), \ + patch.object(Path, "rename", racing_rename), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertIsNotNone(competing_stat) + self.assertEqual(output.stat().st_ino, competing_stat.st_ino) + self.assertEqual(list(output.iterdir()), []) + self.assertEqual(list(Path(root).iterdir()), [output]) + self.assertEqual(stdout.getvalue(), "") + def test_cli_failure_leaves_no_success_artifacts(self): with tempfile.TemporaryDirectory() as root: output = Path(root) / "msa" @@ -194,17 +251,29 @@ def fail_during_write(path, data, **kwargs): expected["alignments"][database]["a3m"]["alignment"]) def test_cli_publish_failure_leaves_no_artifacts(self): - with tempfile.TemporaryDirectory() as root: - output = Path(root) / "msa" - args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] - with patch.object(client.sys, "argv", args), \ - patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ - patch.object(client.requests, "post", return_value=response(data=alignments())), \ - patch.object(Path, "rename", side_effect=OSError("cannot publish")), \ - redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): - self.assertEqual(client.main(), 1) - self.assertEqual(list(Path(root).iterdir()), []) - self.assertEqual(stdout.getvalue(), "") + rename = Path.rename + filenames = ["response.json", *(f"{database}.a3m" for database in client.DATABASES)] + for failing_file in filenames: + with self.subTest(file=failing_file), tempfile.TemporaryDirectory() as root: + output = Path(root) / "msa" + args = ["hosted_search.py", "--sequence", "ACDE", "--output-dir", str(output)] + + def fail_during_publish(path, target): + if path.name == failing_file: + raise OSError("cannot publish") + return rename(path, target) + + with patch.object(client.sys, "argv", args), \ + patch.dict(client.os.environ, {"NGC_API_KEY": "test-credential"}), \ + patch.object(client.requests, "post", return_value=response(data=alignments())): + with patch.object(Path, "rename", fail_during_publish), \ + redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(client.main(), 1) + self.assertEqual(list(Path(root).iterdir()), []) + self.assertEqual(stdout.getvalue(), "") + with redirect_stdout(io.StringIO()): + self.assertEqual(client.main(), 0) + self.assertEqual({path.name for path in output.iterdir()}, set(filenames)) def test_cli_does_not_overwrite_output_created_during_search(self): with tempfile.TemporaryDirectory() as root: From 06a9be9db1920ab8525b8807b92fb4da137c864e Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 1 Oct 2026 18:45:23 +0000 Subject: [PATCH 5/5] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- .../skills/msa-search-nim/BENCHMARK.md | 30 +++++++-------- .../skills/msa-search-nim/skill-card.md | 38 +++++++++---------- .../skills/msa-search-nim/skill.oms.sig | 2 +- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md index ffb2f26..6588795 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/BENCHMARK.md @@ -9,12 +9,12 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `msa-search-nim` -- Evaluation date: 2026-09-30 +- Evaluation date: 2026-10-01 - Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 1 evaluation tasks (1 positive) - Dataset digest: `sha256:da6512955763a8e542da5599461202e0804188cd0eb5cd50377924e40f71d403` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 3 +- Attempts per task: 1 - Environment: `k8s-sandbox` - Tier 2 evidence: required for publication - Tier 3 evidence: required for publication @@ -35,12 +35,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 73.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 100.0% (+50.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 0.0% (-100.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 67.9% → 100.0% (+32.1 points) | 100.0% → 95.0% (-5.0 points) | -| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 94.6% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 50.0% → 100.0% (+50.0 points) | 50.0% → 100.0% (+50.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 90.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 95.0% → 100.0% (+5.0 points) | 92.9% → 100.0% (+7.1 points) | +| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 83.1% — baseline ran, but no comparable score was available; uplift unavailable | **How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. @@ -52,11 +52,11 @@ Actual Tier 3 execution usage is reported for every observed agent/case pair and | Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | |---|---|---:|---:|---:|---:|---| -| claude-code | All cases | 486,740 | 1,135,884 | -649,144 | -57.15% | skill 1/1; base 1/1 | -| claude-code | 1 | 486,740 | 1,135,884 | -649,144 | -57.15% | skill 1/1; base 1/1 | -| codex | All cases | 205,561 | 232,346 | -26,785 | -11.53% | skill 1/1; base 1/1 | -| codex | 1 | 205,561 | 232,346 | -26,785 | -11.53% | skill 1/1; base 1/1 | -| ALL AGENTS | Dataset aggregate | 692,301 | 1,368,230 | -675,929 | -49.40% | skill 2/2; base 2/2 | +| claude-code | All cases | 443,342 | 647,113 | -203,771 | -31.49% | skill 1/1; base 1/1 | +| claude-code | 1 | 443,342 | 647,113 | -203,771 | -31.49% | skill 1/1; base 1/1 | +| codex | All cases | 186,870 | 387,556 | -200,686 | -51.78% | skill 1/1; base 1/1 | +| codex | 1 | 186,870 | 387,556 | -200,686 | -51.78% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 630,212 | 1,034,669 | -404,457 | -39.09% | skill 2/2; base 2/2 | Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. @@ -64,7 +64,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c | Tier | Purpose | Status | Evidence | |---|---|---|---| -| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 38 finding(s) | +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 11 validator(s); 39 finding(s) | | Tier 2 | Semantic deduplication | **PASSED** | 2 validator(s); 0 finding(s) | | Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 1 task(s) | @@ -78,7 +78,7 @@ Prompt tokens include cached reads, so total tokens are `prompt + completion` (c - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.author' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) - **MEDIUM** QUALITY/quality_correctness: SKILL_SPEC recommended field missing: 'metadata.tags' (`skills/bionemo-agent-toolkit/skills/msa-search-nim/SKILL.md`) - **MEDIUM** SCHEMA/folder_hierarchy: Unexpected nesting depth for general skill (`skills/bionemo-agent-toolkit/skills/msa-search-nim`) -- 33 additional finding(s) are available in the full evaluation artifacts. +- 34 additional finding(s) are available in the full evaluation artifacts. diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md index 4eae1c4..88f7d61 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache-2.0 AND CC-BY-4.0
## Use Case:
-Developers and engineers use this skill to generate multiple sequence alignments for protein sequences via GPU-accelerated MMSeqs2. Hosted NVIDIA API and local Docker NIM support homolog search and paired MSA for complexes; structural template retrieval requires a local Docker NIM.
+Developers and life science researchers who need to generate multiple sequence alignments for protein sequences, perform homolog searches, paired MSA searches for protein complexes, or retrieve PDB70 structural templates via hosted NVIDIA API or local Docker deployment.
### Deployment Geography for Use:
Global
@@ -33,8 +33,8 @@ Mitigation: Review and scan skill before deployment.
## Skill Output:
-**Output Type(s):** [API Calls, Shell commands, Files]
-**Output Format:** [JSON responses, A3M alignment files, mmCIF structure files]
+**Output Type(s):** [API Calls, Files, Shell commands]
+**Output Format:** [JSON API responses, A3M alignment files, mmCIF structure files]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
@@ -45,36 +45,36 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-Evaluated against 1 task (1 positive case, 3 attempts per task) in isolated k8s-sandbox pods.
+1 evaluation task (1 positive), each attempt ran in its own isolated sandbox pod. Dataset digest: sha256:da6512955763a8e542da5599461202e0804188cd0eb5cd50377924e40f71d403.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was selected and the workflow executed.
-- Effectiveness: Checks goal completion (50%) and expected workflow adherence (50%).
-- Efficiency: Checks tool-call productivity (50%) and token efficiency (50%).
+- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the final answer is correct against the reference answer.
+- Discoverability: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- Effectiveness: Whether the skill helped complete the user's goal (50% goal completion + 50% expected workflow adherence).
+- Efficiency: Whether the skill avoided wasted tool calls and token usage (50% tool-call productivity + 50% token efficiency).
Underlying evaluation signals used in this run:
-- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
-- `accuracy`: Final-answer correctness against the reference answer.
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity (routing scored under Discoverability).
+- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
-- `skill_efficiency`: Tool-call productivity.
-- `token_efficiency`: Actual uncached prompt plus completion token usage.
+- `token_efficiency`: Actual uncached prompt plus completion usage.
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 95.5% — baseline ran, but no comparable score was available; uplift unavailable | 73.8% — baseline ran, but no comparable score was available; uplift unavailable | -| Security | 50.0% → 100.0% (+50.0 points) | 100.0% → 100.0% (±0.0 points) | -| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 0.0% (-100.0 points) | -| Discoverability | 100.0% — baseline ran, but no comparable score was available; uplift unavailable | 95.0% — baseline ran, but no comparable score was available; uplift unavailable | -| Effectiveness | 67.9% → 100.0% (+32.1 points) | 100.0% → 95.0% (-5.0 points) | -| Efficiency | 77.4% — baseline ran, but no comparable score was available; uplift unavailable | 78.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Overall | 95.5% | 94.6% | +| Security | 50.0% → 100.0% (+50.0 points) | 50.0% → 100.0% (+50.0 points) | +| Correctness | 100.0% → 100.0% (±0.0 points) | 100.0% → 100.0% (±0.0 points) | +| Discoverability | 100.0% | 90.0% | +| Effectiveness | 95.0% → 100.0% (+5.0 points) | 92.9% → 100.0% (+7.1 points) | +| Efficiency | 77.4% | 83.1% | ## Skill Version(s):
0.1.0 (source: pyproject.toml)
diff --git a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig index 95e7ac5..e5cd204 100644 --- a/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig +++ b/skills/bionemo-agent-toolkit/skills/msa-search-nim/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibXNhLXNlYXJjaC1uaW0iLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMDNmODJkYWMxYWY2M2RlMThjNTY4MDdjYjc3MDM1NjhiMTk5YzQ3YTk5Zjg5ZGE4MDZkM2Y4ZGZmN2Q5OTFkMSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2VjNjczMWEyODNlN2FkYzcxNGVjNGJkMWNlYjE1OTgxYWMzNWMzNTBkYTU1M2Y2NDhlNTdlZWIyY2Y4ZDZjMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICIzZDQyYzUxMzZjNWRhMWJjZTg4NjJlNjdmZWEyNTc0MDk1OGM3ZGFiYTFmMjdhMWM2MTI2OTVhYzRkZDAzMDYyIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImNvbmZpZy9za2lsbHNwZWN0b3ItYmFzZWxpbmUueW1sIiwKICAgICAgICAiZGlnZXN0IjogImRhMmQxMjMzNzU1YzZhNzZkNjNhNmFkZDY3ZjBjYmNiZGI2NjY0NzBjNzc4MjE0ZGIxMWNhODdiZThiYmQ0MWYiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvY29uZmlnLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICJiYTJiOGNmMGVhZDEzYmZiNjVkODEzMDllMTYzMTYyZGJjMmM2YzUwNTg5NjRjNGFlYTQxNGUwMDcwODk4YThhIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiOTQwMzlkNmU0ODY3MTljYWFiZjU2YjNkM2I3ZWRjYzk1ZTcyNDhlZjI4NjQwYTM5M2IzZTg2ODA1NTQxYjBhNyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy90cmlnZ2VyX2V2YWxzLmpzb24iLAogICAgICAgICJkaWdlc3QiOiAiMWY3YzUyMWY2OThhYTBiMDA3NmIxNDc2MjI2NWFiZjU3MDAzYWJkN2ZjNzdhM2Y5ZGUyZWY5NjExMWU1M2Y4OCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FwaS5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0ZDIwYTMwMDEwNDdjZDY3NjQ1YTYxNDViMTI4YmYwZTFkODJkZmViZDA0ZmM0YzU0ZmY4NmRmOWQ0NWQ0MTg3IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZXhhbXBsZXMubWQiLAogICAgICAgICJkaWdlc3QiOiAiMzgxNmEyNDcyZmU4ODI4OGM2M2I1NzgxMmQ0N2ZmOGIxNDNmZTI1MWQxZmYyM2MwNjVhODhkOTkyM2JhMzYzYSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3BhcmFtZXRlcnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiMDI1ZDQwMGY3ZWM5Nzg4YWE0YTFiMjk4N2QwOWRkM2I0ZTdkN2RkMTVjMmU4MWM2MTQyYjI3NjNjYmZlYTViNCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3NjaWVuY2UubWQiLAogICAgICAgICJkaWdlc3QiOiAiZTA2MjkzZTRhZWU0MWZhNGQ4NjQxZmM4NTE5NTUwM2Q5NjZhYzQwMmE0N2JhODViZmVmNDM2ZDIzYTdmNzVhZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3ZhbGlkYXRpb24ubWQiLAogICAgICAgICJkaWdlc3QiOiAiZjhkMDVjYjI2MzViYmZmMjY1YWE5MTIyZGNmMWQ1MTU2YmRlOTY2NGIyMDNmMmU2OWUwOTZkMmRiMTE3MmQ3MSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL2hvc3RlZF9zZWFyY2gucHkiLAogICAgICAgICJkaWdlc3QiOiAiYmU5ZDg2MmYxOTkyODg3ZDc2NmZlZjkxOGIzNzJiZTQ4YjIxZTMwYmM1N2Y2YjMyNTg1ZjYzMDYwNWNlMDQ4NSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjJiNjIzNDA4NDUwNjliZDVlODU2YWVjNTdjNTE0M2JlNDg3NzMwZWFjMmY4NTQ4ZjMxNDY2MzlmYjVkZDc2YzQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAidGVzdHMvdGVzdF9ob3N0ZWRfc2VhcmNoLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjBhZjdlNzJmYThhNzM0NDYxZjYwOGYwOTAxMDE5YzVmMzAyZTk4NDBhZGNhZTUzZjBmYjg3ZmNkM2I4NzEwNzUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDA1d1Dqn5+MMZGMIBovLfBRbvPE3aymwq6US0fzjN/gJdmTSH3DV2ad/hQEV2yL3cCMQD9rppSkerLdNWSgbUze1nde+7vu/jrZfVC/36Igr8a4aZLI2jIIQZrIy8FcVKxC8o=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibXNhLXNlYXJjaC1uaW0iLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiY2ZlOTc0MTgzZjg0OThhZTNjNmI3MjNiOTAzMzljOTk5NzEwM2ZlZjlhM2U4MjVjYmYzZWRjOTc1YjRkNWU1YiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjRhZmY5N2Q0YjA2NGJlYzExMTM2MjA4NDhkZjdkM2E0MjI5ZDYyNTI2YjA2NzdjZmYzYmZlOWJmMjk0OTVjNTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiM2Q0MmM1MTM2YzVkYTFiY2U4ODYyZTY3ZmVhMjU3NDA5NThjN2RhYmExZjI3YTFjNjEyNjk1YWM0ZGQwMzA2MiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJjb25maWcvc2tpbGxzcGVjdG9yLWJhc2VsaW5lLnltbCIsCiAgICAgICAgImRpZ2VzdCI6ICJkYTJkMTIzMzc1NWM2YTc2ZDYzYTZhZGQ2N2YwY2JjYmRiNjY2NDcwYzc3ODIxNGRiMTFjYTg3YmU4YmJkNDFmIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2NvbmZpZy55bWwiLAogICAgICAgICJkaWdlc3QiOiAiYmEyYjhjZjBlYWQxM2JmYjY1ZDgxMzA5ZTE2MzE2MmRiYzJjNmM1MDU4OTY0YzRhZWE0MTRlMDA3MDg5OGE4YSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjk0MDM5ZDZlNDg2NzE5Y2FhYmY1NmIzZDNiN2VkY2M5NWU3MjQ4ZWYyODY0MGEzOTNiM2U4NjgwNTU0MWIwYTciCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvdHJpZ2dlcl9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogIjFmN2M1MjFmNjk4YWEwYjAwNzZiMTQ3NjIyNjVhYmY1NzAwM2FiZDdmYzc3YTNmOWRlMmVmOTYxMTFlNTNmODgiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcGkubWQiLAogICAgICAgICJkaWdlc3QiOiAiNGQyMGEzMDAxMDQ3Y2Q2NzY0NWE2MTQ1YjEyOGJmMGUxZDgyZGZlYmQwNGZjNGM1NGZmODZkZjlkNDVkNDE4NyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2V4YW1wbGVzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjM4MTZhMjQ3MmZlODgyODhjNjNiNTc4MTJkNDdmZjhiMTQzZmUyNTFkMWZmMjNjMDY1YTg4ZDk5MjNiYTM2M2EiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9wYXJhbWV0ZXJzLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjAyNWQ0MDBmN2VjOTc4OGFhNGExYjI5ODdkMDlkZDNiNGU3ZDdkZDE1YzJlODFjNjE0MmIyNzYzY2JmZWE1YjQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9zY2llbmNlLm1kIiwKICAgICAgICAiZGlnZXN0IjogImUwNjI5M2U0YWVlNDFmYTRkODY0MWZjODUxOTU1MDNkOTY2YWM0MDJhNDdiYTg1YmZlZjQzNmQyM2E3Zjc1YWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy92YWxpZGF0aW9uLm1kIiwKICAgICAgICAiZGlnZXN0IjogImJkNTRhZDBkZWZhNjc3NjU0MmEwNmRiODI1ODI3YTI2MDBiMzc2YjEzYTZkZTM1NjYzNDRhMDdjMjgwNGY1NTUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9ob3N0ZWRfc2VhcmNoLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjI1MWQ0YjU1Yzg4NDUzODFlODdlYjljMGI1NTE4MWU4YTRiNDQzOWVlYmM5OGNmNTlmNmQ3ODI5ZWQzYTY1ZDEiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI0MDQ1OGUwOTdjNzA0NWVhMDc1Mjg5MTYxMDZiZGU4YzhjNWJhMDYxZGZmMTlkZDYyYzQ3OGY4ZTI1NTI0ZGM1IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInRlc3RzL3Rlc3RfaG9zdGVkX3NlYXJjaC5weSIsCiAgICAgICAgImRpZ2VzdCI6ICIwNjg3MTlmY2I3ZWNmM2JiYmE5YWExNjcxNjYxMDRjM2QwNTc4ZWEwN2ZkZDU5YmVjNjljNzA2NTJkOGQyYTY2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQCDxUUYSuZAkoZM9uKenY5yLoZp4hweNWKW1DGhsf97nrFzKH+3vb2jM/wsFRDDs2gCMCoTUNVyCRu4RpTxt5EnK1hY+Tj+ksCsti9f5LWZcExyS9hMZqPNm6+7ROyhcKkHYQ==","keyid":""}]}} \ No newline at end of file