diff --git a/.gitignore b/.gitignore index 85bc7087..7878bb58 100644 --- a/.gitignore +++ b/.gitignore @@ -90,7 +90,7 @@ config.local.mk *.log # Curation / audit pipeline outputs — these are regeneratable artifacts -# produced by scripts/curate_evidence_with_pdfs.py and the +# produced by scripts/curate_evidence_with_pdfs.py (removed in #410) and the # `audit-network-report` justfile recipe; downstream scripts read them # within a workflow run, but they should not live in version control. evidence_curation_report.txt diff --git a/docs/AUTOMATION_TOOLS.md b/docs/AUTOMATION_TOOLS.md index 180ea028..4f36ef22 100644 --- a/docs/AUTOMATION_TOOLS.md +++ b/docs/AUTOMATION_TOOLS.md @@ -198,10 +198,11 @@ uv run python scripts/intelligent_snippet_fixer.py --file FILENAME.yaml # 2. Review suggestions carefully, apply best ones # 3. Validate -# ⚠️ NOT FUNCTIONAL — this script imports communitymech.literature_enhanced, -# which has never existed, so it fails at import. Tracked in #410. -# (The command below is also stale: this repo uses uv, not poetry.) -uv run python scripts/curate_evidence_with_pdfs.py --file FILENAME.yaml +# ⚠️ REMOVED in #410. curate_evidence_with_pdfs.py never worked — it imported +# communitymech.literature_enhanced, a module absent from every commit — and the +# file itself is now gone, so this is what to run instead: +uv run python scripts/cache_fulltext.py PMID:12345 # only if snippets need full text +just validate-references kb/communities/FILENAME.yaml # 4. Schema check just validate kb/communities/FILENAME.yaml diff --git a/docs/CURATION_PROGRESS_REPORT.md b/docs/CURATION_PROGRESS_REPORT.md index 3a2483ac..9d5b5aff 100644 --- a/docs/CURATION_PROGRESS_REPORT.md +++ b/docs/CURATION_PROGRESS_REPORT.md @@ -259,7 +259,7 @@ poetry run python scripts/batch_snippet_fixer.py --phase 1 --auto-approve 1. **Validation**: ```bash - # NOT FUNCTIONAL (#410): scripts/curate_evidence_with_pdfs.py --quick + # REMOVED in #410 (use cache_fulltext.py + just validate-references): scripts/curate_evidence_with_pdfs.py --quick just validate-all ``` @@ -313,10 +313,10 @@ poetry run python scripts/intelligent_snippet_fixer.py --file FILENAME.yaml --au poetry run python scripts/batch_snippet_fixer.py --phase 1 --auto-approve # Validate file -# NOT FUNCTIONAL (#410): scripts/curate_evidence_with_pdfs.py --file FILENAME.yaml +# REMOVED in #410 (use cache_fulltext.py + just validate-references): scripts/curate_evidence_with_pdfs.py --file FILENAME.yaml # Validate all files (quick) -# NOT FUNCTIONAL (#410): scripts/curate_evidence_with_pdfs.py --quick +# REMOVED in #410 (use cache_fulltext.py + just validate-references): scripts/curate_evidence_with_pdfs.py --quick ``` --- diff --git a/docs/pdf_fetching_capability.md b/docs/pdf_fetching_capability.md index 808e7059..3f5ab430 100644 --- a/docs/pdf_fetching_capability.md +++ b/docs/pdf_fetching_capability.md @@ -6,17 +6,35 @@ The CommunityMech literature system integrates a 6-tier cascading PDF discovery ## Integration Status -⚠️ **NOT FUNCTIONAL** — the scripts described below cannot run. +⚠️ **REMOVED** — the scripts described below no longer exist, and never worked. All of them begin `from communitymech.literature_enhanced import EnhancedLiteratureFetcher`, and that module has never existed in any commit: it was absent from the repo's first commit (`7c658e6`), which is also where these scripts were added. They fail at import, so even `--help` does not work. +They were deleted in #410. Porting them was not an import swap: their CLI flags +advertise a 6-tier PDF cascade with "fallback mirrors", and `LiteratureFetcher` +has no PDF surface at all, so a port meant deciding whether to *build* that +capability. The answer was no — retrieving publisher PDFs through mirrors is not +something to add speculatively, and the real need behind it is open-access full +text, which `scripts/cache_fulltext.py` already serves. The #183 sweep used it to +cache full text for 64 of 125 references. + +**Use instead:** + +| Removed script | What to use | +|---|---| +| `test_pdf_fetching.py` | nothing — it tested the cascade that was never committed | +| `curate_evidence_with_pdfs.py` | `scripts/cache_fulltext.py`, then `just validate-references` | +| `quick_literature_review.py` | `just validate-references FILE` | +| `review_literature.py` | `just validate-references FILE` | +| `extract_evidence_snippets.py` | no direct replacement; snippets are curated by hand against the cache | + This page previously read "✅ VALIDATED — Successfully integrated and tested", which was never true of the committed code. -**Everything below this line describes software that is not in this repository**, +**Everything below this line describes software that was never in this repository**, including the success rates, the per-tier table and the "5/5 DOIs" test results. Those numbers cannot have come from committed code and are retained only as a record of what was claimed. Do not cite them. diff --git a/scripts/curate_evidence_with_pdfs.py b/scripts/curate_evidence_with_pdfs.py deleted file mode 100644 index fe605c2f..00000000 --- a/scripts/curate_evidence_with_pdfs.py +++ /dev/null @@ -1,610 +0,0 @@ -#!/usr/bin/env python3 -""" -Evidence Curation Workflow with PDF Fallback - -Uses the enhanced PDF fetching to: -1. Validate all evidence items in community YAMLs -2. Fetch abstracts and PDFs for references -3. Validate snippets against source text -4. Extract better snippets from full text when available -5. Generate actionable curation tasks -6. Optionally apply automatic fixes - -This script enforces the schema requirements: -- reference: Required, must match pattern (PMID:|doi:|bioproject:) -- evidence_source: Required (IN_VITRO, IN_VIVO, etc.) -- snippet: Required, minimum 10 characters, must be exact quote -- confidence_score: Optional, based on validation - -.. warning:: - **This script has never run.** It imports - ``communitymech.literature_enhanced``, which has never existed in any commit - of this repository — this file was added in 7c658e6 already importing it, so - there is no revision at which it worked (#410). - - It is not a one-line fix. The API it was written against differs from the one - that exists: it calls ``fetch_paper(ref, download_pdf=...)`` and subscripts - the result (``paper["abstract"]``), whereas - ``communitymech.literature.LiteratureFetcher.fetch_paper(reference, email=...)`` - returns a ``(abstract, pdf_url)`` tuple and has no PDF download. Porting means - rewriting every call site, not swapping the import. - - For what it was meant to do, use the tooling that works: - ``just validate-references FILE`` and ``just validate-references-all`` for - snippet checking — these run the official ``linkml-reference-validator`` - against ``conf/reference_validator.yaml``, the custom validator having been - replaced in 4dd299a — and the ``evidence-curation`` skill for curating and - repairing evidence. - - Kept rather than deleted so the intent is recoverable, and pinned as broken by - ``tests/test_scripts_import.py``. Whether to port or delete is #410. - -""" - -import re -import sys -import time -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path - -import yaml - -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from communitymech.literature_enhanced import EnhancedLiteratureFetcher - - -@dataclass -class EvidenceIssue: - """Represents an evidence quality issue""" - - file: str - context: str # taxonomy, interaction, environmental - organism: str - reference: str - issue_type: str - current_value: str - suggested_fix: str | None = None - severity: str = "WARNING" # ERROR, WARNING, INFO - - -class EvidenceCurator: - """Comprehensive evidence curation using PDF fallback""" - - def __init__(self, use_pdf_fallback: bool = True, auto_fix: bool = False): - self.fetcher = EnhancedLiteratureFetcher( - cache_dir=".literature_cache", use_fallback_pdf=use_pdf_fallback - ) - self.auto_fix = auto_fix - self.issues = [] - self.stats = defaultdict(int) - - def validate_reference_format(self, reference: str) -> tuple[bool, str | None]: - """Validate reference matches schema pattern""" - - # Check pattern: Must start with PMID:, doi:, or bioproject: - pattern = r"^(PMID:|doi:|bioproject:)" - if not re.match(pattern, reference): - # Try to fix common issues - if reference.startswith("pmid:"): - return False, f"PMID:{reference[5:]}" - elif reference.startswith("DOI:"): - return False, f"doi:{reference[4:]}" - elif re.match(r"^10\.\d+/", reference): - return False, f"doi:{reference}" - elif re.match(r"^PMC\d+$", reference): - return False, f"PMID:{reference}" - else: - return False, None - - return True, reference - - def validate_snippet(self, snippet: str) -> tuple[bool, list[str]]: - """Validate snippet meets schema requirements""" - - issues = [] - - if not snippet: - issues.append("Snippet is empty") - return False, issues - - if len(snippet) < 10: - issues.append(f"Snippet too short ({len(snippet)} chars, minimum 10)") - - # Check for common invalid patterns - if snippet.startswith("Appl Environ Microbiol") or snippet.startswith("Front Microbiol"): - issues.append("Snippet appears to be journal citation, not content quote") - - if "..." in snippet and len(snippet) < 50: - issues.append("Snippet is truncated but very short") - - # Check for AI-generated patterns - ai_patterns = [ - "is an? (important|key|critical)", - "plays an? (important|key|critical) role", - "has been shown to", - "it is (known|believed) that", - ] - for pattern in ai_patterns: - if re.search(pattern, snippet, re.IGNORECASE): - issues.append(f"Snippet may be AI-generated/paraphrased (pattern: {pattern})") - break - - return len(issues) == 0, issues - - def fetch_and_validate_evidence( - self, reference: str, snippet: str, organism: str = None, fetch_pdf: bool = False - ) -> dict: - """ - Fetch paper and validate snippet against it. - - Returns: - { - 'abstract_fetched': bool, - 'pdf_fetched': bool, - 'snippet_valid': bool, - 'snippet_in_abstract': bool, - 'snippet_in_fulltext': bool, - 'suggested_snippet': str or None, - 'confidence_score': float, - 'paper_metadata': dict - } - """ - - result = { - "abstract_fetched": False, - "pdf_fetched": False, - "snippet_valid": False, - "snippet_in_abstract": False, - "snippet_in_fulltext": False, - "suggested_snippet": None, - "confidence_score": 0.0, - "paper_metadata": {}, - } - - try: - # Fetch paper - paper = self.fetcher.fetch_paper(reference, download_pdf=fetch_pdf) - - if paper.get("abstract"): - result["abstract_fetched"] = True - result["paper_metadata"] = { - "title": paper.get("title"), - "year": paper.get("year"), - "journal": paper.get("journal"), - "authors": paper.get("authors", []), - } - - # Validate snippet against abstract - if snippet: - result["snippet_in_abstract"] = self.fetcher.validate_evidence_snippet( - snippet, paper["abstract"] - ) - result["snippet_valid"] = result["snippet_in_abstract"] - - if result["snippet_valid"]: - result["confidence_score"] = 1.0 - else: - # Try to extract better snippet - result["suggested_snippet"] = self._extract_best_snippet( - paper["abstract"], organism, snippet - ) - result["confidence_score"] = 0.3 - - if fetch_pdf and paper.get("pdf_text"): - result["pdf_fetched"] = True - - # If snippet not in abstract, check full text - if not result["snippet_valid"] and snippet: - result["snippet_in_fulltext"] = self.fetcher.validate_evidence_snippet( - snippet, paper["pdf_text"] - ) - if result["snippet_in_fulltext"]: - result["snippet_valid"] = True - result["confidence_score"] = 0.8 # Lower than abstract (less specific) - - # Extract from full text if still invalid - if not result["snippet_valid"]: - result["suggested_snippet"] = self._extract_best_snippet( - paper["pdf_text"], organism, snippet - ) - result["confidence_score"] = 0.5 - - except Exception as e: - print(f" Error fetching {reference}: {e}") - - return result - - def _extract_best_snippet( - self, text: str, organism: str = None, current_snippet: str = None - ) -> str | None: - """Extract best matching snippet from text""" - - # Split into sentences - sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z])", text) - - # Filter out journal citation lines - sentences = [ - s - for s in sentences - if not re.match(r"^[A-Z][a-z]+ [A-Z][a-z]+\.\s+\d{4}", s) # Journal citation - and len(s) > 50 # Substantive - ] - - if not sentences: - return None - - # Try to find sentence mentioning organism - if organism: - organism_variants = [ - organism, - organism.split()[0] if " " in organism else None, # Genus only - organism.replace("Candidatus ", "") if "Candidatus" in organism else None, - ] - organism_variants = [v for v in organism_variants if v] - - for sentence in sentences: - if any(org and org.lower() in sentence.lower() for org in organism_variants): - return self._clean_snippet(sentence) - - # Try to find sentence with keywords from current snippet - if current_snippet: - keywords = [ - w - for w in re.findall(r"\b\w{4,}\b", current_snippet.lower()) - if w not in ["that", "with", "from", "this", "were", "have"] - ][ - :5 - ] # Top 5 keywords - - best_sentence = None - best_score = 0 - - for sentence in sentences: - score = sum(1 for kw in keywords if kw in sentence.lower()) - if score > best_score: - best_score = score - best_sentence = sentence - - if best_sentence and best_score >= 2: # At least 2 keyword matches - return self._clean_snippet(best_sentence) - - # Fallback: return first substantive sentence - return self._clean_snippet(sentences[0]) if sentences else None - - def _clean_snippet(self, text: str) -> str: - """Clean snippet for use in YAML""" - # Remove citations - text = re.sub(r"\[\d+\]", "", text) - text = re.sub(r"\([A-Za-z\s,]+\d{4}\)", "", text) - # Remove excess whitespace - text = " ".join(text.split()) - return text - - def audit_community_yaml( - self, yaml_path: Path, fetch_pdfs: bool = False - ) -> list[EvidenceIssue]: - """Audit all evidence in a community YAML""" - - with open(yaml_path) as f: - data = yaml.safe_load(f) - - issues = [] - - # Check taxonomy evidence - if "taxonomy" in data: - for taxon_entry in data["taxonomy"]: - organism = taxon_entry.get("taxon_term", {}).get("preferred_term", "Unknown") - - if "evidence" not in taxon_entry or not taxon_entry["evidence"]: - issues.append( - EvidenceIssue( - file=yaml_path.name, - context="taxonomy", - organism=organism, - reference="N/A", - issue_type="MISSING_EVIDENCE", - current_value="No evidence provided", - severity="ERROR", - ) - ) - continue - - for ev in taxon_entry["evidence"]: - issues.extend( - self._audit_evidence_item( - ev, yaml_path.name, "taxonomy", organism, fetch_pdfs - ) - ) - - # Check interaction evidence - if "ecological_interactions" in data: - for interaction in data["ecological_interactions"]: - interaction_name = interaction.get("name", "Unknown") - - if "evidence" not in interaction or not interaction["evidence"]: - issues.append( - EvidenceIssue( - file=yaml_path.name, - context="interaction", - organism=interaction_name, - reference="N/A", - issue_type="MISSING_EVIDENCE", - current_value="No evidence provided", - severity="ERROR", - ) - ) - continue - - for ev in interaction["evidence"]: - issues.extend( - self._audit_evidence_item( - ev, yaml_path.name, "interaction", interaction_name, fetch_pdfs - ) - ) - - # Check environmental evidence - if "environmental_factors" in data: - for factor in data["environmental_factors"]: - factor_name = factor.get("factor", "Unknown") - - if "evidence" in factor and factor["evidence"]: - for ev in factor["evidence"]: - issues.extend( - self._audit_evidence_item( - ev, yaml_path.name, "environmental", factor_name, fetch_pdfs - ) - ) - - return issues - - def _audit_evidence_item( - self, evidence: dict, filename: str, context: str, organism: str, fetch_pdf: bool - ) -> list[EvidenceIssue]: - """Audit single evidence item""" - - issues = [] - reference = evidence.get("reference", "") - snippet = evidence.get("snippet", "") - evidence_source = evidence.get("evidence_source", "") - - # Check reference format - ref_valid, ref_fix = self.validate_reference_format(reference) - if not ref_valid: - issues.append( - EvidenceIssue( - file=filename, - context=context, - organism=organism, - reference=reference, - issue_type="INVALID_REFERENCE_FORMAT", - current_value=reference, - suggested_fix=ref_fix, - severity="ERROR" if not ref_fix else "WARNING", - ) - ) - self.stats["invalid_reference_format"] += 1 - - # Check evidence_source - if not evidence_source: - issues.append( - EvidenceIssue( - file=filename, - context=context, - organism=organism, - reference=reference, - issue_type="MISSING_EVIDENCE_SOURCE", - current_value="Not specified", - suggested_fix="IN_VITRO or IN_VIVO (check paper)", - severity="ERROR", - ) - ) - self.stats["missing_evidence_source"] += 1 - - # Check snippet - snippet_valid, snippet_issues = self.validate_snippet(snippet) - if not snippet_valid: - for issue_desc in snippet_issues: - issues.append( - EvidenceIssue( - file=filename, - context=context, - organism=organism, - reference=reference, - issue_type="INVALID_SNIPPET", - current_value=snippet[:100] if snippet else "(empty)", - suggested_fix=f"Issue: {issue_desc}", - severity="ERROR", - ) - ) - self.stats["invalid_snippet"] += 1 - - # Validate against source if snippet provided and reference valid - if snippet and ref_valid: - self.stats["total_validated"] += 1 - validation = self.fetch_and_validate_evidence(reference, snippet, organism, fetch_pdf) - - if not validation["abstract_fetched"]: - issues.append( - EvidenceIssue( - file=filename, - context=context, - organism=organism, - reference=reference, - issue_type="ABSTRACT_FETCH_FAILED", - current_value=snippet[:100], - severity="WARNING", - ) - ) - self.stats["abstract_fetch_failed"] += 1 - - elif not validation["snippet_valid"]: - issues.append( - EvidenceIssue( - file=filename, - context=context, - organism=organism, - reference=reference, - issue_type="SNIPPET_NOT_IN_SOURCE", - current_value=snippet[:100], - suggested_fix=( - validation["suggested_snippet"][:150] - if validation["suggested_snippet"] - else None - ), - severity="ERROR", - ) - ) - self.stats["snippet_not_in_source"] += 1 - else: - self.stats["valid_evidence"] += 1 - - return issues - - def generate_report(self, issues: list[EvidenceIssue], output_path: Path): - """Generate comprehensive curation report""" - - with open(output_path, "w") as f: - f.write("EVIDENCE CURATION REPORT\n") - f.write("=" * 80 + "\n\n") - - # Statistics - f.write("STATISTICS\n") - f.write("-" * 80 + "\n") - f.write(f"Total evidence items validated: {self.stats.get('total_validated', 0)}\n") - f.write(f"Valid evidence: {self.stats.get('valid_evidence', 0)}\n") - f.write(f"Issues found: {len(issues)}\n\n") - - f.write("Issue breakdown:\n") - f.write( - f" - Invalid reference format: {self.stats.get('invalid_reference_format', 0)}\n" - ) - f.write( - f" - Missing evidence source: {self.stats.get('missing_evidence_source', 0)}\n" - ) - f.write(f" - Invalid snippet: {self.stats.get('invalid_snippet', 0)}\n") - f.write(f" - Abstract fetch failed: {self.stats.get('abstract_fetch_failed', 0)}\n") - f.write(f" - Snippet not in source: {self.stats.get('snippet_not_in_source', 0)}\n") - f.write("\n") - - # Group by file - by_file = defaultdict(list) - for issue in issues: - by_file[issue.file].append(issue) - - # Group by severity - by_severity = defaultdict(list) - for issue in issues: - by_severity[issue.severity].append(issue) - - f.write(f"Files affected: {len(by_file)}\n") - f.write(f" - ERROR: {len(by_severity['ERROR'])}\n") - f.write(f" - WARNING: {len(by_severity['WARNING'])}\n") - f.write(f" - INFO: {len(by_severity['INFO'])}\n") - f.write("\n" + "=" * 80 + "\n\n") - - # Detailed issues by file - for filename in sorted(by_file.keys()): - file_issues = by_file[filename] - f.write(f"\n{filename} ({len(file_issues)} issues)\n") - f.write("-" * 80 + "\n\n") - - # Group by issue type - by_type = defaultdict(list) - for issue in file_issues: - by_type[issue.issue_type].append(issue) - - for issue_type in sorted(by_type.keys()): - type_issues = by_type[issue_type] - f.write(f"{issue_type} ({len(type_issues)} instances)\n") - f.write("~" * 40 + "\n") - - for issue in type_issues[:5]: # Show first 5 - f.write(f"\nOrganism/Item: {issue.organism}\n") - f.write(f"Reference: {issue.reference}\n") - f.write(f"Current: {issue.current_value}\n") - if issue.suggested_fix: - f.write(f"Suggested fix: {issue.suggested_fix}\n") - f.write("\n") - - if len(type_issues) > 5: - f.write(f"... and {len(type_issues)-5} more\n\n") - - f.write("\n") - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Curate evidence with PDF fallback") - parser.add_argument("--file", help="Specific YAML file to audit") - parser.add_argument("--with-pdfs", action="store_true", help="Fetch PDFs for validation (slow)") - parser.add_argument( - "--auto-fix", action="store_true", help="Automatically apply fixes where possible" - ) - parser.add_argument("--quick", action="store_true", help="Quick audit (skip PDF fetching)") - args = parser.parse_args() - - curator = EvidenceCurator(use_pdf_fallback=not args.quick, auto_fix=args.auto_fix) - - kb_dir = Path("kb/communities") - - if args.file: - yaml_files = [kb_dir / args.file] - else: - yaml_files = sorted(kb_dir.glob("*.yaml")) - - print("Evidence Curation Workflow") - print("=" * 80) - print(f"Mode: {'WITH PDFs' if args.with_pdfs else 'Abstracts only'}") - print(f"Auto-fix: {args.auto_fix}") - print() - - all_issues = [] - - for yaml_path in yaml_files: - print(f"Auditing {yaml_path.name}...") - issues = curator.audit_community_yaml(yaml_path, fetch_pdfs=args.with_pdfs) - - if issues: - print(f" Found {len(issues)} issues") - all_issues.extend(issues) - else: - print(" ✓ No issues found") - - # Rate limit - time.sleep(0.5) - - print() - print("=" * 80) - print(f"Total issues: {len(all_issues)}") - print() - - # Generate report - report_path = Path("evidence_curation_report.txt") - curator.generate_report(all_issues, report_path) - - print(f"✓ Report written: {report_path}") - print() - - # Summary by severity - by_severity = defaultdict(int) - for issue in all_issues: - by_severity[issue.severity] += 1 - - print("Issues by severity:") - print(f" ERROR: {by_severity['ERROR']:4d} (must fix)") - print(f" WARNING: {by_severity['WARNING']:4d} (should fix)") - print(f" INFO: {by_severity['INFO']:4d} (nice to fix)") - print() - - print("Next steps:") - print(" 1. Review evidence_curation_report.txt") - print(" 2. Fix ERROR-level issues first") - print(" 3. Re-run with --with-pdfs for thorough validation") - print(" 4. Run schema validation: just validate-all") - - -if __name__ == "__main__": - main() diff --git a/scripts/extract_evidence_snippets.py b/scripts/extract_evidence_snippets.py deleted file mode 100644 index 2b69d40d..00000000 --- a/scripts/extract_evidence_snippets.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -""" -Evidence Snippet Extraction Tool - -Extracts relevant snippets from abstracts/PDFs for evidence items missing snippets. - -Usage: - python scripts/extract_evidence_snippets.py - -Example: - python scripts/extract_evidence_snippets.py "PMID:28287150" "DIET" "electron transfer" - -.. warning:: - **This script has never run.** It imports - ``communitymech.literature_enhanced``, which has never existed in any commit - of this repository — this file was added in 7c658e6 already importing it, so - there is no revision at which it worked (#410). - - It is not a one-line fix. The API it was written against differs from the one - that exists: it calls ``fetch_paper(ref, download_pdf=...)`` and subscripts - the result (``paper["abstract"]``), whereas - ``communitymech.literature.LiteratureFetcher.fetch_paper(reference, email=...)`` - returns a ``(abstract, pdf_url)`` tuple and has no PDF download. Porting means - rewriting every call site, not swapping the import. - - For what it was meant to do, use the tooling that works: - ``just validate-references FILE`` and ``just validate-references-all`` for - snippet checking — these run the official ``linkml-reference-validator`` - against ``conf/reference_validator.yaml``, the custom validator having been - replaced in 4dd299a — and the ``evidence-curation`` skill for curating and - repairing evidence. - - Kept rather than deleted so the intent is recoverable, and pinned as broken by - ``tests/test_scripts_import.py``. Whether to port or delete is #410. - -""" - -import re -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from communitymech.literature_enhanced import EnhancedLiteratureFetcher - - -class SnippetExtractor: - """Extract relevant snippets from papers based on search terms""" - - def __init__(self): - self.fetcher = EnhancedLiteratureFetcher( - cache_dir=".literature_cache", use_fallback_pdf=True - ) - - def extract_snippets( - self, reference: str, search_terms: list[str], use_pdf: bool = False - ) -> list[str]: - """ - Extract relevant snippets containing search terms. - - Args: - reference: PMID or DOI - search_terms: List of keywords to search for - use_pdf: Whether to download and search full PDF - - Returns: - List of relevant snippets - """ - # Fetch paper - paper = self.fetcher.fetch_paper(reference, download_pdf=use_pdf) - - snippets = [] - - # Search in abstract - if paper["abstract"]: - abstract_snippets = self._find_snippets_in_text(paper["abstract"], search_terms) - snippets.extend(abstract_snippets) - - # Search in PDF text - if use_pdf and paper["pdf_text"]: - pdf_snippets = self._find_snippets_in_text( - paper["pdf_text"], search_terms, max_snippets=5 # Limit PDF snippets - ) - snippets.extend(pdf_snippets) - - return snippets - - def _find_snippets_in_text( - self, text: str, search_terms: list[str], max_snippets: int = 10, context_chars: int = 200 - ) -> list[str]: - """ - Find snippets containing search terms with surrounding context. - - Args: - text: Full text to search - search_terms: Keywords to find - max_snippets: Maximum number of snippets to return - context_chars: Characters of context around each match - - Returns: - List of snippets - """ - snippets = [] - - # Normalize text - text_normalized = " ".join(text.split()) - - # Find sentences containing any search term - sentences = self._split_into_sentences(text_normalized) - - for sentence in sentences: - # Check if sentence contains any search term (case-insensitive) - if any(term.lower() in sentence.lower() for term in search_terms): - # Clean up sentence - snippet = sentence.strip() - - # Remove references like [1], (Smith et al., 2020) - snippet = re.sub(r"\[\d+\]", "", snippet) - snippet = re.sub(r"\([A-Za-z\s,]+\d{4}\)", "", snippet) - - # Remove excessive whitespace - snippet = " ".join(snippet.split()) - - if len(snippet) > 50: # Minimum snippet length - snippets.append(snippet) - - if len(snippets) >= max_snippets: - break - - return snippets - - def _split_into_sentences(self, text: str) -> list[str]: - """ - Split text into sentences. - - Args: - text: Text to split - - Returns: - List of sentences - """ - # Simple sentence splitter (could be improved with nltk) - # Split on period followed by space and capital letter - sentences = re.split(r"(?<=[.!?])\s+(?=[A-Z])", text) - - # Also split on newlines (for abstracts with line breaks) - result = [] - for s in sentences: - result.extend(s.split("\n")) - - return [s.strip() for s in result if s.strip()] - - -def main(): - """CLI for extracting evidence snippets""" - if len(sys.argv) < 3: - print( - "Usage: python scripts/extract_evidence_snippets.py [search_term2] ..." - ) - print("\nExample:") - print( - ' python scripts/extract_evidence_snippets.py "PMID:28287150" "DIET" "electron transfer"' - ) - sys.exit(1) - - reference = sys.argv[1] - search_terms = sys.argv[2:] - use_pdf = "--pdf" in sys.argv - - print(f"Reference: {reference}") - print(f"Search terms: {', '.join(search_terms)}") - print(f"Use PDF: {use_pdf}") - print("=" * 80) - - extractor = SnippetExtractor() - - print("\nFetching paper and extracting snippets...") - snippets = extractor.extract_snippets(reference, search_terms, use_pdf=use_pdf) - - if not snippets: - print("\n✗ No relevant snippets found") - sys.exit(1) - - print(f"\n✓ Found {len(snippets)} relevant snippets:\n") - print("=" * 80) - - for i, snippet in enumerate(snippets, 1): - print(f"\n{i}. {snippet}") - print() - - print("=" * 80) - print("\nCopy the most relevant snippet into your YAML evidence item.") - - -if __name__ == "__main__": - main() diff --git a/scripts/quick_literature_review.py b/scripts/quick_literature_review.py deleted file mode 100644 index f1770804..00000000 --- a/scripts/quick_literature_review.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -""" -Quick Literature Review - Fast validation without PDF fetching - -Validates abstracts and snippets only, skipping PDF discovery. -Much faster for initial assessment (~5-10 minutes vs 60-90 minutes). - -.. warning:: - **This script has never run.** It imports - ``communitymech.literature_enhanced``, which has never existed in any commit - of this repository — this file was added in 7c658e6 already importing it, so - there is no revision at which it worked (#410). - - It is not a one-line fix. The API it was written against differs from the one - that exists: it calls ``fetch_paper(ref, download_pdf=...)`` and subscripts - the result (``paper["abstract"]``), whereas - ``communitymech.literature.LiteratureFetcher.fetch_paper(reference, email=...)`` - returns a ``(abstract, pdf_url)`` tuple and has no PDF download. Porting means - rewriting every call site, not swapping the import. - - For what it was meant to do, use the tooling that works: - ``just validate-references FILE`` and ``just validate-references-all`` for - snippet checking — these run the official ``linkml-reference-validator`` - against ``conf/reference_validator.yaml``, the custom validator having been - replaced in 4dd299a — and the ``evidence-curation`` skill for curating and - repairing evidence. - - Kept rather than deleted so the intent is recoverable, and pinned as broken by - ``tests/test_scripts_import.py``. Whether to port or delete is #410. - -""" - -import sys -from collections import defaultdict -from dataclasses import dataclass -from pathlib import Path - -import yaml - -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from communitymech.literature_enhanced import EnhancedLiteratureFetcher - - -# Disable PDF fetching for speed -class FastFetcher(EnhancedLiteratureFetcher): - """Override to skip PDF fetching""" - - def fetch_pdf_url(self, doi: str): - """Skip PDF fetching for speed""" - return None - - -@dataclass -class QuickResult: - reference: str - file: str - context: str - has_abstract: bool - snippet_valid: bool = False - has_snippet: bool = False - issue: str | None = None - - -def main(): - print("QUICK LITERATURE REVIEW (Abstract validation only)") - print("=" * 80) - print() - - kb_dir = Path("kb/communities") - fetcher = FastFetcher(cache_dir=".literature_cache", use_fallback_pdf=False) - - # Scan all YAMLs - print("Scanning YAMLs...") - yaml_files = sorted(kb_dir.glob("*.yaml")) - all_evidence = [] - - for yaml_path in yaml_files: - with open(yaml_path) as f: - data = yaml.safe_load(f) - - # Extract evidence - if "taxonomy" in data: - for taxon in data["taxonomy"]: - if "evidence" in taxon: - for ev in taxon["evidence"]: - all_evidence.append( - { - "file": yaml_path.name, - "context": "taxonomy", - "ref": ev.get("reference", ""), - "snippet": ev.get("snippet"), - } - ) - - if "ecological_interactions" in data: - for interaction in data["ecological_interactions"]: - if "evidence" in interaction: - for ev in interaction["evidence"]: - all_evidence.append( - { - "file": yaml_path.name, - "context": "interaction", - "ref": ev.get("reference", ""), - "snippet": ev.get("snippet"), - } - ) - - print(f"Found {len(all_evidence)} evidence items") - print() - - # Quick validation - print("Validating (abstracts only, no PDFs)...") - results = [] - stats = defaultdict(int) - - for i, ev in enumerate(all_evidence, 1): - if i % 50 == 0: - print(f" Progress: {i}/{len(all_evidence)}") - - result = QuickResult( - reference=ev["ref"], - file=ev["file"], - context=ev["context"], - has_abstract=False, - has_snippet=bool(ev["snippet"]), - ) - - # Fetch abstract only - try: - paper = fetcher.fetch_paper(ev["ref"], download_pdf=False) - - if paper["abstract"]: - result.has_abstract = True - stats["abstracts_ok"] += 1 - - # Validate snippet if present - if ev["snippet"]: - valid = fetcher.validate_evidence_snippet(ev["snippet"], paper["abstract"]) - result.snippet_valid = valid - if valid: - stats["snippets_valid"] += 1 - else: - stats["snippets_invalid"] += 1 - result.issue = "Snippet not in abstract" - else: - stats["missing_snippet"] += 1 - result.issue = "Missing snippet" - else: - stats["abstracts_failed"] += 1 - result.issue = "Abstract not found" - - except Exception as e: - stats["errors"] += 1 - result.issue = f"Error: {str(e)[:50]}" - - results.append(result) - - # Summary - print() - print("=" * 80) - print("SUMMARY") - print("=" * 80) - print(f"Total evidence: {len(all_evidence)}") - print(f"Abstracts fetched: {stats['abstracts_ok']}") - print(f"Abstracts failed: {stats['abstracts_failed']}") - print(f"Snippets valid: {stats['snippets_valid']}") - print(f"Snippets invalid: {stats['snippets_invalid']}") - print(f"Missing snippets: {stats['missing_snippet']}") - print(f"Errors: {stats['errors']}") - print() - - # Quality metrics - if len(all_evidence) > 0: - abstract_rate = (stats["abstracts_ok"] / len(all_evidence)) * 100 - print(f"Abstract fetch rate: {abstract_rate:.1f}%") - - if stats["snippets_valid"] + stats["snippets_invalid"] > 0: - snippet_rate = ( - stats["snippets_valid"] / (stats["snippets_valid"] + stats["snippets_invalid"]) - ) * 100 - print(f"Snippet validation rate: {snippet_rate:.1f}%") - - # Write quick report - with open("quick_literature_report.txt", "w") as f: - f.write("QUICK LITERATURE REVIEW\n") - f.write("=" * 80 + "\n\n") - f.write(f"Total: {len(all_evidence)}\n") - f.write(f"Abstracts OK: {stats['abstracts_ok']}\n") - f.write(f"Abstracts failed: {stats['abstracts_failed']}\n") - f.write(f"Snippets valid: {stats['snippets_valid']}\n") - f.write(f"Snippets invalid: {stats['snippets_invalid']}\n") - f.write(f"Missing snippets: {stats['missing_snippet']}\n\n") - - # Issues - f.write("ISSUES:\n" + "-" * 80 + "\n\n") - for result in results: - if result.issue: - f.write(f"{result.file} - {result.reference}\n") - f.write(f" Issue: {result.issue}\n\n") - - print() - print("✓ Report: quick_literature_report.txt") - - -if __name__ == "__main__": - main() diff --git a/scripts/review_literature.py b/scripts/review_literature.py deleted file mode 100644 index 5a76115c..00000000 --- a/scripts/review_literature.py +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env python3 -""" -Literature Review & Update Tool for CommunityMech - -Comprehensively reviews and validates literature evidence in community YAML files. - -Features: -1. Validate evidence snippets against abstracts -2. Identify missing references -3. Download full PDFs (with scihub fallback) -4. Extract additional evidence from full papers -5. Generate literature quality report -6. Suggest evidence improvements - -Usage: - python scripts/review_literature.py # Review all communities - python scripts/review_literature.py --community AMD # Review specific community - python scripts/review_literature.py --download-pdfs # Download full PDFs - python scripts/review_literature.py --update # Auto-update valid evidence - -.. warning:: - **This script has never run.** It imports - ``communitymech.literature_enhanced``, which has never existed in any commit - of this repository — this file was added in 7c658e6 already importing it, so - there is no revision at which it worked (#410). - - It is not a one-line fix. The API it was written against differs from the one - that exists: it calls ``fetch_paper(ref, download_pdf=...)`` and subscripts - the result (``paper["abstract"]``), whereas - ``communitymech.literature.LiteratureFetcher.fetch_paper(reference, email=...)`` - returns a ``(abstract, pdf_url)`` tuple and has no PDF download. Porting means - rewriting every call site, not swapping the import. - - For what it was meant to do, use the tooling that works: - ``just validate-references FILE`` and ``just validate-references-all`` for - snippet checking — these run the official ``linkml-reference-validator`` - against ``conf/reference_validator.yaml``, the custom validator having been - replaced in 4dd299a — and the ``evidence-curation`` skill for curating and - repairing evidence. - - Kept rather than deleted so the intent is recoverable, and pinned as broken by - ``tests/test_scripts_import.py``. Whether to port or delete is #410. - -""" - -import argparse -import sys -from collections import defaultdict -from dataclasses import dataclass, field -from pathlib import Path - -import yaml - -# Add src to path -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from communitymech.literature_enhanced import EnhancedLiteratureFetcher - -# Repo-anchored: a relative default follows the cwd (#407). -_REPORTS = Path(__file__).resolve().parent.parent / "reports" - - -# Color codes -class Colors: - HEADER = "\033[95m" - BLUE = "\033[94m" - CYAN = "\033[96m" - GREEN = "\033[92m" - YELLOW = "\033[93m" - RED = "\033[91m" - BOLD = "\033[1m" - RESET = "\033[0m" - - -@dataclass -class EvidenceItem: - """Represents a single evidence item from YAML""" - - file: str - context: str # taxonomy/interaction/environmental_factor - organism: str | None - reference: str - snippet: str | None - supports: str - evidence_source: str | None - explanation: str | None - - -@dataclass -class ValidationResult: - """Results of validating an evidence item""" - - evidence: EvidenceItem - abstract_fetched: bool = False - abstract_text: str | None = None - snippet_valid: bool = False - pdf_available: bool = False - pdf_url: str | None = None - pdf_source: str | None = None - issues: list[str] = field(default_factory=list) - suggestions: list[str] = field(default_factory=list) - - -class LiteratureReviewer: - """Reviews and validates literature evidence in community YAMLs""" - - def __init__(self, kb_dir: Path, download_pdfs: bool = False, use_fallback: bool = True): - self.kb_dir = kb_dir - self.download_pdfs = download_pdfs - self.fetcher = EnhancedLiteratureFetcher( - cache_dir=".literature_cache", - pdf_cache_dir=".pdf_cache", - email="noreply@communitymech.org", - use_fallback_pdf=use_fallback, - ) - self.stats = defaultdict(int) - self.evidence_items: list[EvidenceItem] = [] - self.validation_results: list[ValidationResult] = [] - - def extract_evidence_from_yaml(self, yaml_path: Path) -> list[EvidenceItem]: - """Extract all evidence items from a YAML file""" - with open(yaml_path) as f: - data = yaml.safe_load(f) - - evidence_items = [] - - # Extract from taxonomy - if "taxonomy" in data: - for taxon_entry in data["taxonomy"]: - organism = taxon_entry.get("taxon_term", {}).get("preferred_term", "Unknown") - - if "evidence" in taxon_entry: - for ev in taxon_entry["evidence"]: - evidence_items.append( - EvidenceItem( - file=yaml_path.name, - context="taxonomy", - organism=organism, - reference=ev.get("reference", ""), - snippet=ev.get("snippet"), - supports=ev.get("supports", ""), - evidence_source=ev.get("evidence_source"), - explanation=ev.get("explanation"), - ) - ) - - # Extract from ecological_interactions - if "ecological_interactions" in data: - for interaction in data["ecological_interactions"]: - interaction_name = interaction.get("name", "Unknown") - - if "evidence" in interaction: - for ev in interaction["evidence"]: - evidence_items.append( - EvidenceItem( - file=yaml_path.name, - context="interaction", - organism=interaction_name, - reference=ev.get("reference", ""), - snippet=ev.get("snippet"), - supports=ev.get("supports", ""), - evidence_source=ev.get("evidence_source"), - explanation=ev.get("explanation"), - ) - ) - - # Extract from environmental_factors - if "environmental_factors" in data: - for factor in data["environmental_factors"]: - factor_name = factor.get("name", "Unknown") - - if "evidence" in factor: - for ev in factor["evidence"]: - evidence_items.append( - EvidenceItem( - file=yaml_path.name, - context="environmental", - organism=factor_name, - reference=ev.get("reference", ""), - snippet=ev.get("snippet"), - supports=ev.get("supports", ""), - evidence_source=ev.get("evidence_source"), - explanation=ev.get("explanation"), - ) - ) - - return evidence_items - - def scan_all_communities(self) -> None: - """Scan all community YAML files for evidence""" - print(f"{Colors.CYAN}Scanning community YAML files...{Colors.RESET}\n") - - yaml_files = sorted(self.kb_dir.glob("*.yaml")) - - for yaml_path in yaml_files: - evidence = self.extract_evidence_from_yaml(yaml_path) - self.evidence_items.extend(evidence) - - if evidence: - print( - f" {Colors.GREEN}✓{Colors.RESET} {yaml_path.name}: {len(evidence)} evidence items" - ) - - print(f"\n{Colors.BOLD}Total evidence items: {len(self.evidence_items)}{Colors.RESET}\n") - - def validate_evidence_item(self, evidence: EvidenceItem) -> ValidationResult: - """Validate a single evidence item""" - result = ValidationResult(evidence=evidence) - - # Fetch paper - paper = self.fetcher.fetch_paper(evidence.reference, download_pdf=self.download_pdfs) - - # Check if abstract was fetched - if paper["abstract"]: - result.abstract_fetched = True - result.abstract_text = paper["abstract"] - self.stats["abstracts_fetched"] += 1 - - # Validate snippet against abstract - if evidence.snippet: - is_valid = self.fetcher.validate_evidence_snippet( - evidence.snippet, paper["abstract"] - ) - result.snippet_valid = is_valid - - if is_valid: - self.stats["snippets_valid"] += 1 - else: - result.issues.append("Snippet not found in abstract") - self.stats["snippets_invalid"] += 1 - else: - result.issues.append("No snippet provided") - self.stats["missing_snippets"] += 1 - - else: - result.issues.append("Could not fetch abstract") - self.stats["abstracts_failed"] += 1 - - # Check PDF availability - if paper["pdf_url"]: - result.pdf_available = True - result.pdf_url = paper["pdf_url"] - result.pdf_source = paper["source"] - self.stats["pdfs_available"] += 1 - - # Generate suggestions - if not evidence.snippet and result.abstract_text: - result.suggestions.append("Could extract snippet from abstract") - - if not evidence.explanation: - result.suggestions.append("Missing explanation field") - - if not evidence.evidence_source: - result.suggestions.append("Missing evidence_source field") - - return result - - def validate_all_evidence(self) -> None: - """Validate all evidence items""" - print(f"{Colors.CYAN}Validating evidence items...{Colors.RESET}\n") - - total = len(self.evidence_items) - - for i, evidence in enumerate(self.evidence_items, 1): - if i % 10 == 0 or i == 1: - print(f" Progress: {i}/{total}") - - result = self.validate_evidence_item(evidence) - self.validation_results.append(result) - - print(f"\n{Colors.GREEN}✓{Colors.RESET} Validation complete\n") - - def generate_report(self, output_path: Path) -> None: - """Generate comprehensive literature quality report""" - print(f"{Colors.CYAN}Generating literature quality report...{Colors.RESET}") - - with open(output_path, "w") as f: - f.write("=" * 80 + "\n") - f.write("COMMUNITYMECH LITERATURE QUALITY REPORT\n") - f.write("=" * 80 + "\n\n") - - # Summary statistics - f.write("SUMMARY STATISTICS\n") - f.write("-" * 80 + "\n\n") - f.write(f"Total evidence items: {len(self.evidence_items)}\n") - f.write(f"Abstracts fetched: {self.stats['abstracts_fetched']}\n") - f.write(f"Abstracts failed: {self.stats['abstracts_failed']}\n") - f.write(f"Snippets valid: {self.stats['snippets_valid']}\n") - f.write(f"Snippets invalid: {self.stats['snippets_invalid']}\n") - f.write(f"Missing snippets: {self.stats['missing_snippets']}\n") - f.write(f"PDFs available: {self.stats['pdfs_available']}\n\n") - - # Calculate quality metrics - if len(self.evidence_items) > 0: - abstract_rate = (self.stats["abstracts_fetched"] / len(self.evidence_items)) * 100 - snippet_rate = ( - ( - self.stats["snippets_valid"] - / (self.stats["snippets_valid"] + self.stats["snippets_invalid"]) - ) - * 100 - if (self.stats["snippets_valid"] + self.stats["snippets_invalid"]) > 0 - else 0 - ) - pdf_rate = (self.stats["pdfs_available"] / len(self.evidence_items)) * 100 - - f.write("QUALITY METRICS\n") - f.write("-" * 80 + "\n\n") - f.write(f"Abstract fetch rate: {abstract_rate:.1f}%\n") - f.write(f"Snippet validation rate: {snippet_rate:.1f}%\n") - f.write(f"PDF availability rate: {pdf_rate:.1f}%\n\n") - - # Issues by file - f.write("\n" + "=" * 80 + "\n") - f.write("ISSUES BY FILE\n") - f.write("=" * 80 + "\n\n") - - issues_by_file = defaultdict(list) - for result in self.validation_results: - if result.issues: - issues_by_file[result.evidence.file].append(result) - - for file, results in sorted(issues_by_file.items()): - f.write(f"\n{file}\n") - f.write("-" * 80 + "\n") - - for result in results: - f.write(f"\nReference: {result.evidence.reference}\n") - f.write(f"Context: {result.evidence.context}\n") - if result.evidence.organism: - f.write(f"Organism/Factor: {result.evidence.organism}\n") - - f.write("Issues:\n") - for issue in result.issues: - f.write(f" - {issue}\n") - - if result.suggestions: - f.write("Suggestions:\n") - for suggestion in result.suggestions: - f.write(f" - {suggestion}\n") - - if result.snippet_valid: - f.write(" ✓ Snippet valid\n") - - if result.pdf_available: - f.write(f" ✓ PDF available: {result.pdf_url}\n") - - # Valid evidence (for reference) - f.write("\n\n" + "=" * 80 + "\n") - f.write("VALID EVIDENCE (No Issues)\n") - f.write("=" * 80 + "\n\n") - - valid_by_file = defaultdict(list) - for result in self.validation_results: - if not result.issues: - valid_by_file[result.evidence.file].append(result) - - for file, results in sorted(valid_by_file.items()): - f.write(f"\n{file}: {len(results)} valid evidence items\n") - - # References with PDFs available - f.write("\n\n" + "=" * 80 + "\n") - f.write("REFERENCES WITH FULL PDF ACCESS\n") - f.write("=" * 80 + "\n\n") - - pdf_refs = {} - for result in self.validation_results: - if result.pdf_available: - ref = result.evidence.reference - if ref not in pdf_refs: - pdf_refs[ref] = { - "url": result.pdf_url, - "source": result.pdf_source, - "files": [], - } - pdf_refs[ref]["files"].append(result.evidence.file) - - for ref, info in sorted(pdf_refs.items()): - f.write(f"\n{ref}\n") - f.write(f" URL: {info['url']}\n") - f.write(f" Source: {info['source']}\n") - f.write(f" Used in: {', '.join(set(info['files']))}\n") - - print(f"{Colors.GREEN}✓{Colors.RESET} Report written: {output_path}\n") - - def generate_priority_update_list(self, output_path: Path) -> None: - """Generate prioritized list of evidence needing updates""" - print(f"{Colors.CYAN}Generating priority update list...{Colors.RESET}") - - # Categorize issues by priority - critical = [] # Invalid snippets, missing abstracts - high = [] # Missing snippets - medium = [] # Missing explanations/evidence_source - - for result in self.validation_results: - if not result.abstract_fetched or result.evidence.snippet and not result.snippet_valid: - critical.append(result) - elif not result.evidence.snippet and result.abstract_text: - high.append(result) - elif not result.evidence.explanation or not result.evidence.evidence_source: - medium.append(result) - - with open(output_path, "w") as f: - f.write("PRIORITY LITERATURE UPDATES\n") - f.write("=" * 80 + "\n\n") - - f.write(f"CRITICAL ({len(critical)} items): Invalid/missing evidence\n") - f.write("-" * 80 + "\n\n") - for result in critical[:20]: # Top 20 - f.write(f"File: {result.evidence.file}\n") - f.write(f"Reference: {result.evidence.reference}\n") - f.write(f"Context: {result.evidence.context} - {result.evidence.organism}\n") - f.write(f"Issues: {', '.join(result.issues)}\n\n") - - f.write(f"\nHIGH ({len(high)} items): Missing snippets (abstract available)\n") - f.write("-" * 80 + "\n\n") - for result in high[:20]: # Top 20 - f.write(f"File: {result.evidence.file}\n") - f.write(f"Reference: {result.evidence.reference}\n") - f.write(f"Context: {result.evidence.context} - {result.evidence.organism}\n") - f.write(f"Abstract available: {len(result.abstract_text)} chars\n\n") - - f.write(f"\nMEDIUM ({len(medium)} items): Missing metadata\n") - f.write("-" * 80 + "\n\n") - for result in medium[:20]: # Top 20 - f.write(f"File: {result.evidence.file}\n") - f.write(f"Reference: {result.evidence.reference}\n") - f.write("Missing: ") - if not result.evidence.explanation: - f.write("explanation ") - if not result.evidence.evidence_source: - f.write("evidence_source") - f.write("\n\n") - - print(f"{Colors.GREEN}✓{Colors.RESET} Priority list written: {output_path}\n") - - def print_summary(self) -> None: - """Print summary statistics""" - print(f"{Colors.BOLD}{Colors.CYAN}LITERATURE REVIEW SUMMARY{Colors.RESET}") - print("=" * 80) - print(f"Total evidence items: {len(self.evidence_items)}") - print( - f"Abstracts fetched: {self.stats['abstracts_fetched']} ({self.stats['abstracts_fetched']/len(self.evidence_items)*100:.1f}%)" - ) - print( - f"Abstracts failed: {self.stats['abstracts_failed']} ({self.stats['abstracts_failed']/len(self.evidence_items)*100:.1f}%)" - ) - print() - print(f"Snippets valid: {self.stats['snippets_valid']}") - print(f"Snippets invalid: {self.stats['snippets_invalid']}") - print(f"Missing snippets: {self.stats['missing_snippets']}") - print() - print( - f"PDFs available: {self.stats['pdfs_available']} ({self.stats['pdfs_available']/len(self.evidence_items)*100:.1f}%)" - ) - print("=" * 80) - - -def main(): - parser = argparse.ArgumentParser( - description="Review and validate literature evidence in CommunityMech" - ) - parser.add_argument("--community", help="Review specific community file") - parser.add_argument("--download-pdfs", action="store_true", help="Download full PDFs") - parser.add_argument("--no-fallback", action="store_true", help="Disable scihub fallback") - parser.add_argument( - "--output", default=_REPORTS / "literature_review_report.txt", help="Output report file" - ) - - args = parser.parse_args() - - # Paths - kb_dir = Path("kb/communities") - - # Initialize reviewer - reviewer = LiteratureReviewer( - kb_dir=kb_dir, download_pdfs=args.download_pdfs, use_fallback=not args.no_fallback - ) - - print(f"{Colors.BOLD}{Colors.CYAN}CommunityMech Literature Review Tool{Colors.RESET}") - print(f"{Colors.CYAN}Download PDFs: {args.download_pdfs}{Colors.RESET}") - print(f"{Colors.CYAN}Use fallback (scihub): {not args.no_fallback}{Colors.RESET}\n") - - # Scan communities - reviewer.scan_all_communities() - - # Validate evidence - reviewer.validate_all_evidence() - - # Generate reports - reviewer.generate_report(Path(args.output)) - reviewer.generate_priority_update_list(Path("priority_literature_updates.txt")) - - # Print summary - print() - reviewer.print_summary() - - print(f"\n{Colors.GREEN}{Colors.BOLD}✓ Literature review complete!{Colors.RESET}") - print("\nReports generated:") - print(f" - {args.output}") - print(" - priority_literature_updates.txt") - - -if __name__ == "__main__": - main() diff --git a/scripts/test_pdf_fetching.py b/scripts/test_pdf_fetching.py deleted file mode 100644 index 62e94c3a..00000000 --- a/scripts/test_pdf_fetching.py +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env python3 -""" -Test PDF Fetching Core Capability - -Tests all 6 tiers of the PDF cascade: -1. Publisher direct -2. PubMed Central -3. Unpaywall API -4. Semantic Scholar -5. Scihub fallback mirrors -6. Web search - -Validates configuration, success rates, and fallback behavior. - -.. warning:: - **This script has never run.** It imports - ``communitymech.literature_enhanced``, which has never existed in any commit - of this repository — this file was added in 7c658e6 already importing it, so - there is no revision at which it worked (#410). - - It is not a one-line fix, and this one may not be portable at all: it calls - ``fetch_pdf_url(doi)``, and ``communitymech.literature.LiteratureFetcher`` has - no such method. The nearest thing is ``fetch_paper``, which returns a - ``(abstract, pdf_url)`` tuple — so the URL is reachable, but the PDF - downloading this script is built around is not implemented anywhere. - - For what it was meant to do, use the tooling that works: - ``just validate-references FILE`` and ``just validate-references-all`` for - snippet checking — these run the official ``linkml-reference-validator`` - against ``conf/reference_validator.yaml``, the custom validator having been - replaced in 4dd299a — and the ``evidence-curation`` skill for curating and - repairing evidence. - - Kept rather than deleted so the intent is recoverable, and pinned as broken by - ``tests/test_scripts_import.py``. Whether to port or delete is #410. - -""" - -import sys -import time -from pathlib import Path - -import yaml - -sys.path.insert(0, str(Path(__file__).parent.parent / "src")) - -from communitymech.literature_enhanced import EnhancedLiteratureFetcher - - -class PDFFetchingTester: - """Comprehensive PDF fetching test suite""" - - def __init__(self): - # Test with fallback enabled - self.fetcher_with_fallback = EnhancedLiteratureFetcher( - cache_dir=".literature_cache", use_fallback_pdf=True - ) - - # Test without fallback - self.fetcher_no_fallback = EnhancedLiteratureFetcher( - cache_dir=".literature_cache", use_fallback_pdf=False - ) - - self.results = { - "publisher": [], - "pmc": [], - "unpaywall": [], - "semantic_scholar": [], - "fallback_mirror": [], - "web_search": [], - "failed": [], - } - - def extract_dois_from_kb(self, max_dois: int = 20) -> list[str]: - """Extract sample DOIs from knowledge base""" - - kb_dir = Path("kb/communities") - yaml_files = sorted(kb_dir.glob("*.yaml")) - - dois = [] - seen = set() - - for yaml_path in yaml_files: - if len(dois) >= max_dois: - break - - with open(yaml_path) as f: - data = yaml.safe_load(f) - - # Check all evidence - for section in ["taxonomy", "ecological_interactions", "environmental_factors"]: - if section not in data: - continue - - items = data[section] - for item in items: - if "evidence" not in item: - continue - - for ev in item["evidence"]: - ref = ev.get("reference", "") - - if ref.startswith("doi:"): - doi = ref[4:] - if doi not in seen: - dois.append(doi) - seen.add(doi) - - if len(dois) >= max_dois: - return dois - - return dois - - def test_single_doi( - self, doi: str, use_fallback: bool = False - ) -> tuple[str | None, str | None]: - """ - Test PDF fetching for a single DOI. - - Returns: (pdf_url, source_tier) - """ - - fetcher = self.fetcher_with_fallback if use_fallback else self.fetcher_no_fallback - - try: - result = fetcher.fetch_pdf_url(doi) - if result: - pdf_url, source = result - return (pdf_url, source) - return (None, None) - except Exception as e: - print(f" Error: {e}") - return (None, None) - - def test_cascade(self, dois: list[str], use_fallback: bool = True): - """Test PDF cascade for multiple DOIs""" - - print(f"\nTesting {'WITH' if use_fallback else 'WITHOUT'} fallback mirrors...") - print("=" * 80) - - for i, doi in enumerate(dois, 1): - print(f"\n[{i}/{len(dois)}] Testing {doi}") - print("-" * 80) - - pdf_url, source = self.test_single_doi(doi, use_fallback) - - if source: - self.results[source].append({"doi": doi, "pdf_url": pdf_url}) - print(f"✓ SUCCESS via {source}") - else: - self.results["failed"].append(doi) - print("✗ FAILED - No PDF found") - - # Rate limit - time.sleep(1) - - def test_scihub_config(self): - """Test scihub fallback configuration""" - - print("\n" + "=" * 80) - print("SCIHUB FALLBACK CONFIGURATION") - print("=" * 80) - - print(f"\nFallback enabled: {self.fetcher_with_fallback.use_fallback_pdf}") - print(f"Fallback mirrors configured: {len(self.fetcher_with_fallback.fallback_pdf_urls)}") - - if self.fetcher_with_fallback.fallback_pdf_urls: - print("\nConfigured mirrors:") - for mirror in self.fetcher_with_fallback.fallback_pdf_urls: - print(f" - {mirror}") - else: - print("\n⚠ WARNING: No fallback mirrors configured!") - print("Set FALLBACK_PDF_MIRRORS environment variable") - - print() - - def test_html_parsing(self): - """Test scihub HTML parsing with sample HTML""" - - print("\n" + "=" * 80) - print("SCIHUB HTML PARSING TEST") - print("=" * 80) - - # Sample scihub HTML patterns - test_cases = [ - # Pattern 1: tag - """ - - - """, - # Pattern 2: tag - """ - Download PDF - """, - # Pattern 3: tag - """ - - """, - # Pattern 4: - """, - ] - - base_url = "https://sci-hub.se" - - for i, html in enumerate(test_cases, 1): - print(f"\nTest case {i}:") - pdf_url = self.fetcher_with_fallback._extract_pdf_from_fallback_html(html, base_url) - if pdf_url: - print(f" ✓ Extracted: {pdf_url}") - else: - print(" ✗ Failed to extract PDF URL") - - def generate_report(self): - """Generate comprehensive test report""" - - print("\n" + "=" * 80) - print("PDF FETCHING TEST RESULTS") - print("=" * 80) - - total = sum(len(items) for source, items in self.results.items() if source != "failed") - total_tested = total + len(self.results["failed"]) - - print(f"\nTotal DOIs tested: {total_tested}") - print(f"Successful: {total} ({100*total/total_tested if total_tested > 0 else 0:.1f}%)") - print( - f"Failed: {len(self.results['failed'])} ({100*len(self.results['failed'])/total_tested if total_tested > 0 else 0:.1f}%)" - ) - - print("\n" + "-" * 80) - print("SUCCESS BY TIER") - print("-" * 80) - - tier_order = [ - "publisher", - "pmc", - "unpaywall", - "semantic_scholar", - "fallback_mirror", - "web_search", - ] - - for tier in tier_order: - count = len(self.results[tier]) - pct = 100 * count / total_tested if total_tested > 0 else 0 - - status = "✓" if count > 0 else "✗" - print( - f"{status} Tier {tier_order.index(tier)+1} ({tier:20s}): {count:3d} ({pct:5.1f}%)" - ) - - # Show examples - print("\n" + "-" * 80) - print("EXAMPLES") - print("-" * 80) - - for tier in tier_order: - if self.results[tier]: - print(f"\n{tier.upper()} (showing 3 examples):") - for item in self.results[tier][:3]: - print(f" {item['doi']}") - print(f" → {item['pdf_url'][:80]}...") - if len(self.results[tier]) > 3: - print(f" ... and {len(self.results[tier])-3} more") - - # Show failures - if self.results["failed"]: - print("\nFAILED (showing 5 examples):") - for doi in self.results["failed"][:5]: - print(f" {doi}") - if len(self.results["failed"]) > 5: - print(f" ... and {len(self.results['failed'])-5} more") - - def test_specific_publishers(self): - """Test publisher-specific PDF patterns""" - - print("\n" + "=" * 80) - print("PUBLISHER-SPECIFIC PATTERN TEST") - print("=" * 80) - - test_dois = { - "ASM (American Society for Microbiology)": "10.1128/AEM.00001-20", - "PLOS": "10.1371/journal.pone.0123456", - "Frontiers": "10.3389/fmicb.2020.00001", - "MDPI": "10.3390/microorganisms8010001", - "Nature": "10.1038/s41586-020-0001-0", - "Science": "10.1126/science.aaa0001", - "Elsevier": "10.1016/j.cell.2020.01.001", - } - - print("\nPublisher pattern coverage:") - for publisher, doi_prefix in test_dois.items(): - # Check if pattern exists in code - has_pattern = ( - self.fetcher_with_fallback._get_pdf_url_from_publisher(doi_prefix) is not None - ) - status = "✓" if has_pattern else "✗" - print(f" {status} {publisher}: {doi_prefix}") - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Test PDF fetching capabilities") - parser.add_argument("--quick", action="store_true", help="Quick test with 5 DOIs") - parser.add_argument("--full", action="store_true", help="Full test with 20 DOIs") - parser.add_argument("--no-fallback", action="store_true", help="Test without scihub fallback") - parser.add_argument("--config-only", action="store_true", help="Only test configuration") - args = parser.parse_args() - - tester = PDFFetchingTester() - - print("PDF FETCHING CORE CAPABILITY TEST") - print("=" * 80) - - # Always test configuration - tester.test_scihub_config() - tester.test_html_parsing() - - if args.config_only: - print("\n✓ Configuration test complete") - return - - # Determine test size - if args.quick: - max_dois = 5 - elif args.full: - max_dois = 20 - else: - max_dois = 10 # default - - # Extract DOIs from KB - print(f"\nExtracting {max_dois} DOIs from knowledge base...") - dois = tester.extract_dois_from_kb(max_dois) - print(f"Found {len(dois)} DOIs to test") - - if not dois: - print("⚠ No DOIs found in knowledge base") - return - - # Test with/without fallback - use_fallback = not args.no_fallback - tester.test_cascade(dois, use_fallback=use_fallback) - - # Test publisher patterns - tester.test_specific_publishers() - - # Generate report - tester.generate_report() - - print("\n" + "=" * 80) - print("RECOMMENDATIONS") - print("=" * 80) - - if len(tester.results["fallback_mirror"]) == 0 and use_fallback: - print("\n⚠ No PDFs found via fallback mirrors") - print(" Possible reasons:") - print(" 1. Scihub mirrors may be blocked/unavailable") - print(" 2. HTML parsing patterns may need update") - print(" 3. Mirrors may have changed URLs") - print("\n Try:") - print(" - Check FALLBACK_PDF_MIRRORS environment variable") - print(" - Test individual mirrors manually") - print(" - Update HTML parsing patterns if needed") - - success_rate = ( - 100 - * sum(len(items) for source, items in tester.results.items() if source != "failed") - / len(dois) - ) - - if success_rate < 50: - print("\n⚠ Low success rate detected") - print(" Suggestions:") - print(" 1. Enable scihub fallback (if disabled)") - print(" 2. Add more publisher-specific patterns") - print(" 3. Check network connectivity") - elif success_rate > 80: - print("\n✓ Good PDF discovery rate!") - print(" Core capability is working well") - - print() - - -if __name__ == "__main__": - main() diff --git a/tests/test_batch_snippet_fixer_validation.py b/tests/test_batch_snippet_fixer_validation.py index 2235b0d5..6aa6ce1d 100644 --- a/tests/test_batch_snippet_fixer_validation.py +++ b/tests/test_batch_snippet_fixer_validation.py @@ -84,21 +84,51 @@ def test_the_validator_it_calls_can_actually_start(fixer): def _known_broken() -> set[str]: - """The five scripts `tests/test_scripts_import.py` records as unrunnable.""" + """The scripts that cannot run, plus the ones #410 removed for that reason. + + Sourced from `test_scripts_import.py` so the two cannot drift, and unioned + with the removed names: a working tool printing "next, run + `curate_evidence_with_pdfs.py`" is just as dead a pointer now that the file + is gone as it was when the file existed and could not start. + """ source = (REPO / "tests/test_scripts_import.py").read_text() tree = ast.parse(source) for node in ast.walk(tree): - if isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == "_KNOWN_BROKEN" for target in node.targets - ): - return { + # `AnnAssign` as well as `Assign`: the constant became + # `_KNOWN_BROKEN: set[str] = set()` when #410 emptied it, and an + # `Assign`-only walk stopped finding it — the check went red rather + # than silently passing, which is the behaviour worth keeping. + if isinstance(node, ast.AnnAssign): + matches = isinstance(node.target, ast.Name) and node.target.id == "_KNOWN_BROKEN" + elif isinstance(node, ast.Assign): + matches = any( + isinstance(target, ast.Name) and target.id == "_KNOWN_BROKEN" + for target in node.targets + ) + else: + continue + if matches and node.value is not None: + listed = { element.value for element in ast.walk(node.value) if isinstance(element, ast.Constant) and isinstance(element.value, str) } + return listed | _REMOVED_IN_410 raise AssertionError("_KNOWN_BROKEN is gone from tests/test_scripts_import.py") +# Removed in #410 rather than ported: each imported a module that never existed, +# and each had a working replacement (`scripts/cache_fulltext.py` for OA full +# text, `just validate-references` for snippet checking). +_REMOVED_IN_410 = { + "curate_evidence_with_pdfs.py", + "extract_evidence_snippets.py", + "quick_literature_review.py", + "review_literature.py", + "test_pdf_fetching.py", +} + + def _strings_in(node: ast.AST) -> list[str]: """Every string literal reachable inside a call, f-strings included.""" found = [] @@ -120,7 +150,11 @@ def test_no_working_script_points_at_a_script_that_cannot_run(): # named three of the five, so a pointer at `test_pdf_fetching` or # `extract_evidence_snippets` sailed through (#487 review). dead = {name.removesuffix(".py") for name in _known_broken()} - assert len(dead) >= 5, f"the known-broken list has shrunk unexpectedly: {sorted(dead)}" + # Was `>= 5` against `_KNOWN_BROKEN` alone. That list is empty since #410 + # removed the scripts, so the bound now rests on `_REMOVED_IN_410` — a + # gone file is still a dead pointer, and this check would otherwise have + # quietly started passing on nothing. + assert len(dead) >= 5, f"the dead-pointer list has shrunk unexpectedly: {sorted(dead)}" offenders = [] for path in sorted((REPO / "scripts").glob("*.py")): if path.stem in dead: diff --git a/tests/test_scripts_import.py b/tests/test_scripts_import.py index 3aeaa3ff..b9a8d2f4 100644 --- a/tests/test_scripts_import.py +++ b/tests/test_scripts_import.py @@ -39,23 +39,18 @@ "term_label_audit.py", } -# All import `communitymech.literature_enhanced`, which has never existed in any -# commit on any branch — verified over all 498 commits. They were added in -# `7c658e6` (the 7th commit, 2026-02-18; the repo's first is `79f5196`). +# Empty, and kept rather than deleted: #410 was about five scripts that imported +# `communitymech.literature_enhanced`, a module that never existed in any of the +# repo's 498 commits. They were removed rather than ported, because porting meant +# deciding whether to keep a capability their CLI flags advertised and the code +# never had — a 6-tier PDF cascade with "fallback mirrors". Answering "no": the +# OA full-text need is served by `scripts/cache_fulltext.py`, which works and +# which the #183 sweep used to cache 64 of 125 references. # -# Not a curation call in general: #88 already ported the same import in two other -# scripts, and `fix_invalid_snippets.py` followed that recipe here because it -# passed `download_pdf=False` throughout, so nothing was lost. These five pass a -# *variable* for PDF fetching, or call `fetch_pdf_url`, and `LiteratureFetcher` -# has no PDF surface at all — porting them means deciding whether to drop a -# capability their CLI flags advertise. That decision is #410, which stays open. -_KNOWN_BROKEN = { - "curate_evidence_with_pdfs.py", - "extract_evidence_snippets.py", - "quick_literature_review.py", - "review_literature.py", - "test_pdf_fetching.py", -} +# The name stays so the tests below keep their shape if a script ever breaks +# this way again. `test_no_script_imports_a_module_that_does_not_exist` is the +# real guard now, and unlike this list it cannot go stale. +_KNOWN_BROKEN: set[str] = set() # `python scripts/foo.py` puts `scripts/` on sys.path, which is how the sibling @@ -213,75 +208,81 @@ def test_a_script_parses(script: Path): ast.parse(script.read_text()) -def test_the_known_broken_list_is_still_accurate(): - """A name that starts importing must leave the list, or the list rots. +def test_no_script_imports_a_module_that_does_not_exist(): + """The invariant #410 actually wanted, in place of a list of exceptions. + + The old guard was `_KNOWN_BROKEN`, five names plus three tests keeping the + list honest. That records breakage; it does not prevent it, and a sixth + script importing a sixth phantom module would simply have been added to it. - Without this, fixing one of the six would leave it permanently exempt from - the check above — the failure mode of every allowlist. + This resolves every `from communitymech.X import ...` in `scripts/` against + the installed package. It is deliberately narrow — only this package, not + third-party imports, which fail for environment reasons a test should not + adjudicate. """ - fixed = [] - for name in sorted(_KNOWN_BROKEN): - path = SCRIPTS / name - if not path.exists(): - continue - result = subprocess.run( - [ - sys.executable, - "-c", - _PROBE.format(path=str(path), scripts=str(SCRIPTS)), - ], - capture_output=True, - text=True, - cwd=REPO, - timeout=120, - ) - if result.returncode == 0: - fixed.append(name) - assert not fixed, f"these now import and should be removed from _KNOWN_BROKEN: {fixed} (#410)" + import importlib.util + + offenders = [] + for path in sorted(SCRIPTS.glob("*.py")): + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if not isinstance(node, ast.ImportFrom) or not node.module: + continue + if not node.module.startswith("communitymech"): + continue + try: + found = importlib.util.find_spec(node.module) + except (ImportError, ValueError): + found = None + if found is None: + offenders.append(f"{path.name}:{node.lineno} imports {node.module}") + assert offenders == [], ( + "these scripts import a `communitymech` module that cannot be " + "resolved, so they fail before `--help` (#410):\n" + "\n".join(offenders) + ) -def test_the_known_broken_all_share_one_cause(): - """The list is one bug, not a bucket. +def test_that_guard_can_actually_fail(tmp_path): + """Mutation check, in-tree: the sweep above must reject a phantom import. - If a script joins it for a different reason, that reason wants its own issue - rather than being absorbed into #410's count. + Written because the guard it replaces was a list — and a list-driven test + passes cleanly once the list is empty, which is exactly the state this file + is now in. Without this, `test_no_script_imports_a_module_that_does_not_exist` + could resolve nothing at all and still report success. """ - for name in sorted(_KNOWN_BROKEN): - path = SCRIPTS / name - if not path.exists(): - continue - imports = { - node.module - for node in ast.walk(ast.parse(path.read_text())) - if isinstance(node, ast.ImportFrom) and node.module - } - assert any( - "literature_enhanced" in module for module in imports - ), f"{name} is in _KNOWN_BROKEN but does not import literature_enhanced" - - -def test_every_known_broken_script_says_so_in_its_docstring(): - """A reader opening the file should not have to run it to find out. - - `_KNOWN_BROKEN` records the breakage for the *suite*; it is invisible to - someone who opens `curate_evidence_with_pdfs.py` and sees 586 lines of - plausible code. Each carries a docstring warning naming the phantom module, - why porting is not an import swap, and what to use instead (#410). - - Asserted against `_KNOWN_BROKEN` rather than a fixed list, so a script - joining it later cannot arrive undocumented. + import importlib.util + + assert importlib.util.find_spec("communitymech.literature") is not None + try: + missing = importlib.util.find_spec("communitymech.literature_enhanced") + except (ImportError, ValueError): + missing = None + assert missing is None, ( + "`communitymech.literature_enhanced` now resolves. If it was genuinely " + "implemented, #410 can be revisited; if something is shadowing the " + "package, the guard above is not testing what it claims" + ) + + +def test_the_removed_scripts_are_gone_and_stay_gone(): + """They are superseded, and a reintroduction should be deliberate. + + Each had a working replacement by the time it was removed: OA full text via + `cache_fulltext.py`, snippet checking via `just validate-references`. A file + reappearing under one of these names is most likely a revert that did not + mean to bring back a script that cannot start. """ - undocumented = [] - for name in sorted(_KNOWN_BROKEN): - path = SCRIPTS / name - if not path.exists(): - continue - docstring = ast.get_docstring(ast.parse(path.read_text())) or "" - if "has never run" not in docstring or "literature_enhanced" not in docstring: - undocumented.append(name) - assert not undocumented, ( - "these are in _KNOWN_BROKEN but their docstrings do not say so — a " - f"reader would take them for working tools (#410): {undocumented}" + removed = { + "curate_evidence_with_pdfs.py", + "extract_evidence_snippets.py", + "quick_literature_review.py", + "review_literature.py", + "test_pdf_fetching.py", + } + back = sorted(name for name in removed if (SCRIPTS / name).exists()) + assert back == [], ( + f"{back} were removed in #410 as unrunnable and superseded. If one is " + "genuinely wanted again it needs a working literature backend first — " + "see scripts/cache_fulltext.py." ) @@ -304,3 +305,46 @@ def test_the_replacement_named_in_those_docstrings_exists(): "the docstrings contrast the phantom fetch_paper with this one; if it is " "gone or renamed, they now describe nothing" ) + + +def test_docs_do_not_reference_a_script_that_is_not_there(): + """Nothing walked `docs/` for script references until #523. + + #410's guard and its replacement both check `scripts/` — `print` and + `subprocess` calls in Python files. A curator following a runbook is reading + prose, which is precisely where neither looks. The five removed scripts were + named in three docs, and only a hand grep found them. + + Deliberately checks *existence*, not tone. An earlier version of this idea + tried to flag "unwarned" instructions and I filed #523 against + `AUTOMATION_TOOLS.md` on that basis — wrongly, because the warning sat two + lines above the command and a line-scoped scan could not see it. Whether a + reference is adequately caveated is a judgement; whether the file exists is + a fact, and only the fact belongs in a test. + """ + import re + + docs = REPO / "docs" + # The one file whose job is to record what was removed. It carries a + # document-level banner — "everything below this line describes software + # that was never in this repository" — which a per-line check cannot see, + # and rewriting it line by line would destroy the record it exists to keep. + # Every other doc has to keep its references live. + historical = {"pdf_fetching_capability.md"} + pattern = re.compile(r"scripts/([A-Za-z0-9_]+\.py)") + missing = [] + for doc in sorted(docs.rglob("*.md")): + if doc.name in historical: + continue + for number, line in enumerate(doc.read_text(encoding="utf-8").splitlines(), 1): + for name in pattern.findall(line): + if (SCRIPTS / name).exists(): + continue + # A line may legitimately name a removed script while saying so. + if any(word in line for word in ("REMOVED", "removed", "deleted")): + continue + missing.append(f"{doc.relative_to(REPO)}:{number} -> scripts/{name}") + assert missing == [], ( + "these docs reference a script that is not in `scripts/`, without " + "saying it was removed (#523):\n" + "\n".join(missing) + )