TIMX 655 - handle deletes for researchdatabases (AZ)

README.md

@@ -59,6 +59,7 @@ WORKSPACE=### Set to `dev` for local development, this will be set to `stage` an
 WARNING_ONLY_LOGGERS=### Comma-seperated list of logger names to set as WARNING only, e.g. 'botocore,charset_normalizer,smart_open'
 LIBGUIDES_API_TOKEN=### Libguides API token [required for libguides source]
 LIBGUIDES_CLIENT_ID=### Libguides account id [required for libguides source] 
+TIMDEX_DATASET_LOCATION=### Location of the TIMDEX dataset 
 ```
 
 ## CLI commands

tests/fixtures/researchdatabases/researchdatabases-2026-08-11-full-extracted-records-to-index.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+    <responseDate>2026-08-11T09:30:27Z</responseDate>
+    <request verb="ListRecords" metadataPrefix="oai_dc" set="az">
+        https://libguides.mit.edu/oai.php
+    </request>
+    <ListRecords>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257807</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title>
+                        <![CDATA[Linguistics and Language Behavior Abstracts (LLBA)]]></dc:title>
+                    <dc:subject><![CDATA[Humanities]]></dc:subject>
+                    <dc:type><![CDATA[Articles]]></dc:type>
+                    <dc:type><![CDATA[Dissertations & theses]]></dc:type>
+                    <dc:type><![CDATA[eBooks]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p>The most comprehensive index to articles in Linguistics and Language Development and use.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/llba</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257808</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title><![CDATA[ADS (NASA Astrophysics Data System)]]></dc:title>
+                    <dc:subject><![CDATA[Engineering]]></dc:subject>
+                    <dc:subject><![CDATA[Science]]></dc:subject>
+                    <dc:type><![CDATA[Articles]]></dc:type>
+                    <dc:type><![CDATA[Data & statistics]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p>Indexes major astronomy and astrophysics journals; includes abstracts for most entries and full text scans of tens of thousands of articles.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/ads</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+        <record>
+            <header>
+                <identifier>oai:libguides.com:az/65257809</identifier>
+                <datestamp>2025-12-01T14:59:03Z</datestamp>
+                <setSpec>az</setSpec>
+            </header>
+            <metadata>
+                <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
+                           xmlns:dc="http://purl.org/dc/elements/1.1/"
+                           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                           xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
+                    <dc:title><![CDATA[Ethnic NewsWatch]]></dc:title>
+                    <dc:subject><![CDATA[Social sciences]]></dc:subject>
+                    <dc:type><![CDATA[Newspapers & magazine articles]]></dc:type>
+                    <dc:type><![CDATA[Primary sources]]></dc:type>
+                    <dc:description>
+                        <![CDATA[<p><strong>Coverage:</strong> 1990 - present<br>A full-text collection of newspapers, journals and magazines published by ethnic and minority presses.</p>]]></dc:description>
+                    <dc:date>2022-01-28 22:15:37</dc:date>
+                    <dc:identifier>https://libguides.mit.edu/ethnic</dc:identifier>
+                </oai_dc:dc>
+            </metadata>
+        </record>
+    </ListRecords>
+</OAI-PMH>

tests/sources/json/test_libguides.py

@@ -401,7 +401,7 @@ def test_libguides_api_client_fetch_guides_expands_sub_pages_into_rows():
     mock_response.json.return_value = mock_api_response
 
     with patch(
-        "transmogrifier.sources.json.libguides.requests.get",
+        "transmogrifier.helpers.requests.get",
         return_value=mock_response,
     ):
         df = client.fetch_guides("fake-token")

tests/sources/xml/test_researchdatabases.py

@@ -0,0 +1,147 @@
+# ruff: noqa: PLR2004, SLF001
+
+from unittest.mock import patch
+
+import pytest
+from bs4 import Tag
+
+from transmogrifier.helpers import LibGuidesAPIClient
+from transmogrifier.sources.xml.researchdatabases import ResearchDatabases
+
+
+@pytest.fixture(autouse=True)
+def _test_env_libguides():
+    with (
+        patch("transmogrifier.config.LIBGUIDES_CLIENT_ID", "123"),
+        patch("transmogrifier.config.LIBGUIDES_API_TOKEN", "aaabbbdddccc"),
+        patch("transmogrifier.config.TIMDEX_DATASET_LOCATION", "s3://timdex/dataset"),
+    ):
+        yield
+
+
+@pytest.fixture
+def mocked_current_az_identifiers():
+    """Mock current AZ identifiers from LibGuides API, none matching previous ones."""
+    with patch.object(
+        LibGuidesAPIClient,
+        "get_current_az_identifiers",
+        return_value=["7777", "8888", "9999"],
+    ):
+        yield
+
+
+@pytest.fixture
+def mocked_timdex_dataset_az_identifiers():
+    with patch.object(
+        ResearchDatabases,
+        "_get_current_dataset_az_identifiers",
+        return_value=[
+            "1234",  # mocked as present only in dataset, not current/public
+            "7777",
+            "8888",
+            "9999",
+        ],
+    ):
+        yield
+
+
+@pytest.fixture
+def researchdatabases_transformer():
+
+    return ResearchDatabases.load(
+        "researchdatabases",
+        (
+            "tests/fixtures/researchdatabases/researchdatabases-"
+            "2026-08-11-full-extracted-records-to-index.xml"
+        ),
+    )
+
+
+def test_researchdatabases_yields_oai_records_pass(researchdatabases_transformer):
+
+    oai_records = list(
+        researchdatabases_transformer._yield_oai_xml_records_for_indexing(
+            researchdatabases_transformer.source_file
+        )
+    )
+
+    assert len(oai_records) == 3
+    assert isinstance(oai_records[0], Tag)
+
+
+def test_researchdatabases_yields_deleted_records_success(
+    researchdatabases_transformer,
+    mocked_timdex_dataset_az_identifiers,
+    mocked_current_az_identifiers,
+):
+    deleted_records = list(
+        researchdatabases_transformer._yield_api_records_for_deleting()
+    )
+    assert len(deleted_records) == 1
+    assert isinstance(deleted_records[0], Tag)
+
+
+def test_researchdatabases_synthetic_deleted_record(
+    researchdatabases_transformer,
+    mocked_timdex_dataset_az_identifiers,
+    mocked_current_az_identifiers,
+):
+    """Assert that Transformer injects synthetic OAI delete records.
+
+    Example record:
+
+    <?xml version="1.0" encoding="utf-8"?>
+    <record>
+        <header status="deleted">
+            <identifier>oai:libguides.com:az/1234</identifier>
+            <datestamp>2026-08-11T14:13:38Z</datestamp>
+            <setSpec>az</setSpec>
+        </header>
+        <note>This is a synthetic delete record created by Transmogrifier.</note>
+        <metadata/>
+    </record>
+    """
+    deleted_record = next(researchdatabases_transformer._yield_api_records_for_deleting())
+
+    header = deleted_record.find("record").find("header")
+
+    assert header.get("status") == "deleted"
+    assert header.find("identifier").string == "oai:libguides.com:az/1234"
+    assert (
+        deleted_record.find("note").string
+        == "This is a synthetic delete record created by Transmogrifier."
+    )
+
+
+@pytest.mark.parametrize(
+    "env_var",
+    [
+        "LIBGUIDES_API_TOKEN",
+        "LIBGUIDES_CLIENT_ID",
+        "TIMDEX_DATASET_LOCATION",
+    ],
+)
+def test_researchdatabases_env_vars_not_set_skips_deletes(
+    env_var,
+    caplog,
+    researchdatabases_transformer,
+    mocked_current_az_identifiers,
+):
+    caplog.set_level("WARNING")
+    with patch(f"transmogrifier.config.{env_var}", None):
+        assert list(researchdatabases_transformer._yield_api_records_for_deleting()) == []
+
+    assert (
+        "Skipping deleting record detection, not all required env vars are set."
+        in caplog.text
+    )
+
+
+def test_research_databases_custom_parse_source_file_yields_oai_and_synthetic_records(
+    researchdatabases_transformer,
+    mocked_timdex_dataset_az_identifiers,
+    mocked_current_az_identifiers,
+):
+    records = list(researchdatabases_transformer.source_records)
+
+    assert len(records) == 4  # 3 OAI + 1 synthetic delete

transmogrifier/config.py

@@ -127,7 +127,7 @@
     "researchdatabases": {
         "name": "Research Databases",
         "base-url": "https://libguides.mit.edu/",
-        "transform-class": "transmogrifier.sources.xml.springshare.SpringshareOaiDc",
+        "transform-class": "transmogrifier.sources.xml.researchdatabases.ResearchDatabases",  # noqa: E501
     },
     "whoas": {
         "name": "Woods Hole Open Access Server",
@@ -149,6 +149,7 @@
 )
 LIBGUIDES_API_TOKEN = os.getenv("LIBGUIDES_API_TOKEN")
 LIBGUIDES_CLIENT_ID = os.getenv("LIBGUIDES_CLIENT_ID")
+TIMDEX_DATASET_LOCATION = os.getenv("TIMDEX_DATASET_LOCATION")
 
 
 def configure_logger(

transmogrifier/helpers.py

@@ -1,7 +1,12 @@
 import logging
+import re
 from datetime import UTC, datetime
 
+import pandas as pd
+import requests
+
 import transmogrifier.models as timdex
+from transmogrifier import config
 from transmogrifier.config import DATE_FORMATS
 
 logger = logging.getLogger(__name__)
@@ -131,3 +136,111 @@ def validate_date_range(
         end_date,
     )
     return False
+
+
+class LibGuidesAPIClient:
+    """Client for LibGuides API communication and data retrieval.
+
+    This class retrieves metadata about all LibGuides via an API, retrieving data that is
+    not found in the OAI-PMH XML records or the websites themselves.  This valuable data
+    is used during transformation to identify records for exclusion, occasionally
+    provide friendlier URLs, and other data augmentation.
+
+    This class is instantiated as a singleton object in this module.  Once instantiated,
+    it is attached to the Libguides transformer instance.  This allows class methods
+    on the transformer to access cached data from this singleton object, ultimately
+    resulting in only a single API call per multiple record transformation run.
+
+    This class relies on two environment variables:
+        - LIBGUIDES_CLIENT_ID
+        - LIBGUIDES_API_TOKEN
+    """
+
+    def __init__(self) -> None:
+        if not config.LIBGUIDES_CLIENT_ID:
+            raise RuntimeError("Required env var 'LIBGUIDES_CLIENT_ID' is not set")
+        if not config.LIBGUIDES_API_TOKEN:
+            raise RuntimeError("Required env var 'LIBGUIDES_API_TOKEN' is not set")
+
+        self.client_id = str(config.LIBGUIDES_CLIENT_ID)
+        self.client_secret = config.LIBGUIDES_API_TOKEN
+        self._api_guides_df: pd.DataFrame | None = None
+
+    @property
+    def api_guides_df(self) -> pd.DataFrame:
+        if self._api_guides_df is None:
+            self._api_guides_df = self.fetch_guides(self.get_api_token())
+        return self._api_guides_df
+
+    def get_api_token(self) -> str:
+        data = {
+            "grant_type": "client_credentials",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+        }
+        response = requests.post(
+            config.LIBGUIDES_TOKEN_URL, headers={}, data=data, timeout=60
+        )
+        response.raise_for_status()
+        payload = response.json()
+        return payload.get("access_token")
+
+    def fetch_guides(self, token: str) -> pd.DataFrame:
+        """Retrieve metadata for all LibGuides.
+
+        Each guide may contain a 'pages' key with a list of sub-page dicts.  These
+        sub-pages are expanded into their own rows in the returned DataFrame, inheriting
+        any columns from the parent guide that the sub-page does not have.
+        """
+        logger.debug("Retrieving all guides from Libguides API.")
+        headers = {"Authorization": f"Bearer {token}"}
+        response = requests.get(config.LIBGUIDES_GUIDES_URL, headers=headers, timeout=60)
+        response.raise_for_status()
+        guides = response.json()
+
+        all_rows: list[dict] = []
+        for guide in guides:
+            pages = guide.get("pages", [])
+            all_rows.append(guide)
+            for page in pages:
+                # inherit parent columns, then overlay page-specific columns
+                page_row = {**guide, **page}
+                all_rows.append(page_row)
+
+        return pd.DataFrame(all_rows)
+
+    def get_guide_by_url(self, url: str) -> pd.Series:
+        """Get metadata for a single guide via a URL."""
+        # strip GET parameter preview=...; duplicate for base URL
+        url = re.sub(r"([&?])preview=.*", "", url)
+        url = url.removesuffix("/")
+
+        matches = self.api_guides_df[
+            (self.api_guides_df.url.str.lower() == url.lower())
+            | (self.api_guides_df.friendly_url.str.lower() == url.lower())
+        ]
+        if len(matches) == 1:
+            return matches.iloc[0]
+
+        raise ValueError(f"Found {len(matches)} guide ids for URL: {url}, expecting one.")
+
+    def fetch_az(self, token: str) -> pd.DataFrame:
+        """Retrieve AZ items from API."""
+        headers = {"Authorization": f"Bearer {token}"}
+        response = requests.get(
+            "https://lgapi-us.libapps.com/1.2/az?expand=pages",
+            headers=headers,
+            timeout=60,
+        )
+        response.raise_for_status()
+        return pd.DataFrame(response.json())
+
+    def get_current_az_identifiers(self) -> list[str]:
+        """Get list of identifiers for non-hidden / public AZ items.
+
+        When filtering to enable_hidden = 0, the count matches the OAI-PMH full harvest
+        for AZ items.
+        """
+        az_df = self.fetch_az(self.get_api_token())
+        non_hidden_az_df = az_df[az_df.enable_hidden == "0"]
+        return list(non_hidden_az_df.id)