From fad99fac2da8c4f0faea599bcbc131cd82ba3223 Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Thu, 20 Aug 2026 15:36:30 +0200 Subject: [PATCH 01/55] first version refactor assisted with AI --- README.md | 2 +- pyproject.toml | 1 + src/mcpdiffusion/config/__init__.py | 1 + src/mcpdiffusion/config/settings.py | 62 ++ src/mcpdiffusion/config/tool_metadata.py | 370 ++++++++++ src/mcpdiffusion/core/__init__.py | 1 + src/mcpdiffusion/core/errors.py | 34 + src/mcpdiffusion/{helpers => core}/logging.py | 32 +- src/mcpdiffusion/core/middleware.py | 60 ++ src/mcpdiffusion/data/__init__.py | 1 + src/mcpdiffusion/data/geography.py | 10 + src/mcpdiffusion/data/indicators.py | 64 ++ src/mcpdiffusion/data/themes.py | 117 +++ src/mcpdiffusion/helpers/__init__.py | 1 - src/mcpdiffusion/helpers/es.py | 65 -- src/mcpdiffusion/helpers/rmes.py | 415 ----------- src/mcpdiffusion/helpers/schemas.py | 54 -- src/mcpdiffusion/infra/__init__.py | 1 + src/mcpdiffusion/infra/elasticsearch.py | 39 + src/mcpdiffusion/infra/http.py | 33 + src/mcpdiffusion/infra/sparql.py | 24 + src/mcpdiffusion/middleware.py | 61 -- src/mcpdiffusion/models/__init__.py | 1 + src/mcpdiffusion/models/feedback.py | 30 + src/mcpdiffusion/models/insee.py | 269 +++++++ src/mcpdiffusion/models/melodi.py | 147 ++++ src/mcpdiffusion/models/rmes.py | 162 +++++ src/mcpdiffusion/server.py | 41 +- src/mcpdiffusion/services/__init__.py | 1 + src/mcpdiffusion/services/feedback.py | 42 ++ src/mcpdiffusion/services/insee_document.py | 189 +++++ .../es_search.py => services/insee_search.py} | 105 +-- src/mcpdiffusion/services/melodi.py | 329 +++++++++ src/mcpdiffusion/services/rmes.py | 524 +++++++++++++ src/mcpdiffusion/tools/__init__.py | 12 +- src/mcpdiffusion/tools/env.py | 686 ------------------ .../tools/extras_send_feedback.py | 97 +-- src/mcpdiffusion/tools/insee_get_document.py | 261 +------ src/mcpdiffusion/tools/insee_get_homepage.py | 37 +- .../tools/insee_search_chiffrecle.py | 91 +-- .../tools/insee_search_conjoncture.py | 81 +-- .../tools/insee_search_documents.py | 90 +-- .../tools/melodi_get_observations.py | 155 +--- .../tools/melodi_search_datasets.py | 182 +---- .../tools/melodi_search_modalities.py | 155 +--- .../tools/rmes_describe_resource.py | 102 +-- src/mcpdiffusion/tools/rmes_list_graphs.py | 135 +--- src/mcpdiffusion/tools/rmes_run_sparql.py | 102 +-- tests/conftest.py | 18 +- tests/test_middleware.py | 105 ++- tests/test_rmes_helpers.py | 30 +- tests/test_rmes_tools.py | 2 +- uv.lock | 2 + 53 files changed, 2720 insertions(+), 2911 deletions(-) create mode 100644 src/mcpdiffusion/config/__init__.py create mode 100644 src/mcpdiffusion/config/settings.py create mode 100644 src/mcpdiffusion/config/tool_metadata.py create mode 100644 src/mcpdiffusion/core/__init__.py create mode 100644 src/mcpdiffusion/core/errors.py rename src/mcpdiffusion/{helpers => core}/logging.py (78%) create mode 100644 src/mcpdiffusion/core/middleware.py create mode 100644 src/mcpdiffusion/data/__init__.py create mode 100644 src/mcpdiffusion/data/geography.py create mode 100644 src/mcpdiffusion/data/indicators.py create mode 100644 src/mcpdiffusion/data/themes.py delete mode 100644 src/mcpdiffusion/helpers/__init__.py delete mode 100644 src/mcpdiffusion/helpers/es.py delete mode 100644 src/mcpdiffusion/helpers/rmes.py delete mode 100644 src/mcpdiffusion/helpers/schemas.py create mode 100644 src/mcpdiffusion/infra/__init__.py create mode 100644 src/mcpdiffusion/infra/elasticsearch.py create mode 100644 src/mcpdiffusion/infra/http.py create mode 100644 src/mcpdiffusion/infra/sparql.py delete mode 100644 src/mcpdiffusion/middleware.py create mode 100644 src/mcpdiffusion/models/__init__.py create mode 100644 src/mcpdiffusion/models/feedback.py create mode 100644 src/mcpdiffusion/models/insee.py create mode 100644 src/mcpdiffusion/models/melodi.py create mode 100644 src/mcpdiffusion/models/rmes.py create mode 100644 src/mcpdiffusion/services/__init__.py create mode 100644 src/mcpdiffusion/services/feedback.py create mode 100644 src/mcpdiffusion/services/insee_document.py rename src/mcpdiffusion/{helpers/es_search.py => services/insee_search.py} (52%) create mode 100644 src/mcpdiffusion/services/melodi.py create mode 100644 src/mcpdiffusion/services/rmes.py delete mode 100644 src/mcpdiffusion/tools/env.py diff --git a/README.md b/README.md index 1b1c646..2d68deb 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ cp mcpdiffusion/.env.example mcpdiffusion/.env ### 3. Install dependencies ```bash -pip install -r mcpdiffusion/requirements.txt +uv sync ``` ### 4. (Alternative) Build the Docker image diff --git a/pyproject.toml b/pyproject.toml index 7d1e2cb..6bb3bf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "starlette==1.6.0", "httpx==0.28.1", "python-dotenv==1.2.3", + "pydantic-settings>=2.0.0", "trafilatura==2.2.0", "limits>=5.8.0", ] diff --git a/src/mcpdiffusion/config/__init__.py b/src/mcpdiffusion/config/__init__.py new file mode 100644 index 0000000..ed78ad4 --- /dev/null +++ b/src/mcpdiffusion/config/__init__.py @@ -0,0 +1 @@ +"""Centralized configuration.""" diff --git a/src/mcpdiffusion/config/settings.py b/src/mcpdiffusion/config/settings.py new file mode 100644 index 0000000..b1b9bce --- /dev/null +++ b/src/mcpdiffusion/config/settings.py @@ -0,0 +1,62 @@ +"""Centralized application settings validated at import time via Pydantic.""" +from __future__ import annotations + +import os +from functools import lru_cache +from typing import Optional + +from pydantic import Field +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + # Elasticsearch + es_host: Optional[str] = Field(default=None, alias="ES_HOST") + es_index_produits: str = Field(default="produit", alias="ES_INDEX_PRODUITS") + es_index_melodi_datasets: str = Field( + default="melodi_datasets", alias="ES_INDEX_MELODI_DATASETS" + ) + es_index_melodi_columns: str = Field( + default="melodi_columns", alias="ES_INDEX_MELODI_COLUMNS" + ) + + # TLS + tls_verify: bool = Field(default=True, alias="TLS_VERIFY") + + # Server + mcp_host: str = Field(default="0.0.0.0", alias="MCP_HOST") + mcp_port: int = Field(default=8000, alias="MCP_PORT") + allowed_hosts: str = Field(default="*", alias="ALLOWED_HOSTS") + forwarded_allow_ips: str = Field(default="*", alias="FORWARDED_ALLOW_IPS") + + # Rate limiting + global_request_min: int = Field(default=100, alias="GLOBAL_REQUEST_MIN") + tz: str = Field(default="Europe/Paris", alias="TZ") + + # Logging + log_level: str = Field(default="INFO", alias="LOG_LEVEL") + + # Tool selection + toollist: Optional[str] = Field(default=None, alias="TOOLLIST") + + # RMES / SPARQL + rmes_endpoint: str = Field( + default="https://rdf.insee.fr/sparql", alias="RMES_ENDPOINT" + ) + + # Melodi + melodi_data_base_url: str = Field( + default="https://api.insee.fr/melodi/data", alias="MELODI_DATA_BASE_URL" + ) + + # INSEE.fr + insee_base_url: str = Field( + default="https://www.insee.fr", alias="INSEE_BASE_URL" + ) + + model_config = {"env_file": ".env", "extra": "ignore", "populate_by_name": True} + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py new file mode 100644 index 0000000..4a5fa57 --- /dev/null +++ b/src/mcpdiffusion/config/tool_metadata.py @@ -0,0 +1,370 @@ +"""Tool metadata (name, description, version). + +Design notes: +- Tool *names* are English snake_case; French is kept only where it is + actual data (enum literals that hit the ES index, user-supplied queries). +- `CURRENT_DATE` is computed lazily so long-running servers always report + today's date, not the day the process started. +- Tool descriptions describe the *final* schemas; rewrite in lockstep + when schemas change. +""" +from datetime import date + + +def current_date_iso() -> str: + """Return today's date as ISO-8601.""" + return date.today().isoformat() + + +# --- MELODI tools ----------------------------------------------------------- + +GET_DATASET = { + "tool_name": "get_melodi_observations", + "tool_description": ( + "Retrieve a filtered set of observations from a Melodi dataset. " + "The Melodi API holds official, high-granularity statistics " + "(prices, mortality, names, etc.).\n" + "\n" + "WHEN TO USE\n" + "- You already know the exact `dataset_id` (from `search_melodi_datasets`) " + "AND the modality codes you want to filter on " + "(from `search_melodi_modalities`).\n" + "\n" + "WHEN NOT TO USE\n" + "- You are still looking for the right dataset. Use `search_melodi_datasets` first.\n" + "- You need concept definitions or code-list vocabularies. Use `query_insee_rmes`.\n" + "\n" + "WORKFLOW (chain with companion tools)\n" + "1. `search_melodi_datasets` -> dataset_id + column ids\n" + "2. `search_melodi_modalities` -> exact modality codes for filtering\n" + "3. THIS TOOL (`get_melodi_observations`) -> final observations\n" + "\n" + "OUTPUT\n" + "A list of observations with dimensions, attributes and the numeric " + "measure (with unit). Returns an empty list when no rows match; " + "a structured error when the upstream API fails or inputs are invalid.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +SEARCH_DATASET = { + "tool_name": "search_melodi_datasets", + "tool_description": ( + "Search the INSEE Melodi dataset catalogue by French-language natural " + "language query. Each dataset has a unique `dataset_id`; the tool maps " + "the query to internal metadata to return the most relevant matches.\n" + "\n" + "WHEN TO USE\n" + "- The user asks for a specific statistic (price of a product, " + "mortality by region, frequency of a name, etc.) and you need to " + "locate the right dataset before fetching rows.\n" + "\n" + "WHEN NOT TO USE\n" + "- Generic, up-to-date indicator questions (use `get_insee_homepage`).\n" + "- Full-text analysis of a published report (use `search_insee_documents`).\n" + "- Definition/ontology lookups (use `query_insee_rmes`).\n" + "\n" + "TIPS\n" + "- Matching is lexical. Make `french_query` explicit and rich in French " + "synonyms: e.g. `\"indice des prix a la consommation\"`, " + "`\"deces par departement\"`, `\"prenoms des nouveau-nes\"`.\n" + "- Use `start_year` / `end_year` to narrow the temporal range. Leaving " + "both at default covers all years.\n" + "\n" + "NEXT STEP\n" + "Pass the returned `dataset_id` and column ids to " + "`search_melodi_modalities`, then feed the resolved codes into " + "`get_melodi_observations`.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +SEARCH_MODALITIES = { + "tool_name": "search_melodi_modalities", + "tool_description": ( + "Given a Melodi dataset and one or more column identifiers, rank the " + "most relevant modalities (codes/labels) for a free-text French query. " + "The result is what you need to filter rows in `get_melodi_observations`.\n" + "\n" + "WHEN TO USE\n" + "- You have a `dataset_id` (from `search_melodi_datasets`) and want " + "to find the exact modality code for a concept like `cote de boeuf`, " + "`Ile-de-France`, or `female Maria`.\n" + "\n" + "WHEN NOT TO USE\n" + "- You don't yet know the dataset. Run `search_melodi_datasets` first.\n" + "\n" + "INPUT\n" + "- `dataset_id` -- from a previous search result.\n" + "- `columns_id` -- which columns to search (e.g. `[\"PRICES\", \"GEO\"]`).\n" + "- `french_query` -- natural-language query in French.\n" + "\n" + "OUTPUT\n" + "A list of matching columns, each containing its `code`, metadata text " + "and the top-scoring `matching_modalities` with `code`, `label_fr`, " + "`label_en` and `score`. Empty list when nothing matches.\n" + "\n" + "NEXT STEP\n" + "Use the modality `code` values as entries in " + "`get_melodi_observations.dict_of_columns_and_values`.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +# --- INSEE.fr tools --------------------------------------------------------- + +GET_DOCUMENT = { + "tool_name": "get_insee_document", + "tool_description": ( + "Fetch and parse a single INSEE publication from a known URL and " + "return its full text in markdown. Use ONLY when you already have one " + "or more explicit URLs (e.g. from `search_insee_documents` or from " + "the `link` fields returned by `get_insee_homepage`).\n" + "\n" + "WHEN TO USE\n" + "- You have a concrete URL of the form `/fr/statistiques/` or " + "`/fr/statistiques/?sommaire=`.\n" + "\n" + "WHEN NOT TO USE\n" + "- You are still looking for the right publication. Use " + "`search_insee_documents` first.\n" + "- You need a quick, up-to-date indicator. Use `get_insee_homepage`.\n" + "\n" + "INPUT\n" + "- `list_of_url` -- list of relative URLs to fetch (e.g. " + "`[\"/fr/statistiques/4277658?sommaire=4318291\"]`).\n" + "- `include_sommaire` -- also parse the page's table-of-contents " + "section. Use once to discover the structure of a multi-section " + "publication, then turn it off for subsequent requests on the same page.\n" + "- `truncate_content` -- when True (default), long markdown bodies are " + "clipped to keep the response compact for the model; set to False only " + "when you genuinely need the full text.\n" + "\n" + "OUTPUT\n" + "A uniform envelope: `{ status, results: [ { id, status, " + "markdown_content, sommaire, error, truncated } ], count }`. Each " + "per-URL entry has the same keys whether it succeeded or failed, so " + "downstream code can iterate without type-sniffing.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +SEARCH_DOCUMENTS = { + "tool_name": "search_insee_documents", + "tool_description": ( + "Search the INSEE catalogue of official statistical publications " + "(Insee Premiere, Insee Analyses, Dossiers, References, Focus, ...). " + "Returns structured publication records; pass the URL of a record to " + "`get_insee_document` to fetch the full text.\n" + "\n" + "ROUTING PRIORITY\n" + "- Simple statistics (population, inflation, chomage, PIB, salaires) " + "by region/department? -> Use `search_chiffres_clefs_insee` FIRST.\n" + "- Granular product data (e.g., beef rib price 2000)? -> Use " + "`search_melodi_datasets` FIRST.\n" + "- This tool is for ANALYSIS, CONTEXT, and COMPLEX NARRATIVES.\n" + "\n" + "WHEN TO USE THIS TOOL\n" + "- Impact analyses (e.g., 'covid effects on tourism').\n" + "- Historical evolution and trends (e.g., 'unemployment 1990-2026').\n" + "- Detailed methodological or definitional content.\n" + "- Regional/departmental profiles with socioeconomic context.\n" + "- Specific thematic deep-dives (demography, labour market, inequalities, " + "environment, housing, ...). \n" + "- Comparative studies or cross-cutting analyses.\n" + "\n" + "WHEN NOT TO USE THIS TOOL\n" + "- Simple factual questions ('What is X region's population?') -> " + "`search_chiffres_clefs_insee`.\n" + "- Quick, up-to-date headline indicators -> `get_insee_homepage`.\n" + "- Latest monthly/quarterly rapid releases -> `search_insee_conjoncture`.\n" + "- Vocabulary / code definitions / classifications -> `query_insee_rmes`.\n" + "- Granular historical time series (product prices, individual wages) -> " + "`search_melodi_datasets`.\n" + "\n" + "HOW TO SEARCH WELL\n" + "- `query` -- rich natural-language query with synonyms, context, " + "and target year/geography if relevant.\n" + "- `chiffre_clef=False` (default) -- general publications. Set to True " + "ONLY for 'essentials sur...' publications (essentiel sur l'inflation, " + "etc.), but prefer `search_chiffres_clefs_insee` for those instead.\n" + "- `geo_niveau` + `geo_keyword` -- territorial filtering " + "(COM/DEP/REG/INTER/COMPRD/FRANCE).\n" + "- `theme` -- restrict to top-level theme (Demographie, " + "Marche du travail, Economie, etc.). Default ALL.\n" + "- `year_of_reference` -- hard filter on publication year; null = all years.\n" + "\n" + "OUTPUT\n" + "List of publications: `{ id, score, titre, soustitre, chapo, " + "anneediffusion, zone, theme, url }`. Feed `url` to `get_insee_document`.\n" + "\n" + f"Current date is {current_date_iso()}.\n" + ), + "tool_metadata": {"version": "6.0", "author": "mirlon"}, +} + +SEARCH_CHIFFRECLEF = { + "tool_name": "search_insee_chiffrecle", + "tool_description": "Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, \n" + "comparaisons regionales/departementales et statistiques factuelles simples.\n" + "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" + "prix par categorie, comparaisons geographiques (region, departement, commune).\n" + "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" + "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" + "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" + "ou search_insee_documents selon le contexte).\n" + "Retourne directement les tableaux synthetiques prets a l'emploi.\n", + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +SEARCH_CONJONCTURE = { + "tool_name": "search_insee_conjoncture", + "tool_description": ( + "Search INSEE Rapid Releases (Informations rapides): short, recurring " + "publications reporting the latest monthly/quarterly/annual results for " + "major economic and social indicators (prices, employment, production, " + "housing, wages, national accounts, ...).\n" + "\n" + "WHEN TO USE\n" + "- The user asks for the *latest* monthly/quarterly release of a " + "named indicator (e.g. last month's consumer confidence, " + "last quarter's GDP estimate). Prefer the most recent edition.\n" + "\n" + "WHEN NOT TO USE\n" + "- Generic up-to-date indicator on the homepage: `get_insee_homepage`.\n" + "- Deep, peer-reviewed analysis: `search_insee_documents`.\n" + "\n" + "HOW TO SEARCH WELL\n" + "- `query` -- provide several synonyms and related notions; the " + "search is lexical and rewards keyword breadth.\n" + "- `theme_conjoncture` -- optional broad category (Industrial " + "production and activity, Inflation and producer prices, " + "Employment, unemployment and labour market, ...). Leave null to " + "search across all categories.\n" + "- `year_of_reference` -- hard filter on publication year; leave null " + "to search all years.\n" + "\n" + "OUTPUT\n" + "A list of publications: `{ id, score, titre, soustitre, chapo, " + "anneediffusion, zone, theme, url }`.\n" + "\n" + f"Current date is {current_date_iso()}.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +GET_HOMEPAGE = { + "tool_name": "get_insee_homepage", + "tool_description": ( + "Retrieve the INSEE home page with the latest key indicators at national level" + "published by the institute (population, inflation, unemployment, " + "GDP growth, ...).\n" + "\n" + "WHEN TO USE -- preferred FIRST step for any generic, up-to-date " + "statistical question. It gives the most recent official figure " + "instantly, without searching individual documents.\n" + "\n" + "WHEN NOT TO USE\n" + "- User asks for a previous year's figure. Use `search_insee_documents` " + "or `search_insee_conjoncture` with `year_of_reference`.\n" + "\n" + "OUTPUT\n" + "- `mainIndicators` -- each with name, value, description and a link " + "to the underlying official product (pass the link to `get_insee_document`).\n" + "- `lastArticles` -- recent short articles with title, date, " + "collection and link.\n" + "- `keyGraphics` -- selection of recent graphical publications.\n" + "\n" + "WORKFLOW\n" + "1. Call this tool.\n" + "2. Present the indicator value + description + link.\n" + "3. Follow up with `search_insee_documents` or `search_insee_conjoncture` " + "only if the user needs deeper tables or historic series.\n" + "\n" + f"Current date is {current_date_iso()}.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +# --- RMES (SPARQL) ---------------------------------------------------------- + +RMES_LIST_GRAPHS = { + "tool_name": "RMES_list_graphs", + "tool_description": ( + "Liste les graphes nommes disponibles dans la base RDF de l'INSEE (RMES). " + "Utilise ce tool EN PREMIER pour decouvrir quels graphes existent avant " + "d'ecrire une requete SPARQL avec RMES_run_sparql -- il y a plus de 700 graphes.\n" + "\n" + "Par defaut (`category=ALL`), le resultat est une vue CONDENSEE par categorie, " + "avec un compteur et quelques URIs d'exemple par categorie -- pas la liste plate " + "des 700+ graphes. Choisis une categorie precise dans le parametre `category` " + "pour cibler une famille, ou utilise `contains` pour une recherche libre par " + "sous-chaine. Une categorie \"autre\" recueille tout graphe ne correspondant a " + "aucune famille connue." + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +RMES_DESCRIBE_RESOURCE = { + "tool_name": "RMES_describe_resource", + "tool_description": ( + "Recupere toutes les proprietes connues (predicat -> valeur) d'une ressource RDF " + "identifiee par son URI complete. Combine automatiquement les proprietes ou la " + "ressource est sujet ET celles ou elle est objet (utile pour remonter des relations " + "skos:broader par exemple). Restreins avec `graph` si tu sais deja ou chercher -- " + "sinon la recherche se fait sur tous les graphes, ce qui est plus lent." + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +RMES_RUN_SPARQL = { + "tool_name": "RMES_run_sparql", + "tool_description": ( + "Execute une requete SPARQL libre sur RMES, la base de metadonnees, nomenclatures " + "et definitions de l'INSEE (elle ne contient PAS les chiffres/donnees, voir " + "get_MELODI_datasets pour ca).\n" + "\n" + "AVANT d'ecrire une requete complexe : appelle RMES_list_graphs pour connaitre les " + "categories de graphes disponibles.\n" + "\n" + "Bonnes pratiques :\n" + "- Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou " + " VALUES ?g { } plutot que de scanner tous les graphes.\n" + "- Toujours ajouter FILTER(lang(?label) = \"fr\") sur les litteraux SKOS pour eviter " + " les doublons multilingues.\n" + "- Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee " + " automatiquement (indique dans la reponse via `limit_added`/`hint`).\n" + "- Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures " + " statistiques : ClassificationLevel, ExplanatoryNote), dcterms (metadonnees), " + " rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE).\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} + +SEND_FEEDBACK = { + "tool_name": "send_feedback", + "tool_description": ( + "Submit structured feedback about the MCP tools, server behavior, or user experience. " + "This tool appends a timestamped Markdown entry to the feedback log for administrator review.\n" + "\n" + "WHEN TO USE\n" + "- The user reports a bug, error, or unexpected behavior in any tool.\n" + "- The user suggests an improvement, new feature, or enhancement.\n" + "- The assistant encounters an issue during tool execution that should be logged.\n" + "- After completing a complex workflow where feedback on tool quality would be valuable.\n" + "\n" + "WHEN NOT TO USE\n" + "- For transient debugging or one-off troubleshooting (use terminal/logs instead).\n" + "- For questions about tool usage (ask the user or consult documentation).\n" + "\n" + "INPUT\n" + "- `username` -- identifier for the feedback author (e.g., user name, role, or session ID).\n" + "- `feedback` -- clear, actionable Markdown describing the issue or suggestion. " + "Include context (which tool, what happened), expected vs actual behavior, and " + "proposed solutions if applicable. Write as if filing a GitHub issue.\n" + "\n" + "OUTPUT\n" + "Confirmation message with the timestamp and path where feedback was recorded.\n" + ), + "tool_metadata": {"version": "5.0", "author": "mirlon"}, +} diff --git a/src/mcpdiffusion/core/__init__.py b/src/mcpdiffusion/core/__init__.py new file mode 100644 index 0000000..f4552e7 --- /dev/null +++ b/src/mcpdiffusion/core/__init__.py @@ -0,0 +1 @@ +"""Shared cross-cutting concerns (logging, errors, middleware).""" diff --git a/src/mcpdiffusion/core/errors.py b/src/mcpdiffusion/core/errors.py new file mode 100644 index 0000000..33c3bbd --- /dev/null +++ b/src/mcpdiffusion/core/errors.py @@ -0,0 +1,34 @@ +"""Standardized error conventions for MCP tools.""" +from __future__ import annotations + +from typing import Literal + +from fastmcp.exceptions import ToolError + + +ErrorCode = Literal[ + "INVALID_INPUT", + "EMPTY_RESULT", + "BACKEND_UNAVAILABLE", + "UPSTREAM_ERROR", + "PARSE_ERROR", + "INVALID_QUERY", + "NOT_FOUND", + "UNKNOWN", +] + + +def fail( + code: ErrorCode, + message: str, + retryable: bool = False, +) -> None: + """Raise a standardized tool error. + + `message` should be actionable: name the offending parameter, suggest + the next step, include the shortest useful excerpt of the upstream error. + """ + prefix = f"[{code}] " + if retryable: + prefix = f"[{code}, retryable] " + raise ToolError(prefix + message) diff --git a/src/mcpdiffusion/helpers/logging.py b/src/mcpdiffusion/core/logging.py similarity index 78% rename from src/mcpdiffusion/helpers/logging.py rename to src/mcpdiffusion/core/logging.py index cee29b2..7ceca27 100644 --- a/src/mcpdiffusion/helpers/logging.py +++ b/src/mcpdiffusion/core/logging.py @@ -1,27 +1,20 @@ -""" -Structured logging config + per-tool decorator. - -- `MAIN_LOGGER_NAME` (`mcp.main`) is the application root. -- `TOOLS_LOGGER_NAME` (`mcp.tools`) is the tool-call stream. -- `@log_tool` works for both sync and async tool functions and emits: - * entry (tool name + kwargs preview, secrets scrubbed) - * exit (duration ms, result count when applicable) - * error (error code + short message) -""" +"""Structured logging config + per-tool decorator.""" from __future__ import annotations import functools import inspect import logging -import os import time from typing import Any, Callable, TypeVar +from ..config.settings import get_settings + +_settings = get_settings() MAIN_LOGGER_NAME = "mcp.main" logging.basicConfig( - level=os.getenv("LOG_LEVEL", "INFO"), + level=_settings.log_level, format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", force=True, ) @@ -41,7 +34,7 @@ } }, "root": { - "level": os.getenv("LOG_LEVEL", "INFO"), + "level": _settings.log_level, "handlers": ["default"], }, } @@ -52,14 +45,11 @@ _F = TypeVar("_F", bound=Callable[..., Any]) - -# Fields whose values we never want to log in plain text. _SCRUB_FIELDS = {"password", "mdp", "token", "secret", "auth", "api_key"} _KWARGS_PREVIEW_LIMIT = 800 def _scrub(kwargs: dict) -> str: - """Return a bounded, redacted repr of kwargs suitable for logs.""" safe = {} for k, v in kwargs.items(): if any(s in k.lower() for s in _SCRUB_FIELDS): @@ -73,7 +63,6 @@ def _scrub(kwargs: dict) -> str: def _result_count(result: Any) -> int | None: - """Best-effort count for result preview. None if unknown shape.""" if result is None: return 0 if isinstance(result, (list, tuple)): @@ -83,7 +72,6 @@ def _result_count(result: Any) -> int | None: return len(result["results"]) if "count" in result: return result["count"] - # Pydantic models with a .results attribute. r = getattr(result, "results", None) if isinstance(r, list): return len(r) @@ -91,12 +79,7 @@ def _result_count(result: Any) -> int | None: def log_tool(func: _F) -> _F: - """Decorator that logs entry, exit (duration + count) and errors. - - Supports both sync and async tool functions. The FastMCP tool registry - expects the decorated function to have the original signature; we - preserve it via `inspect.signature`. - """ + """Decorator that logs entry, exit (duration + count) and errors.""" is_async = inspect.iscoroutinefunction(func) name = func.__name__ @@ -143,6 +126,5 @@ def sync_wrapper(*args, **kwargs): return result wrapper = sync_wrapper - # Preserve the original signature for FastMCP introspection. wrapper.__signature__ = inspect.signature(func) # type: ignore[attr-defined] return wrapper # type: ignore[return-value] diff --git a/src/mcpdiffusion/core/middleware.py b/src/mcpdiffusion/core/middleware.py new file mode 100644 index 0000000..70a7178 --- /dev/null +++ b/src/mcpdiffusion/core/middleware.py @@ -0,0 +1,60 @@ +"""Rate-limiting middleware with injected settings.""" +from __future__ import annotations + +from datetime import datetime +from zoneinfo import ZoneInfo + +from limits import parse, storage, strategies +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.status import HTTP_429_TOO_MANY_REQUESTS + +from ..config.settings import Settings, get_settings + + +class RateLimitMiddleware(BaseHTTPMiddleware): + """Middleware that applies per-IP rate limiting. + + Accepts an optional ``settings`` parameter for dependency injection + (used by tests). Falls back to ``get_settings()`` when not provided. + """ + + def __init__(self, app, settings: Settings | None = None): + super().__init__(app) + self._settings = settings or get_settings() + self._tz = ZoneInfo(self._settings.tz) + self._storage = storage.MemoryStorage() + self._limiter = strategies.MovingWindowRateLimiter(self._storage) + self._rate = parse(f"{self._settings.global_request_min}/minute") + + async def dispatch(self, request: Request, call_next): + client_ip = request.client.host if request.client else "unknown" + rate_key = f"{client_ip}" + + if not self._limiter.hit(self._rate, rate_key): + retry_after_ts = self._limiter.get_window_stats(self._rate, rate_key)[0] + retry_after_time = datetime.fromtimestamp( + retry_after_ts, tz=self._tz + ).strftime("%H:%M:%S") + return JSONResponse( + status_code=HTTP_429_TOO_MANY_REQUESTS, + content={ + "detail": f"Trop de requetes. Reessayez apres {retry_after_time}.", + "retry_after": retry_after_time, + }, + headers={ + "Retry-After": str(int(retry_after_ts)), + "X-RateLimit-Limit": str(self._settings.global_request_min), + "X-RateLimit-Remaining": "0", + }, + ) + + response = await call_next(request) + + remaining = self._limiter.get_window_stats(self._rate, rate_key)[1] + response.headers["X-RateLimit-Limit"] = str(self._settings.global_request_min) + response.headers["X-RateLimit-Remaining"] = str(remaining) + response.headers["X-RateLimit-Window"] = f"{60}s" + + return response diff --git a/src/mcpdiffusion/data/__init__.py b/src/mcpdiffusion/data/__init__.py new file mode 100644 index 0000000..d67f3a1 --- /dev/null +++ b/src/mcpdiffusion/data/__init__.py @@ -0,0 +1 @@ +"""Static reference data (indicators, themes, geography).""" diff --git a/src/mcpdiffusion/data/geography.py b/src/mcpdiffusion/data/geography.py new file mode 100644 index 0000000..657f7e9 --- /dev/null +++ b/src/mcpdiffusion/data/geography.py @@ -0,0 +1,10 @@ +"""Geographic level mappings.""" + +DICT_GEO = { + "COMMUNE": "COM", + "DEPARTEMENT": "DEP", + "REGION": "REG", + "INTERNATIONAL": "INTER", + "INTER REGION": "COMPRD", + "FRANCE": "FRANCE", +} diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/indicators.py new file mode 100644 index 0000000..038cbbc --- /dev/null +++ b/src/mcpdiffusion/data/indicators.py @@ -0,0 +1,64 @@ +"""Curated INSEE key indicators (homepage data).""" + +DICT_KV = [ + {"cle": "clé", "alias": "alias", "valeur": "valeur"}, + {"cle": "estimation de population France", "alias": "", "valeur": "Au 1er janvier 2026, la population résidant en France est estimée à 69,1 millions d'habitants."}, + {"cle": "population légale France", "alias": "", "valeur": "Au 1er janvier 2023, la population de la France hors Mayotte s'établit officiellement à 68 094 000 habitants."}, + {"cle": "immigrés France", "alias": "", "valeur": "En 2025, 8,0 millions d'immigrés vivent en France, soit 11,6 % de la population totale."}, + {"cle": "population étrangère France", "alias": "", "valeur": "En 2025, la population étrangère vivant en France s'élève à 6,3 millions de personnes, soit 9,1 % de la population totale."}, + {"cle": "naissances France", "alias": "", "valeur": "En 2025, le nombre de naissances en France est estimé à 645 000, soit une baisse de -2,1 % par rapport à 2024."}, + {"cle": "indicateur conjoncturel de fécondité", "alias": "", "valeur": "En 2025, l'indicateur conjoncturel de fécondité (ICF) continue de diminuer. Il s'établit à 1,56 enfant par femme (1,53 en France métropolitaine), après 1,61 en 2024 (1,58 en France métropolitaine)."}, + {"cle": "décès France", "alias": "", "valeur": "En 2025, le nombre de décès en France est estimé à 651 000, en hausse de 1,5 % par rapport à 2024, après +0,3 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile)."}, + {"cle": "espérance de vie France", "alias": "", "valeur": "En 2025, l'espérance de vie à la naissance s'élève à 85,9 ans pour les femmes et à 80,3 ans pour les hommes. Elle augmente en 2025, de +0,1 an pour les femmes comme pour les hommes, pour atteindre un niveau historiquement élevé."}, + {"cle": "mariages France", "alias": "", "valeur": "En 2025, le nombre de mariages célébrés en France est estimé à 251 000, dont 244 000 entre personnes de sexe différent et 7 000 entre personnes de même sexe. Le nombre de mariages augmente de 1,4 % par rapport à 2024, après +2,7 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile), alors que la tendance était plutôt à la baisse avant la crise sanitaire."}, + {"cle": "ménages France", "alias": "", "valeur": "En 2023, la France hors Mayotte compte 31,3 millions de ménages."}, + {"cle": "divorces France", "alias": "", "valeur": "128 043 divorces en 2016. Note : jusqu'en 2016, les divorces étaient des décisions de justice prononcées par un juge ; depuis 2017, les divorces par consentement mutuel passent par un acte notarié et ne sont plus comptabilisés de la même façon."}, + {"cle": "inflation", "alias": "Indice des prix à la consommation – IPC ", "valeur": "En juin 2026, les prix à la consommation (IPC) augmentent de 1,8 % sur un an. Sur un mois, l'indice des prix à la consommation diminue de 0,3 %."}, + {"cle": "Chômage BIT ", "alias": "", "valeur": "Au premier trimestre 2026, le taux de chômage en France (hors Mayotte) augmente de 0,2 point et atteint 8,1 % . Le nombre de chômeurs est de 2,6 millions de personnes."}, + {"cle": "emploi BIT", "alias": "", "valeur": "En moyenne sur l'année 2025, parmi les personnes âgées de 15 à 64 ans vivant en France, 69,3 % sont en emploi au sens du Bureau international du travail (BIT)."}, + {"cle": "PIB trimestriel", "alias": "croissance trimestrielle", "valeur": "Au premier trimestre 2026, le produit intérieur brut (PIB) en volume se replie légèrement (-0,1 %)."}, + {"cle": "PIB annuel", "alias": "croissance annuelle", "valeur": "En 2025, le PIB croît de 0,8 % en volume aux prix de l'année précédente."}, + {"cle": "Dépenses de consommation des ménages en biens", "alias": "", "valeur": "En mai 2026, les dépenses de consommation des ménages en biens rebondissent sur un mois (+0,5 % en volume après -0,5 % en avril). Les volumes sont mesurés aux prix de l'année précédente chaînés (en milliards d'euros 2020) et corrigés des variations saisonnières et des effets des jours ouvrables (CVS-CJO)."}, + {"cle": "Climat des affaires", "alias": "", "valeur": "En juin 2026, l'indicateur synthétique du climat des affaires, calculé à partir des réponses des chefs d'entreprise des principaux secteurs d'activité marchands rebondit très légèrement, à 94, en deçà de son niveau moyen."}, + {"cle": "climat de l'emploi", "alias": "", "valeur": "En juin 2026, l'indicateur du climat de l'emploi perd de nouveau trois points (après arrondi) et s'établit à 89, son niveau le plus bas depuis juin 2013 (hors crise sanitaire)."}, + {"cle": "production manufacturière", "alias": "Indice de la production industrielle - IPI", "valeur": "En mai 2026, après deux mois de hausse, la production se replie nettement dans l'industrie manufacturière (-1,0 % après +0,6 % en avril 2026). Dans l'ensemble de l'industrie, elle se replie aussi mais plus légèrement (-0,1 % après +0,3 %)."}, + {"cle": "niveau de vie", "alias": "", "valeur": "En 2024, en France métropolitaine, le niveau de vie médian de la population s'élève à 26 740 euros annuels. Il correspond à un revenu disponible de 2 228 euros par mois pour une personne seule."}, + {"cle": "pouvoir d'achat", "alias": "", "valeur": "En 2025, le pouvoir d'achat du revenu disponible (RDB) des ménages se replie de 0,4 % après une hausse de 2,7 % en 2024. Ramené au niveau individuel et en tenant compte de l'évolution de la taille des ménages, le pouvoir d'achat baisse de 0,7 % après une hausse de 2,2 % en 2024"}, + {"cle": "balance commerciale", "alias": "", "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l'activité en 2025, à hauteur de -0,2 point de PIB, après l'avoir fortement soutenue en 2023 et 2024. "}, + {"cle": "pauvreté monétaire", "alias": "", "valeur": "En 2024, 9,8 millions de personnes vivent avec un niveau de vie inférieur au seuil de pauvreté monétaire, soit 15,4 % de la population vivant dans un logement ordinaire en France métropolitaine."}, + {"cle": "patrimoine", "alias": "", "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine. "}, + {"cle": "état santé", "alias": "", "valeur": "En 2024, deux tiers des personnes âgées de 16 ans ou plus se déclarent en bonne ou très bonne santé. À l'opposé, près de 10 % jugent leur état de santé mauvais voire très mauvais."}, + {"cle": "prestation handicap", "alias": "", "valeur": "Selon leur âge et leur situation, les personnes en situation de handicap ou de perte d'autonomie peuvent prétendre à différentes prestations. Fin 2023, 44 000 personnes ont un droit ouvert à l'allocation compensatrice pour tierce personne (ACTP) et 407 000 à la prestation de compensation du handicap (PCH). Par ailleurs, 1,4 million de personnes de 60 ans ou plus ont perçu l'allocation personnalisée d'autonomie (APA) au titre du mois de décembre 2023."}, + {"cle": "dépenses liées à la culture", "alias": "", "valeur": "En 2025, les dépenses liées à la culture, au sport et aux loisirs s'élèvent à 108 milliards d'euros. Les services récréatifs, sportifs et culturels rassemblent 45 % de ces dépenses."}, + {"cle": "Parc de logements", "alias": "", "valeur": "Au 1er janvier 2025, la France hors Mayotte compte 38,4 millions de logements. 82,5 % des logements sont des résidences principales et 54,4 % des logements individuels (maisons)."}, + {"cle": "logements vacants", "alias": "", "valeur": "Après avoir fortement augmenté entre 2005 et 2019, la part des logements vacants diminue, passant de 8,1 % en 2019 à 7,7 % en 2025 ; en 2025, 3,0 millions de logements sont vacants."}, + {"cle": "résidences secondaires ou logements occasionnels", "alias": "", "valeur": "Au 1er janvier 2025, 3,8 millions de logements sont des résidences secondaires ou des logements occasionnels ; après avoir augmenté entre 2011 et 2017, leur part dans l'ensemble du parc est stable."}, + {"cle": "ménages sont propriétaires de leur résidence principale", "alias": "", "valeur": "Au 1er janvier 2025, 57,4 % des ménages sont propriétaires de leur résidence principale."}, + {"cle": "smic", "alias": "Salaire minimum interprofessionnel de croissance", "valeur": "Depuis le 1er janvier 2026, le Smic brut s'élève à 12,02 euros par heure, soit 1 823,03 euros par mois pour 151,67 heures de travail."}, + {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur privé", "alias": "", "valeur": "En 2023, le salaire mensuel moyen en équivalent temps plein (EQTP) dans le secteur privé est de 2 730 euros, nets de cotisations et contributions sociales."}, + {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur public", "alias": "", "valeur": "Dans la fonction publique, tous statuts confondus, un salarié gagne en moyenne 2 650 euros nets par mois en EQTP en 2023."}, + {"cle": "revenus non salariés", "alias": "", "valeur": "En 2023, hors agriculture, les non-salariés classiques (micro-entrepreneurs exclus) retirent en moyenne 4 040 euros par mois de leur activité non salariée. Cette moyenne recouvre de fortes disparités selon la nature des emplois."}, + {"cle": "salaires horaires", "alias": "", "valeur": "Au premier trimestre 2026, les salaires horaires augmentent de 0,3 % sur le trimestre et de 2,0 % sur un an"}, + {"cle": "coût horaire du travail", "alias": "Indice du coût du travail – ICT", "valeur": "Au premier trimestre 2026, le coût horaire du travail (salaires, cotisations et taxes, déduction faite des exonérations et subventions) de l'ensemble du secteur marchand non agricole (hors services aux ménages) freine significativement, dans le sillage des salaires : +0,5 % sur le trimestre et + 2,3 % sur un an."}, + {"cle": "création entreprises", "alias": "", "valeur": "En 2025, 1 165 800 entreprises ont été créées en France, dont 758 500 sous forme d'entrepreneurs individuels ayant adopté le régime de la microentreprise (micro-entrepreneurs)."}, + {"cle": "défaillances d'entreprises", "alias": "", "valeur": "En 2025, 68 872 unités légales ont été en situation de défaillance."}, + {"cle": "entreprises marchandes non agricoles et non financières en France", "alias": "", "valeur": "En 2023, en France, les secteurs marchands non agricoles et non financiers (incluant toutefois les exploitations forestières, les auxiliaires de services financiers et d'assurance et les holdings) comptent 5,2 millions d'entreprises. Ces entreprises emploient 15,9 millions de salariés en équivalent temps plein (EQTP)."}, + {"cle": "exploitations agricoles", "alias": "", "valeur": "Dans le secteur agricole, l'usage est de compter plutôt des exploitations agricoles ; en 2023, la France métropolitaine en compte 349 600 et la main d'œuvre agricole s'élève à 663 200 EQTP."}, + {"cle": "commerce", "alias": "", "valeur": "En 2023, le commerce rassemble 739 128 entreprises. Elles réalisent un chiffre d'affaires de 1 728 milliards d'euros et dégagent une valeur ajoutée (VA) de 272 milliards d'euros. Fin 2024, 3,4 millions de personnes occupent un emploi salarié dans le commerce."}, + {"cle": "industrie", "alias": "", "valeur": "En 2023, l'industrie rassemble 322 386 entreprises. Elles réalisent un chiffre d'affaire de 1 544 milliards d'euros et dégagent une valeur ajoutée (VA) de 368 milliards d'euros. Fin 2024, 3,3 millions de personnes occupent un emploi salarié dans l'industrie."}, + {"cle": "construction", "alias": "", "valeur": "En 2023, la construction rassemble 587 898 entreprises. Elles réalisent un chiffre d'affaires de 405 milliards d'euros et dégagent une valeur ajoutée (VA) de 128 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans la construction."}, + {"cle": "services", "alias": "", "valeur": "En 2023, les services principalement marchands non financiers comptent plus de 2,3 millions d'entreprises. Ces entreprises réalisent un chiffre d'affaires de 995 milliards d'euros et dégagent une valeur ajoutée (VA) de 475 milliards d'euros. Fin 2024, 7,5 millions de personnes (y compris les intérimaires) occupent un emploi salarié dans les services principalement marchands non financiers."}, + {"cle": "transports", "alias": "", "valeur": "En 2023, les transports et l'entreposage rassemblent 193 101 entreprises. Elles réalisent un chiffre d'affaires de 267 milliards d'euros et dégagent une valeur ajoutée (VA) de 102 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans les transports et l'entreposage."}, + {"cle": "entreprises de l'économie sociale", "alias": "", "valeur": "Les entreprises de l'économie sociale se caractérisent par leur famille de l'économie sociale, à la fois privé et à caractère essentiellement non lucratif. En 2022, elles représentent 9,8 % de l'emploi salarié total en équivalent temps plein. Les associations emploient 73 % de ce volume de travail salarié ; 14 % est employé par les coopératives, 6 % par les mutuelles, 5 % par les fondations et 3 % par les autres organismes privés à but non-lucratif."}, + {"cle": "Population quartiers prioritaires de la politique de la ville", "alias": "QPV", "valeur": "Les quartiers prioritaires de la politique de la ville (QPV) tels que définis par le décret n° 2015-1138 du 14 septembre 2015 regroupent 7,9 % de la population en 2020."}, + {"cle": "Population unités urbaines", "alias": "", "valeur": "Les unités urbaines rassemblent toujours plus d'habitants. En 2022, en France métropolitaine, elles représentent 78,8 % de la population, soit 51,9 millions d'habitants. À l'exception de l'unité urbaine de Paris qui concentre près de 11 millions d'habitants, les 10 plus grandes unités urbaines françaises comptent chacune entre 0,5 et 2 millions d'habitants."}, + {"cle": "mode déplacement domicile travail", "alias": "", "valeur": "Pour se rendre au travail, les personnes en emploi se déplacent majoritairement en voiture ou en deux-roues motorisés (71 % en 2022). 15 % des personnes en emploi empruntent les transports en commun."}, + {"cle": "dépense nationale protection de l'environnement", "alias": "", "valeur": "En 2022, la dépense nationale en faveur de la protection de l'environnement s'élève à 63,7 milliards d'euros (Md€). Elle est dédiée à la protection de l'air, de la biodiversité et des paysages, la collecte et traitement des déchets, la protection et dépollution des sols et des eaux, la lutte contre le bruit et d'autres activités de protection de l'environnement (frais de fonctionnement de l'administration publique et des opérateurs chargés des questions environnementales notamment). Les entreprises sont les principaux financeurs des dépenses de protection de l'environnement (22,6 Md€, soit 35 %), devant les administrations publiques (État et ses ministères, collectivités locales, organismes publics) (22,2 Md€, soit 35 %) et les ménages (18,1 Md€, soit 28 %)."}, + {"cle": "indice de référence des loyers", "alias": "IRL", "valeur": "Au deuxième trimestre 2026, l'indice de référence des loyers s'établit à 148,37. Sur un an, il augmente de 1,15 % après +0,78 % au trimestre précédent."}, + {"cle": "indice des loyers commerciaux", "alias": "ILC", "valeur": "Au premier trimestre 2026, l'indice des loyers commerciaux s'établit à 135,26. Sur un an, il baisse de 0,45 % (après -0,50 % au trimestre précédent)."}, + {"cle": "indice des loyers des activités tertiaires", "alias": "ILAT", "valeur": "Au premier trimestre 2026, l'indice des loyers des activités tertiaires s'établit à 137,42. Sur un an, il augmente de 0,09 % (après -0,06 % au trimestre précédent)."}, + {"cle": "indice du coût de la construction", "alias": "ICC", "valeur": "L'indice du coût de la construction (ICC) s'établit à 2 084 au premier trimestre 2026. Il est en hausse de 1,26 % sur un trimestre (après +0,10 % au trimestre précédent). Sur un an, il baisse de 2,89 % (après -2,37 % au trimestre précédent)."}, + {"cle": "index du bâtiment tous corps d'état", "alias": "BT01 ; index bâtiment BT01", "valeur": "En mai 2026, l'index Bâtiment BT01 « Tous corps d'état » s'établit à 137,9, en référence 100 en 2010."}, + {"cle": "index général des travaux publics", "alias": "TP01 ; index travaux publics TP01", "valeur": "En mai 2026, l'index Travaux publics TP01 « Index général tous travaux » s'établit à 140,4, en référence 100 en 2010."}, + {"cle": "index ingénierie", "alias": "ING ; indice ING", "valeur": "En mai 2026, l'index divers de la construction ING « Ingénierie » s'établit à 138,3, en référence 100 en 2010."}, +] diff --git a/src/mcpdiffusion/data/themes.py b/src/mcpdiffusion/data/themes.py new file mode 100644 index 0000000..125e104 --- /dev/null +++ b/src/mcpdiffusion/data/themes.py @@ -0,0 +1,117 @@ +"""INSEE theme mappings and conjoncture sub-themes.""" + +KEYS_THEME_NIV1 = { + "Demographie": 0, + "Conditions de vie - Societe": 6, + "Marche du travail - Salaires": 20, + "Economie - Conjoncture - Comptes nationaux": 27, + "Entreprises": 37, + "Secteurs d'activite": 44, + "Territoires, villes et quartiers": 68, + "Developpement durable - Environnement": 74, + "Revenus - Pouvoir d'achat - Consommation": 80, + "Methodes": 86, +} + + +DICT_THEME_CONJ: dict[str, list[str]] = { + "Industrial production and activity": [ + "Indice de la production industrielle ", + "Enquete mensuelle de conjoncture dans l'industrie", + "Enquete trimestrielle de conjoncture dans l'industrie", + "Chiffre d'affaires dans l'industrie et la construction", + "Indices des commandes en valeur recues dans l'industrie", + "Enquete sur les investissements dans l'industrie", + "Enquete de tresorerie dans l'industrie", + ], + "Construction and building sector": [ + "Enquete mensuelle de conjoncture dans l'industrie du batiment", + "Enquete trimestrielle dans les travaux publics", + "Enquete trimestrielle dans l'artisanat du batiment", + "Construction de locaux", + "Index batiment, travaux publics et divers de la construction", + "Indices des couts de production dans la construction", + "Indice des prix d'entretien-amelioration des batiments", + "Indice du cout de la construction", + ], + "Housing and real estate": [ + "Enquete trimestrielle dans la promotion immobiliere", + "Indice de reference des loyers", + "Indice des loyers commerciaux", + "Indice des loyers des activites tertiaires", + "Indices des loyers d'habitation", + "Indice des prix des logements neufs et anciens", + "Indices des prix des logements anciens", + "Commercialisation de logements neufs - Ventes aux particuliers et ventes aux institutionnels", + ], + "Retail, wholesale and services": [ + "Enquete mensuelle de conjoncture dans le commerce de detail et le commerce et la reparation automobiles", + "Enquete mensuelle de conjoncture dans les services", + "Enquete bimestrielle de conjoncture dans le commerce de gros", + "Volume des ventes dans le commerce de detail et les services personnels ", + "Volume des ventes dans le commerce", + "Chiffre d'affaires dans le commerce de gros et divers services aux entreprises", + "Indice de production dans les services", + "Chiffre d'affaires des grandes surfaces alimentaires (parution arretee aux resultats de decembre 2022)", + ], + "Business demographics and confidence": [ + "Creations d'entreprises", + "Defaillances d'entreprises (parution arretee aux resultats de juillet 2012)", + "Climat des affaires", + "Notes et Points de conjoncture nationaux", + "Conjoncture regionale", + ], + "Employment, unemployment and labour market": [ + "Estimation flash de l'emploi salarie", + "Emploi salarie", + "Emploi et taux de chomage localises (par region et departement)", + "Emploi salarie, salaires de base et duree du travail (resultats definitifs)", + "Emploi salarie, salaires de base et duree du travail (resultats provisoires)", + "Chomage au sens du BIT et indicateurs sur le marche du travail (resultats de l'enquete Emploi)", + "Les inscrits a France Travail", + ], + "Wages and labour costs": [ + "Indice du cout horaire du travail revise - Tous salaries (ICHT, ICHTrev-TS) - Publication arretee depuis le 06/10/2023", + "Indice du cout du travail (ICT) - Resultats detailles", + "Indice du cout du travail (ICT) - Estimation flash", + "Salaires de base - Comparaison France-Allemagne", + ], + "Public sector employment and pay": [ + "L'emploi dans la fonction publique", + "Indice de traitement brut dans la fonction publique d'Etat - grille indiciaire", + "Les salaires dans la fonction publique", + ], + "Households, consumption and health": [ + "Consommation de soins et biens medicaux (CSBM)", + "Prestations et ressources de protection sociale", + "Depenses de consommation des menages en biens", + "Enquete mensuelle de conjoncture aupres des menages ", + ], + "Inflation and producer prices": [ + "Prix a la consommation - moyennes annuelles", + "Indice des prix a la consommation - resultats definitifs", + "Indice des prix a la consommation - resultats provisoires", + "Indices de prix de production et d'importation de l'industrie", + "Indices des prix de production des services ", + "Indices des prix agricoles", + "Prix des energies et des matieres premieres importees", + "Indice des prix dans la grande distribution (parution arretee aux resultats de decembre 2025)", + ], + "National accounts and public finance": [ + "Comptes nationaux trimestriels - premiere estimation", + "Comptes nationaux trimestriels - deuxieme estimation", + "Comptes nationaux trimestriels - resultats detailles", + "Comptes nationaux annuels - revision des principaux agregats", + "Comptes nationaux des administrations publiques - premiers resultats", + "Situation mensuelle budgetaire de l'Etat", + "Dette trimestrielle de Maastricht des administrations publiques", + "Recettes fiscales de l'Etat", + ], + "Transport and tourism": [ + "Immatriculations de vehicules neufs", + "Frequentation touristique dans les hotels, campings et autres hebergements collectifs touristiques", + ], + "Business financing": [ + "Enquete annuelle credit-bail", + ], +} diff --git a/src/mcpdiffusion/helpers/__init__.py b/src/mcpdiffusion/helpers/__init__.py deleted file mode 100644 index f1fcfd0..0000000 --- a/src/mcpdiffusion/helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Shared helpers for the mcp-diffusion server.""" diff --git a/src/mcpdiffusion/helpers/es.py b/src/mcpdiffusion/helpers/es.py deleted file mode 100644 index ffb9215..0000000 --- a/src/mcpdiffusion/helpers/es.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Centralized Elasticsearch client and index-name constants. - -Why a module-level singleton: -- Tools used to each build their own `Elasticsearch(...)` at import time. - That made the whole server refuse to start when ES was down, even though - one tool (`query_insee_rmes`) doesn't need ES at all. -- This module builds the client lazily on first `get_es_client()` call, - and tools catch ES connection errors so a backend outage degrades to - a structured tool error instead of a boot failure. - -Configuration: -- `ES_HOST` (required) -- single endpoint URL, e.g. http://localhost:9200. -- `TLS_VERIFY` -- "true" (default) or "false". -- Credentials are intentionally NOT supported; rely on network-level auth. - Add basic_auth here if that assumption changes. -""" -from __future__ import annotations - -import logging -import os - -from elasticsearch import Elasticsearch - - -logger = logging.getLogger("mcp.main") - -# Index names -- kept as constants so they can be overridden from env if needed. -INDEX_PRODUITS = os.getenv("ES_INDEX_PRODUITS", "produit") -INDEX_MELODI_DATASETS = os.getenv("ES_INDEX_MELODI_DATASETS", "melodi_datasets") -INDEX_MELODI_COLUMNS = os.getenv("ES_INDEX_MELODI_COLUMNS", "melodi_columns") - - -_client: Elasticsearch | None = None - - -def _tls_verify() -> bool: - return os.getenv("TLS_VERIFY", "true").strip().lower() != "false" - - -def get_es_client() -> Elasticsearch: - """Return the shared Elasticsearch client, building it on first call.""" - global _client - if _client is None: - host = os.getenv("ES_HOST") - if not host: - raise RuntimeError( - "ES_HOST environment variable is not set. " - "See .env.example for the expected value." - ) - _client = Elasticsearch( - host, - verify_certs=_tls_verify(), - request_timeout=30, - max_retries=2, - retry_on_timeout=True, - ) - logger.info("Elasticsearch client initialized for %s", host) - return _client - - -def reset_es_client() -> None: - """Drop the cached client. Used by tests / long-running reconfiguration.""" - global _client - _client = None diff --git a/src/mcpdiffusion/helpers/rmes.py b/src/mcpdiffusion/helpers/rmes.py deleted file mode 100644 index 79dabe9..0000000 --- a/src/mcpdiffusion/helpers/rmes.py +++ /dev/null @@ -1,415 +0,0 @@ -""" -Shared infrastructure for RMES (INSEE SPARQL) tools. - -Contains: HTTP client, SPARQL execution engine, error types, category -taxonomy, graph cache, and all constants used by the three RMES tools. -""" - -import logging -import re -import time -from enum import StrEnum -from typing import Any, Optional - -import httpx -from pydantic import BaseModel - -logger = logging.getLogger("mcp.rmes") - -ENDPOINT = "https://rdf.insee.fr/sparql" -HEADERS_BASE = {"User-Agent": "MCP-RMeS/2.0"} - -DEFAULT_TIMEOUT = 20.0 -MAX_TIMEOUT = 60.0 -DEFAULT_ROW_LIMIT = 200 -MAX_ROW_LIMIT = 2000 - -GRAPH_BASE = "http://rdf.insee.fr/graphes/" - -# Cache mémoire très simple pour la liste brute des graphes, coûteuse -# (COUNT sur 700+ graphes) et rarement volatile. -_GRAPH_CACHE: dict[str, Any] = {"data": None, "ts": 0.0} -_GRAPH_CACHE_TTL = 3600.0 # 1h - -_client: httpx.AsyncClient | None = None - - -def _get_client() -> httpx.AsyncClient: - """Client HTTP partagé (pooling de connexions), recréé s'il a été fermé.""" - global _client - if _client is None or _client.is_closed: - _client = httpx.AsyncClient(headers=HEADERS_BASE) - return _client - - -# --------------------------------------------------------------------------- -# Taxonomie des graphes (règles internes, non exposées telles quelles au LLM) -# --------------------------------------------------------------------------- -# -# Familles identifiées manuellement en inspectant le contenu réel des graphes -# (rdf:type dominants), codées en dur car stables dans le temps. Le premier -# "match" gagne -- les règles spécifiques (ex: exclusions "codes/nomenclatures") -# précèdent les règles génériques par préfixe (ex: "codes/"). - -CategoryMatcher = Any # Callable[[str], bool], alias pour lisibilité - - -class _CategoryRule: - __slots__ = ("key", "label", "description", "match") - - def __init__(self, key: str, label: str, description: str, match: CategoryMatcher): - self.key = key - self.label = label - self.description = description - self.match = match - - -def _exact(*paths: str) -> CategoryMatcher: - allowed = set(paths) - return lambda path: path in allowed - - -def _prefix(prefix: str) -> CategoryMatcher: - return lambda path: path.startswith(prefix) - - -CATEGORY_DEFS: list[_CategoryRule] = [ - _CategoryRule( - key="qualite_rapports", - label="Rapports qualité", - description=( - "Un graphe par opération statistique documentée (sdmx-mm:MetadataReport), " - "structuré selon le standard européen SIMS. Contient les dimensions qualité " - "(pertinence, précision, actualité, cohérence...) sous forme de " - "sdmx-mm:ReportedAttribute. Tous ces graphes ont un schéma identique." - ), - match=_prefix("qualite/rapport/"), - ), - _CategoryRule( - key="qualite_referentiels", - label="Référentiels qualité", - description=( - "Vocabulaire SIMS-FR (simsv2fr), documents annexes (documents) et référentiel " - "territorial (territoires) associés aux rapports qualité." - ), - match=_exact("qualite/documents", "qualite/simsv2fr", "qualite/territoires"), - ), - _CategoryRule( - key="codes_concepts_generiques", - label="Concepts génériques de codification", - description=( - "Concepts transverses qualifiant des opérations ou nomenclatures (Fréquence, " - "Langue, ModeCollecte, UniteEnquetee, CategorieSource, StatutEnquete...) et " - "notes explicatives xkos. Ce n'est PAS une nomenclature métier -- voir " - "'nomenclatures' pour NAF/PCS/COICOP/etc." - ), - match=_exact("codes", "codes/nomenclatures"), - ), - _CategoryRule( - key="nomenclatures", - label="Nomenclatures (classifications officielles)", - description=( - "Nomenclatures statistiques officielles et leurs versions successives : " - "activités (NAF/NAFR), produits (CPF), professions et catégories " - "socioprofessionnelles (PCS/PCSESE), consommation (COICOP), catégories " - "juridiques (CJ), emplois (EAP/EMB par année), tables de correspondance entre " - "versions (ex: nafr2-cpfr21)." - ), - match=_prefix("codes/"), - ), - _CategoryRule( - key="operations_statistiques", - label="Opérations statistiques", - description=( - "Catalogue des opérations (StatisticalOperation), séries et familles " - "d'enquêtes/collectes de l'Insee. C'est la cible (sdmx-mm:target) de chaque " - "rapport qualité." - ), - match=_exact("operations"), - ), - _CategoryRule( - key="demographie", - label="Démographie", - description="Populations légales par année (popleg).", - match=_prefix("demo/"), - ), - _CategoryRule( - key="geographie", - label="Géographie", - description="Code officiel géographique (COG) : communes, découpages administratifs.", - match=_prefix("geo/"), - ), - _CategoryRule( - key="organisations", - label="Organisations", - description=( - "Organismes producteurs de statistiques (services statistiques ministériels...) " - "et unités organisationnelles internes de l'Insee." - ), - match=_prefix("organisations"), - ), - _CategoryRule( - key="concepts", - label="Concepts et définitions statistiques", - description="Thèmes statistiques et définitions de notions utilisées dans les publications.", - match=_prefix("concepts"), - ), - _CategoryRule( - key="produits", - label="Produits / indicateurs statistiques", - description="Indicateurs statistiques publiés (StatisticalIndicator).", - match=_exact("produits"), - ), - _CategoryRule( - key="catalogue", - label="Catalogue DCAT", - description="Métadonnées de catalogage (dcat:Dataset, dcat:CatalogRecord).", - match=_exact("catalogue"), - ), - _CategoryRule( - key="ontologies", - label="Ontologies / schéma RDF", - description=( - "Définitions de classes et propriétés OWL/RDFS (def/base, def/geo, def/demo) " - "qui structurent les autres graphes. À consulter pour comprendre le schéma " - "d'un graphe de données, pas pour y chercher des données elles-mêmes." - ), - match=_prefix("def/"), - ), -] - -_CATEGORY_AUTRE = _CategoryRule( - key="autre", - label="Autre / non catégorisé", - description=( - "Graphes ne correspondant à aucune famille connue ci-dessus. Catégorie de secours : " - "si l'INSEE ajoute de nouveaux graphes sans mise à jour de ce serveur, ils " - "apparaissent ici plutôt que d'être mal classés." - ), - match=lambda path: True, -) - -_ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] -_RULES_BY_KEY = {r.key: r for r in _ALL_RULES} - - -def _relative_path(graph_uri: str) -> str: - if graph_uri.startswith(GRAPH_BASE): - return graph_uri[len(GRAPH_BASE):] - return graph_uri - - -def _categorize(graph_uri: str) -> _CategoryRule: - path = _relative_path(graph_uri) - for cat in CATEGORY_DEFS: - if cat.match(path): - return cat - return _CATEGORY_AUTRE - - -# --------------------------------------------------------------------------- -# Enum exposé au LLM pour le paramètre `category` (non-optionnel, choix guidé) -# --------------------------------------------------------------------------- - -class GraphCategoryChoice(StrEnum): - ALL = "ALL" - QUALITE_RAPPORTS = "qualite_rapports" - QUALITE_REFERENTIELS = "qualite_referentiels" - CODES_CONCEPTS_GENERIQUES = "codes_concepts_generiques" - NOMENCLATURES = "nomenclatures" - OPERATIONS_STATISTIQUES = "operations_statistiques" - DEMOGRAPHIE = "demographie" - GEOGRAPHIE = "geographie" - ORGANISATIONS = "organisations" - CONCEPTS = "concepts" - PRODUITS = "produits" - CATALOGUE = "catalogue" - ONTOLOGIES = "ontologies" - AUTRE = "autre" - - -def _category_choices_doc() -> str: - """Construit la liste 'clé (label): description' pour la description du champ.""" - lines = ["ALL (Toutes catégories): pas de filtre, vue condensée de tout."] - for rule in _ALL_RULES: - lines.append(f"{rule.key} ({rule.label}): {rule.description}") - return "\n".join(f"- {line}" for line in lines) - - -_CATEGORY_FIELD_DESCRIPTION = ( - "Catégorie de graphes à cibler. Attention les graphes des qualites sont nombreux (600 au total)\n" -) - - -# Note sur les vocabulaires connus, injectée dans la description de run_sparql. -KNOWN_VOCABULARIES_NOTE = """ -Vocabulaires principaux rencontrés dans cette base (au-delà de skos/xkos/dcterms) : -- sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualité. Un sdmx-mm:MetadataReport - a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des - sdmx-mm:ReportedAttribute rattachés via sdmx-mm:metadataReport. -- rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, StatisticalOperationSeries, - StatisticalOperationFamily (graphe "operations"), StatisticalIndicator (graphe "produits"), - StatutDiffusion... -- org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes - "organisations" et "organisations/insee"). -- dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). -Utilise RMES_list_graphs pour voir les grandes catégories de graphes avant de creuser -avec ce tool. -""".strip() - - -# --------------------------------------------------------------------------- -# Schémas Pydantic -- erreurs -# --------------------------------------------------------------------------- - -class GraphRow(BaseModel): - graph: str - triples: int - - -class SparqlErrorType(StrEnum): - INVALID_QUERY_FORM = "INVALID_QUERY_FORM" - TIMEOUT = "TIMEOUT" - SYNTAX_ERROR = "SYNTAX_ERROR" - HTTP_ERROR = "HTTP_ERROR" - NETWORK_ERROR = "NETWORK_ERROR" - EMPTY_QUERY = "EMPTY_QUERY" - - -class SparqlError(BaseModel): - type: SparqlErrorType - message: str - query: str - endpoint_message: Optional[str] = None - - -def _error_payload(error_type: SparqlErrorType, message: str, query: str, **extra: Any) -> dict[str, Any]: - payload = {"type": error_type, "message": message, "query": query} - payload.update(extra) - return {"error": payload} - - -# --------------------------------------------------------------------------- -# Helpers d'analyse de requête SPARQL -# --------------------------------------------------------------------------- - -_STRIP_PREFIX_RE = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) -_QUERY_FORM_RE = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") -_LIMIT_RE = re.compile(r"(?i)\bLIMIT\s+\d+\b") - - -def _detect_query_form(query: str) -> str: - body = _STRIP_PREFIX_RE.sub("", query) - match = _QUERY_FORM_RE.search(body) - return match.group(1).upper() if match else "UNKNOWN" - - -def _ensure_limit(query: str, query_form: str, max_rows: int) -> tuple[str, bool]: - if query_form not in ("SELECT", "CONSTRUCT"): - return query, False - if _LIMIT_RE.search(query): - return query, False - return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True - - -def _accept_header(query_form: str) -> str: - if query_form in ("SELECT", "ASK"): - return "application/sparql-results+json" - return "text/turtle" - - -# --------------------------------------------------------------------------- -# Exécution bas niveau (retourne un dict brut -- succès ou {"error": {...}}) -# --------------------------------------------------------------------------- - -async def _execute_sparql(query: str, timeout: float, max_rows: int) -> dict[str, Any]: - query_form = _detect_query_form(query) - - if query_form == "UNKNOWN": - return _error_payload( - SparqlErrorType.INVALID_QUERY_FORM, - "Impossible de détecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requête. " - "Vérifie la syntaxe SPARQL (pas GraphQL).", - query, - ) - - effective_query, limit_added = _ensure_limit(query, query_form, max_rows) - accept = _accept_header(query_form) - - try: - client = _get_client() - response = await client.post( - ENDPOINT, - data={"query": effective_query}, - headers={"Accept": accept}, - timeout=min(timeout, MAX_TIMEOUT), - ) - response.raise_for_status() - - except httpx.TimeoutException: - return _error_payload( - SparqlErrorType.TIMEOUT, - f"Le endpoint n'a pas répondu en moins de {timeout}s. " - "Restreins la requête (ajoute une clause GRAPH précise, réduis le LIMIT, " - "évite les scans sans filtre sur tous les graphes).", - query, - ) - - except httpx.HTTPStatusError as exc: - status = exc.response.status_code - body = exc.response.text[:2000] - if status == 400: - return _error_payload( - SparqlErrorType.SYNTAX_ERROR, - "Le endpoint a rejeté la requête (erreur de syntaxe SPARQL probable).", - query, - endpoint_message=body, - ) - return _error_payload( - SparqlErrorType.HTTP_ERROR, - f"Le endpoint a répondu {status}.", - query, - endpoint_message=body, - ) - - except httpx.RequestError as exc: - logger.warning("Erreur réseau vers %s: %s", ENDPOINT, exc) - return _error_payload( - SparqlErrorType.NETWORK_ERROR, - f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", - query, - ) - - if accept == "text/turtle": - return {"format": "turtle", "limit_added": limit_added, "data": response.text} - - result = response.json() - if limit_added: - result.setdefault("_meta", {})["limit_added"] = max_rows - result["_meta"]["hint"] = ( - f"Aucune clause LIMIT trouvée : une limite de {max_rows} a été ajoutée " - "automatiquement pour éviter une réponse trop volumineuse. " - "Passe max_rows pour l'augmenter si besoin." - ) - return result - - -async def _get_raw_graph_rows() -> dict[str, Any]: - """{"rows": [...]} en cas de succès, {"error": {...}} sinon.""" - now = time.time() - if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: - query = ( - "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } " - "GROUP BY ?g ORDER BY DESC(?nbTriples)" - ) - result = await _execute_sparql(query, timeout=45.0, max_rows=1000) - if "error" in result: - return result - rows = [ - {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} - for b in result["results"]["bindings"] - ] - _GRAPH_CACHE["data"] = rows - _GRAPH_CACHE["ts"] = now - - return {"rows": _GRAPH_CACHE["data"]} diff --git a/src/mcpdiffusion/helpers/schemas.py b/src/mcpdiffusion/helpers/schemas.py deleted file mode 100644 index d3bfe37..0000000 --- a/src/mcpdiffusion/helpers/schemas.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Shared response conventions. - -Policy (report decision): -- Successes are typed Pydantic models returned directly by the tool. -- Failures are raised as `fastmcp.exceptions.ToolError`. The MCP protocol - transports these as proper tool errors -- clients see an `isError=true` - payload, models don't mistake them for real data. - -This means: no more `{"ERROR": "..."}` string payloads, no more caller-side -type-sniffing, no more bare `print(log)`. - -Use `fail(code, message, retryable)` for the three common shapes: - - INVALID_INPUT -- caller passed something the tool can't use. - - EMPTY_RESULT -- nothing matched; NOT an error, just an empty list. - (only raise when the tool genuinely can't produce a meaningful result) - - BACKEND_UNAVAILABLE -- ES / upstream HTTP is down. retryable=True. - - UPSTREAM_ERROR -- upstream returned a non-4xx/5xx we don't handle. - - PARSE_ERROR -- we got a response but couldn't parse it. - - INVALID_QUERY -- (RMES) SPARQL parse error. -""" -from __future__ import annotations - -from typing import Literal - -from fastmcp.exceptions import ToolError - - -ErrorCode = Literal[ - "INVALID_INPUT", - "EMPTY_RESULT", - "BACKEND_UNAVAILABLE", - "UPSTREAM_ERROR", - "PARSE_ERROR", - "INVALID_QUERY", - "NOT_FOUND", - "UNKNOWN", -] - - -def fail( - code: ErrorCode, - message: str, - retryable: bool = False, -) -> None: - """Raise a standardized tool error. - - `message` should be actionable: name the offending parameter, suggest - the next step, include the shortest useful excerpt of the upstream error. - """ - prefix = f"[{code}] " - if retryable: - prefix = f"[{code}, retryable] " - raise ToolError(prefix + message) diff --git a/src/mcpdiffusion/infra/__init__.py b/src/mcpdiffusion/infra/__init__.py new file mode 100644 index 0000000..f0a00f5 --- /dev/null +++ b/src/mcpdiffusion/infra/__init__.py @@ -0,0 +1 @@ +"""Infrastructure: external clients (ES, HTTP, SPARQL).""" diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py new file mode 100644 index 0000000..6e1b9dd --- /dev/null +++ b/src/mcpdiffusion/infra/elasticsearch.py @@ -0,0 +1,39 @@ +"""Centralized Elasticsearch client singleton with injected settings.""" +from __future__ import annotations + +import logging + +from elasticsearch import Elasticsearch + +from ..config.settings import Settings, get_settings + +logger = logging.getLogger("mcp.main") + +_client: Elasticsearch | None = None + + +def get_es_client(settings: Settings | None = None) -> Elasticsearch: + """Return the shared Elasticsearch client, building it on first call.""" + global _client + if _client is None: + s = settings or get_settings() + if not s.es_host: + raise RuntimeError( + "ES_HOST environment variable is not set. " + "See .env.example for the expected value." + ) + _client = Elasticsearch( + s.es_host, + verify_certs=s.tls_verify, + request_timeout=30, + max_retries=2, + retry_on_timeout=True, + ) + logger.info("Elasticsearch client initialized for %s", s.es_host) + return _client + + +def reset_es_client() -> None: + """Drop the cached client. Used by tests / long-running reconfiguration.""" + global _client + _client = None diff --git a/src/mcpdiffusion/infra/http.py b/src/mcpdiffusion/infra/http.py new file mode 100644 index 0000000..7e893be --- /dev/null +++ b/src/mcpdiffusion/infra/http.py @@ -0,0 +1,33 @@ +"""Shared HTTP client factory with centralized TLS settings.""" +from __future__ import annotations + +import httpx + +from ..config.settings import Settings, get_settings + + +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" +) +_DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) + + +def create_async_client( + *, + settings: Settings | None = None, + headers: dict[str, str] | None = None, + follow_redirects: bool = False, + timeout: httpx.Timeout | None = None, +) -> httpx.AsyncClient: + """Create an httpx.AsyncClient with centralized TLS and timeout settings.""" + s = settings or get_settings() + merged_headers = {"User-Agent": _USER_AGENT} + if headers: + merged_headers.update(headers) + return httpx.AsyncClient( + verify=s.tls_verify, + headers=merged_headers, + follow_redirects=follow_redirects, + timeout=timeout or _DEFAULT_TIMEOUT, + ) diff --git a/src/mcpdiffusion/infra/sparql.py b/src/mcpdiffusion/infra/sparql.py new file mode 100644 index 0000000..55bbe52 --- /dev/null +++ b/src/mcpdiffusion/infra/sparql.py @@ -0,0 +1,24 @@ +"""Low-level SPARQL HTTP client for RMES.""" +from __future__ import annotations + +import httpx + +from ..config.settings import Settings, get_settings + +HEADERS_BASE = {"User-Agent": "MCP-RMeS/2.0"} + +_client: httpx.AsyncClient | None = None + + +def get_sparql_client(settings: Settings | None = None) -> httpx.AsyncClient: + """Return a shared httpx.AsyncClient for SPARQL queries, recreated if closed.""" + global _client + if _client is None or _client.is_closed: + _client = httpx.AsyncClient(headers=HEADERS_BASE) + return _client + + +def reset_sparql_client() -> None: + """Drop the cached client. Used by tests.""" + global _client + _client = None diff --git a/src/mcpdiffusion/middleware.py b/src/mcpdiffusion/middleware.py deleted file mode 100644 index 2aa1470..0000000 --- a/src/mcpdiffusion/middleware.py +++ /dev/null @@ -1,61 +0,0 @@ -from limits import storage, strategies, parse -from datetime import datetime -from zoneinfo import ZoneInfo -import os -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.status import HTTP_429_TOO_MANY_REQUESTS -from starlette.middleware.base import BaseHTTPMiddleware - -GLOBAL_REQUEST_MIN = int(os.getenv("GLOBAL_REQUEST_MIN", "100")) -_TZ = ZoneInfo(os.getenv("TZ", "Europe/Paris")) - -_limits_storage = storage.MemoryStorage() -_limiter = strategies.MovingWindowRateLimiter(_limits_storage) -_rate = parse(f"{GLOBAL_REQUEST_MIN}/minute") - - - -class RateLimitMiddleware(BaseHTTPMiddleware): - """Middleware qui applique le rate limiting par IP. - - Fonctionnement : - 1. Extrait l'IP du client - 2. Determine la limite applicable (specifique ou par defaut) - 3. Verifie si la requete est autorisee - 4. Ajoute les headers standard de rate limiting a la reponse - """ - - async def dispatch(self, request: Request, call_next): - # Identifier le client par son IP - client_ip = request.client.host if request.client else "unknown" - - rate_key = f"{client_ip}" - - # Verifier le rate limit - if not _limiter.hit(_rate, rate_key): - retry_after_ts = _limiter.get_window_stats(_rate, rate_key)[0] - retry_after_time = datetime.fromtimestamp(retry_after_ts, tz=_TZ).strftime("%H:%M:%S") - return JSONResponse( - status_code=HTTP_429_TOO_MANY_REQUESTS, - content={ - "detail": f"Trop de requetes. Reessayez apres {retry_after_time}.", - "retry_after": retry_after_time, - }, - headers={ - "Retry-After": str(int(retry_after_ts)), - "X-RateLimit-Limit": str(GLOBAL_REQUEST_MIN), - "X-RateLimit-Remaining": "0", - }, - ) - - # Requete autorisee : executer l'endpoint - response = await call_next(request) - - # Ajouter les headers de rate limiting a la reponse - remaining = _limiter.get_window_stats(_rate, rate_key)[1] - response.headers["X-RateLimit-Limit"] = str(GLOBAL_REQUEST_MIN) - response.headers["X-RateLimit-Remaining"] = str(remaining) - response.headers["X-RateLimit-Window"] = f"{60}s" - - return response \ No newline at end of file diff --git a/src/mcpdiffusion/models/__init__.py b/src/mcpdiffusion/models/__init__.py new file mode 100644 index 0000000..abd834a --- /dev/null +++ b/src/mcpdiffusion/models/__init__.py @@ -0,0 +1 @@ +"""Pydantic schemas for tool inputs and outputs.""" diff --git a/src/mcpdiffusion/models/feedback.py b/src/mcpdiffusion/models/feedback.py new file mode 100644 index 0000000..e013cb5 --- /dev/null +++ b/src/mcpdiffusion/models/feedback.py @@ -0,0 +1,30 @@ +"""Pydantic schemas for the feedback tool.""" +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class SendFeedbackInput(BaseModel): + username: str = Field( + description="Identifier for the feedback author (e.g., user name, role, or session ID).", + examples=["alice", "data_analyst", "session_abc123"], + ) + feedback: str = Field( + description=( + "Clear, actionable Markdown describing the issue or suggestion. Include context " + "(which tool, what happened), expected vs actual behavior, and proposed solutions " + "if applicable. Write as if filing a GitHub issue." + ), + examples=[ + "## Bug Report\n\n**Tool:** search_melodi_datasets\n\n**Issue:** No results returned " + "for 'prix du pain' even though dataset DS_PRIX exists.\n\n**Expected:** Should find " + "at least one matching dataset.\n\n**Proposed fix:** Check if the Elasticsearch index " + "includes this dataset.", + ], + ) + + +class SendFeedbackOutput(BaseModel): + status: str = "success" + message: str + timestamp: str diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py new file mode 100644 index 0000000..926213a --- /dev/null +++ b/src/mcpdiffusion/models/insee.py @@ -0,0 +1,269 @@ +"""Pydantic schemas for INSEE.fr tools.""" +from __future__ import annotations + +from enum import StrEnum +from typing import Optional + +from pydantic import BaseModel, Field + + +class INSEETheme(StrEnum): + ALL = "ALL" + METHODES = "Methodes" + DEMOGRAPHIE = "Demographie" + REVENUS = "Revenus - Pouvoir d'achat - Consommation" + CONDITIONS = "Conditions de vie - Societe" + TRAVAIL = "Marche du travail - Salaires" + ECONOMIE = "Economie - Conjoncture - Comptes nationaux" + DD = "Developpement durable - Environnement" + ENTREPRISES = "Entreprises" + SECTEURS = "Secteurs d'activite" + TERRITOIRES = "Territoires, villes et quartiers" + + +class INSEEGeo(StrEnum): + COM = "COM" + DEP = "DEP" + REG = "REG" + INTER = "INTER" + COMPRD = "COMPRD" + FRANCE = "FRANCE" + + +class ThemeConjoncture(StrEnum): + INDUSTRY = "Industrial production and activity" + BUILDING = "Construction and building sector" + HOUSING = "Housing and real estate" + RETAIL = "Retail, wholesale and services" + BUSINESS = "Business demographics and confidence" + EMPLOYMENT = "Employment, unemployment and labour market" + WAGES = "Wages and labour costs" + PUBLIC_SECTOR = "Public sector employment and pay" + CONSUMPTION = "Households, consumption and health" + PRICES = "Inflation and producer prices" + ACCOUNTING = "National accounts and public finance" + TRANSPORT = "Transport and tourism" + FINANCE = "Business financing" + + +# --- Shared output model --- + +class DocumentHit(BaseModel): + """Whitelisted publication record returned by INSEE.fr search tools.""" + id: str = Field(description="Elasticsearch document id.") + score: float = Field(description="Relevance score from Elasticsearch.") + titre: Optional[str] = None + soustitre: Optional[str] = None + chapo: Optional[str] = None + anneediffusion: Optional[str] = Field( + default=None, description="Publication year as indexed." + ) + zone: Optional[str] = Field( + default=None, description="Geographic zone (e.g. 'France', 'Bretagne')." + ) + theme: Optional[str] = None + collection_libelle: Optional[str] = Field( + default=None, + description="Collection the publication belongs to " + "(e.g. 'Insee Premiere', 'Informations rapides').", + ) + idproduit: Optional[str] = Field( + default=None, + description="INSEE product identifier (often equal to the ES id).", + ) + url: str = Field( + description="Relative URL ready to feed into `get_insee_document`." + ) + + +# --- search_insee_documents --- + +class SearchInseeDocumentsInput(BaseModel): + query: str = Field( + description="Natural-language search query describing the statistics to retrieve.", + examples=["population de Lyon", "taux de chomage 2024", "PIB France"], + ) + theme: INSEETheme = Field( + default=INSEETheme.ALL, + description="Optional top-level INSEE theme used to restrict the search. Default: ALL.", + ) + year_of_reference: Optional[int] = Field( + default=None, + description=( + "Hard filter on publication year (e.g. 2024). Leave null to " + "search all years." + ), + ) + geo_niveau: INSEEGeo = Field( + default=INSEEGeo.FRANCE, + description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", + ) + geo_keyword: Optional[str] = Field( + default=None, + description=( + "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " + "'Bouches-du-Rhone'). Leave null to skip geographic filtering." + ), + ) + number_of_results: int = Field( + default=10, + description="Maximum number of results to return.", + ge=1, + le=20, + ) + + +class SearchInseeDocumentsOutput(BaseModel): + results: list[DocumentHit] + count: int + + +# --- search_insee_chiffrecle --- + +class SearchInseeChiffrecleInput(BaseModel): + query: str = Field( + description="Natural-language search query describing the statistics to retrieve.", + examples=["population de Lyon", "taux de chomage 2024", "PIB France"], + ) + year_of_reference: Optional[int] = Field( + default=None, + description=( + "Hard filter on publication year (e.g. 2024). Leave null to " + "search all years." + ), + ) + geo_niveau: INSEEGeo = Field( + default=INSEEGeo.FRANCE, + description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", + ) + geo_keyword: Optional[str] = Field( + default=None, + description=( + "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " + "'Bouches-du-Rhone'). Leave null to skip geographic filtering." + ), + ) + number_of_results: int = Field( + default=10, + description="Maximum number of results to return.", + ge=1, + le=20, + ) + + +class SearchInseeChiffrecleOutput(BaseModel): + results: list[DocumentHit] + count: int + + +# --- search_insee_conjoncture --- + +class SearchInseeConjonctureInput(BaseModel): + query: str = Field( + description=( + "Natural-language query. The search is lexical and rewards " + "keyword breadth -- provide several synonyms and related notions." + ), + examples=["consommation", "hotel", "PIB"], + ) + theme_conjoncture: Optional[ThemeConjoncture] = Field( + default=None, + description=( + "Optional broad category to restrict the search. Each category " + "contains multiple sub-themes. Leave null to search across all." + ), + ) + year_of_reference: Optional[int] = Field( + default=None, + description=( + "Hard filter on publication year (e.g. 2024). Leave null to " + "search all years; for 'latest release' use cases, prefer " + "leaving null so the freshest match wins by score." + ), + ) + number_of_results: int = Field( + default=10, + description="Maximum number of results to return.", + ge=1, + le=20, + ) + + +class SearchInseeConjonctureOutput(BaseModel): + results: list[DocumentHit] + count: int + + +# --- get_insee_document --- + +class GetInseeDocumentInput(BaseModel): + list_of_url: list[str] = Field( + description=( + "List of relative URLs to retrieve (e.g. " + "'/fr/statistiques/4277658?sommaire=4318291')." + ), + examples=[["/fr/statistiques/4277658?sommaire=4318291"]], + ) + include_sommaire: bool = Field( + default=True, + description=( + "If True, parse the page's table-of-contents section alongside " + "the main content. Use once to discover structure, then False " + "for subsequent requests on the same page." + ), + ) + truncate_content: bool = Field( + default=True, + description=( + "If True (default), long markdown bodies are clipped to keep the " + "response compact for the model. Set to False only when the full " + "text is required." + ), + ) + + +class DocumentResult(BaseModel): + id: str = Field(description="The input URL that produced this entry.") + status: str = Field(description="'success' or 'error'.") + markdown_content: Optional[str] = None + sommaire: Optional[dict[str, dict[str, str]]] = Field( + default=None, + description=( + "Parsed table of contents as " + "{category: {title: url}}. None when include_sommaire=False " + "or when the page has no sommaire." + ), + ) + truncated: bool = Field( + default=False, + description="True if markdown_content was clipped due to size.", + ) + error: Optional[str] = Field( + default=None, + description="Human-readable error message when status == 'error'.", + ) + + +class GetInseeDocumentOutput(BaseModel): + results: list[DocumentResult] + count: int + + +# --- get_insee_homepage --- + +class KeyValueIndicator(BaseModel): + key: str = Field(description="Indicator name (e.g. 'smic', 'PIB annuel').") + alias: str = Field( + default="", + description="Optional alias / alternative name for the indicator.", + ) + value: str = Field( + description="Pre-computed textual description of the latest figure." + ) + + +class KeyIndicatorsOutput(BaseModel): + indicators: list[KeyValueIndicator] = Field( + description="Curated key indicators: name, alias and latest value.", + ) + count: int = Field(description="Number of indicators returned.") diff --git a/src/mcpdiffusion/models/melodi.py b/src/mcpdiffusion/models/melodi.py new file mode 100644 index 0000000..d8ef8dc --- /dev/null +++ b/src/mcpdiffusion/models/melodi.py @@ -0,0 +1,147 @@ +"""Pydantic schemas for Melodi tools.""" +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +# --- get_melodi_observations --- + +class GetMelodiObservationsInput(BaseModel): + dataset_id: str = Field( + description="Identifier of the Melodi dataset (from search_melodi_datasets).", + examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], + ) + list_of_year: list[int] = Field( + default_factory=list, + description=( + "Years to keep in the result set. Leave empty (the default) to " + "return all available years. Pass e.g. [2020, 2021, 2022] to keep " + "only those years." + ), + examples=[[], [2020, 2021, 2022]], + ) + dict_of_columns_and_values: dict[str, str] = Field( + default_factory=dict, + description=( + "Filters based on modality codes of columns. Leave empty to " + "return all rows. Keys are column ids (e.g. 'PRICES', 'GEO'); " + "values are the exact modality codes returned by " + "`search_melodi_modalities`." + ), + examples=[ + {"PRICES": "D"}, + {"PCS": "6", "GEO": "2025-FRANCE-FM"}, + ], + ) + number_of_results: int = Field( + default=100, + description="Maximum number of observations to return.", + ge=1, + le=1000, + ) + + +class GetMelodiObservationsOutput(BaseModel): + dataset_id: str + observations: list[dict[str, Any]] + count: int + + +# --- search_melodi_datasets --- + +class SearchMelodiDatasetsInput(BaseModel): + french_query: str = Field( + description=( + "Explicit French description of the statistical dataset to search. " + "Mention the phenomenon (inflation, births, unemployment), " + "geographic level, population or product if known. " + "Do NOT provide codes." + ), + examples=[ + "indice des prix a la consommation", + "deces par departement", + "prenoms des nouveau-nes", + "population communale", + "salaires des enseignants", + ], + ) + start_year: int = Field( + default=1900, + description="Dataset must contain data from at least this year.", + ) + end_year: int = Field( + default=2100, + description="Dataset must contain data up to at least this year.", + ) + number_of_results: int = Field( + default=5, + description="Maximum number of datasets to return, ordered by relevance.", + ge=1, + le=20, + ) + + +class DatasetDescription(BaseModel): + content: str + lang: str + + +class DatasetSearchResult(BaseModel): + dataset_id: str + dataset_columns: str = Field( + description=( + "Pipe-separated list of available columns formatted as " + "'COLUMN_ID Label'." + ) + ) + dataset_description: DatasetDescription + dataset_score: float + + +class SearchMelodiDatasetsOutput(BaseModel): + results: list[DatasetSearchResult] + + +# --- search_melodi_modalities --- + +class SearchMelodiModalitiesInput(BaseModel): + dataset_id: str = Field( + description="Identifier of the Melodi dataset (from search_melodi_datasets).", + examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], + ) + columns_id: list[str] = Field( + description="Identifiers of the columns within the dataset to search.", + examples=[["PRICES"], ["PRICES", "GEO"]], + ) + french_query: str = Field( + description=( + "Natural-language French query describing the modalities to " + "retrieve (e.g. 'cote de boeuf', 'Ile-de-France', 'female Maria')." + ), + examples=["prix", "boissons non alcoolisees"], + ) + number_of_results: int = Field( + default=10, + description="Maximum number of modalities to return per column.", + ge=1, + le=50, + ) + + +class Modality(BaseModel): + code: str + label_en: str + label_fr: str + score: float + + +class ColumnResult(BaseModel): + column_code: str + metadata_columns: str + matching_modalities: list[Modality] + + +class SearchMelodiModalitiesOutput(BaseModel): + results: list[ColumnResult] diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py new file mode 100644 index 0000000..e56e6d0 --- /dev/null +++ b/src/mcpdiffusion/models/rmes.py @@ -0,0 +1,162 @@ +"""Pydantic schemas for RMES (SPARQL) tools.""" +from __future__ import annotations + +from enum import StrEnum +from typing import Any, Literal, Optional + +from pydantic import BaseModel, Field + + +# --- Shared RMES constants exposed to tools --- + +DEFAULT_TIMEOUT = 20.0 +MAX_TIMEOUT = 60.0 +DEFAULT_ROW_LIMIT = 200 +MAX_ROW_LIMIT = 2000 + +GRAPH_BASE = "http://rdf.insee.fr/graphes/" + + +# --- Graph taxonomy --- + +class GraphCategoryChoice(StrEnum): + ALL = "ALL" + QUALITE_RAPPORTS = "qualite_rapports" + QUALITE_REFERENTIELS = "qualite_referentiels" + CODES_CONCEPTS_GENERIQUES = "codes_concepts_generiques" + NOMENCLATURES = "nomenclatures" + OPERATIONS_STATISTIQUES = "operations_statistiques" + DEMOGRAPHIE = "demographie" + GEOGRAPHIE = "geographie" + ORGANISATIONS = "organisations" + CONCEPTS = "concepts" + PRODUITS = "produits" + CATALOGUE = "catalogue" + ONTOLOGIES = "ontologies" + AUTRE = "autre" + + +# --- Error types --- + +class SparqlErrorType(StrEnum): + INVALID_QUERY_FORM = "INVALID_QUERY_FORM" + TIMEOUT = "TIMEOUT" + SYNTAX_ERROR = "SYNTAX_ERROR" + HTTP_ERROR = "HTTP_ERROR" + NETWORK_ERROR = "NETWORK_ERROR" + EMPTY_QUERY = "EMPTY_QUERY" + + +class SparqlError(BaseModel): + type: SparqlErrorType + message: str + query: str + endpoint_message: Optional[str] = None + + +class GraphRow(BaseModel): + graph: str + triples: int + + +# --- RMES_list_graphs --- + +class ListGraphsInput(BaseModel): + contains: Optional[str] = Field( + default=None, + description=( + "Filtre les graphes dont l'URI contient cette sous-chaine (insensible a la " + "casse), ex. 'naf' ou 'qualite/rapport'. Active automatiquement le detail " + "complet (`graphs`) dans les categories retenues." + ), + examples=["naf", "qualite/rapport", "geo"], + ) + category: GraphCategoryChoice = Field( + default=GraphCategoryChoice.ALL, + description="Categorie de graphes a cibler.", + ) + expand: bool = Field( + default=False, + description=( + "Si True, inclut la liste complete des graphes (URI + nb de triplets) pour " + "chaque categorie retenue, au lieu de seulement quelques exemples. Se " + "declenche automatiquement si `contains` est fourni ou `category != ALL`." + ), + ) + + +class CategoryBucket(BaseModel): + category: str + label: str + description: str + count: int + total_triples: int + examples: list[str] + graphs: Optional[list[GraphRow]] = None + + +class ListGraphsOutput(BaseModel): + total_graphs_matched: int + categories: list[CategoryBucket] + error: Optional[SparqlError] = None + + +# --- RMES_describe_resource --- + +class DescribeResourceInput(BaseModel): + uri: str = Field( + description="URI complete de la ressource RDF a decrire.", + examples=["http://id.insee.fr/codes/naf2025/section/A"], + ) + graph: str | None = Field( + default=None, + description=( + "URI d'un graphe nomme pour restreindre la recherche. Sans cette valeur (None par defaut), " + "la recherche se fait sur tous les graphes (plus lent)." + ), + ) + + +class ResourceProperty(BaseModel): + graph: str + direction: Literal["outgoing", "incoming"] + predicate: str + value: str + value_type: Optional[str] = None + lang: Optional[str] = None + + +class DescribeResourceOutput(BaseModel): + uri: str + properties: list[ResourceProperty] + count: int + error: Optional[SparqlError] = None + + +# --- RMES_run_sparql --- + +class RunSparqlInput(BaseModel): + full_sparql_query: str = Field( + description="Requete SPARQL complete (SELECT / ASK / CONSTRUCT / DESCRIBE).", + ) + timeout: float = Field( + default=DEFAULT_TIMEOUT, + description=f"Timeout en secondes (plafonne a {MAX_TIMEOUT}s).", + gt=0, + ) + max_rows: int = Field( + default=DEFAULT_ROW_LIMIT, + description=f"Limite de lignes ajoutee si absente de la requete (plafonnee a {MAX_ROW_LIMIT}).", + ge=1, + le=MAX_ROW_LIMIT, + ) + + +class RunSparqlOutput(BaseModel): + format: Literal["json", "turtle"] = "json" + limit_added: Optional[int] = None + hint: Optional[str] = None + variables: Optional[list[str]] = None + bindings: Optional[list[dict[str, Any]]] = None + turtle: Optional[str] = None + error: Optional[SparqlError] = None diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index b561556..12931d0 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -6,36 +6,30 @@ from __future__ import annotations import logging -import os import sys -from pathlib import Path # noqa: F401 (kept for future config discovery) from dotenv import load_dotenv from fastmcp import FastMCP from starlette.middleware.trustedhost import TrustedHostMiddleware -from .helpers.logging import MAIN_LOGGER_NAME, UVICORN_LOGGING_CONFIG +from .config.settings import get_settings +from .core.logging import MAIN_LOGGER_NAME, UVICORN_LOGGING_CONFIG +from .core.middleware import RateLimitMiddleware from .tools import register_tools -from mcpdiffusion.middleware import RateLimitMiddleware - load_dotenv() +settings = get_settings() logger = logging.getLogger(MAIN_LOGGER_NAME) - mcp = FastMCP("INSEE-mcp-diffusion") -toollist=os.getenv("TOOLLIST", None) - -register_tools(mcp, toollist=toollist) +register_tools(mcp, toollist=settings.toollist) app = mcp.http_app() - -# TrustedHostMiddleware: default permits any host. In production, set -# ALLOWED_HOSTS to a comma-separated list behind your reverse proxy. -_allowed_hosts_raw = os.getenv("ALLOWED_HOSTS", "*").strip() +# TrustedHostMiddleware +_allowed_hosts_raw = settings.allowed_hosts.strip() _allowed_hosts = ( ["*"] if _allowed_hosts_raw == "*" else [h.strip() for h in _allowed_hosts_raw.split(",") if h.strip()] @@ -46,30 +40,17 @@ "Set ALLOWED_HOSTS before exposing the server publicly." ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=_allowed_hosts) -app.add_middleware(RateLimitMiddleware) +app.add_middleware(RateLimitMiddleware, settings=settings) if __name__ == "__main__": import uvicorn - port_str = os.getenv("MCP_PORT", "8000") - host_str = os.getenv("MCP_HOST", "0.0.0.0") - try: - port = int(port_str) - except ValueError: - print( - f"Error: invalid MCP_PORT environment variable: {port_str!r}", - file=sys.stderr, - ) - sys.exit(1) - - forwarded_ips = os.getenv("FORWARDED_ALLOW_IPS", "*") - uvicorn.run( app, - host=host_str, - port=port, + host=settings.mcp_host, + port=settings.mcp_port, proxy_headers=True, - forwarded_allow_ips=forwarded_ips, + forwarded_allow_ips=settings.forwarded_allow_ips, log_level="info", log_config=UVICORN_LOGGING_CONFIG, ) diff --git a/src/mcpdiffusion/services/__init__.py b/src/mcpdiffusion/services/__init__.py new file mode 100644 index 0000000..de2060f --- /dev/null +++ b/src/mcpdiffusion/services/__init__.py @@ -0,0 +1 @@ +"""Business logic services.""" diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py new file mode 100644 index 0000000..db0ce0f --- /dev/null +++ b/src/mcpdiffusion/services/feedback.py @@ -0,0 +1,42 @@ +"""Business logic for the feedback tool.""" +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +from ..models.feedback import SendFeedbackInput, SendFeedbackOutput + +_FEEDBACK_DIR = Path(__file__).resolve().parent.parent / "feedback" +_FEEDBACK_FILE = _FEEDBACK_DIR / "feedback.md" + + +def _ensure_feedback_file() -> Path: + _FEEDBACK_DIR.mkdir(parents=True, exist_ok=True) + if not _FEEDBACK_FILE.exists(): + _FEEDBACK_FILE.write_text( + "# Feedback Log\n\n" + "This file collects feedback from users and the assistant about MCP tools, " + "server behavior, and suggestions for improvement. Each entry is timestamped " + "and formatted as Markdown for easy review.\n\n---\n\n", + encoding="utf-8", + ) + return _FEEDBACK_FILE + + +async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: + feedback_path = _ensure_feedback_file() + timestamp = datetime.now().isoformat(timespec="seconds") + + entry = ( + f"## {timestamp} — {params.username}\n\n" + f"{params.feedback}\n\n" + "---\n\n" + ) + + with feedback_path.open("a", encoding="utf-8") as f: + f.write(entry) + + return SendFeedbackOutput( + message="Feedback recorded successfully.", + timestamp=timestamp, + ) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py new file mode 100644 index 0000000..2096b4e --- /dev/null +++ b/src/mcpdiffusion/services/insee_document.py @@ -0,0 +1,189 @@ +"""Business logic for get_insee_document tool.""" +from __future__ import annotations + +from collections import defaultdict +from typing import Optional +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup +from trafilatura import extract +from trafilatura.settings import Extractor + +from ..config.settings import Settings, get_settings +from ..core.errors import fail +from ..infra.http import create_async_client +from ..models.insee import ( + DocumentResult, + GetInseeDocumentInput, + GetInseeDocumentOutput, +) + +_TRAFILATURA_OPTIONS = Extractor( + output_format="markdown", + links=True, + formatting=True, + source="insee.fr", + with_metadata=True, +) + +_MAX_MARKDOWN_CHARS = 30_000 + + +def _as_relative(url: str) -> str: + p = urlparse(url) + return f"{p.path}?{p.query}" if p.query else p.path + + +def _parse_sommaire(html: str, base_url: str) -> list[dict[str, str]]: + soup = BeautifulSoup(html, "lxml") + results: list[dict[str, str]] = [] + + sommaire_section = soup.find( + lambda t: t.has_attr("class") and any("sommaire" in c for c in t["class"]) + ) + if not sommaire_section: + return [] + + outer_ul = sommaire_section.find("ul", class_="sommaire") + if not outer_ul: + return [] + + for top_li in outer_ul.find_all("li", recursive=False): + heading_tag = top_li.find("h2") + if heading_tag: + category_name = heading_tag.get_text(strip=True) + inner_ul = top_li.find("ul", class_="sommaire") + if not inner_ul: + continue + for link_li in inner_ul.find_all("li", class_="lien-produit"): + a = link_li.find("a") + if not a: + continue + title = a.get_text(strip=True) + absolute = urljoin(base_url, a.get("href", "")) + rel_url = _as_relative(absolute) + results.append( + {"category": category_name, "title": title, "url": rel_url} + ) + else: + a = top_li.find("a") + if not a: + continue + title = a.get_text(strip=True) + absolute = urljoin(base_url, a.get("href", "")) + rel_url = _as_relative(absolute) + results.append({"category": "", "title": title, "url": rel_url}) + return results + + +def _format_sommaire(flat_items: list[dict[str, str]]) -> dict[str, dict[str, str]]: + grouped: dict[str, dict[str, str]] = defaultdict(dict) + for entry in flat_items: + grouped[entry["category"]][entry["title"]] = entry["url"] + return dict(grouped) + + +def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: + if len(text) <= limit: + return text, False + head_size = (limit * 2) // 3 + tail_size = limit - head_size - 200 + marker = ( + "\n\n\n\n" + ) + return text[:head_size] + marker + text[-tail_size:], True + + +async def _fetch_html(url: str, settings: Settings) -> str: + import httpx + full_url = settings.insee_base_url + url if not url.startswith(("http://", "https://")) else url + try: + async with create_async_client( + settings=settings, follow_redirects=True, + ) as client: + response = await client.get(full_url) + response.raise_for_status() + return response.text + except httpx.TimeoutException as exc: + fail( + "BACKEND_UNAVAILABLE", + f"insee.fr timed out fetching {full_url}: {exc}", + retryable=True, + ) + raise + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + fail( + "NOT_FOUND", + f"INSEE document not found at {full_url} (HTTP 404). " + "Verify the URL with `search_insee_documents`.", + ) + else: + fail( + "UPSTREAM_ERROR", + f"insee.fr returned HTTP {exc.response.status_code} for {full_url}.", + retryable=(500 <= exc.response.status_code < 600), + ) + raise + except httpx.HTTPError as exc: + fail( + "BACKEND_UNAVAILABLE", + f"Network error fetching {full_url}: {exc}", + retryable=True, + ) + raise + + +async def get_insee_document( + params: GetInseeDocumentInput, + *, + settings: Settings | None = None, +) -> GetInseeDocumentOutput: + s = settings or get_settings() + + if not params.list_of_url: + fail( + "INVALID_INPUT", + "list_of_url must contain at least one URL. " + "Use `search_insee_documents` to find URLs first.", + ) + + results: list[DocumentResult] = [] + for url in params.list_of_url: + try: + html = await _fetch_html(str(url), s) + markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" + if params.truncate_content: + markdown, truncated = _truncate(markdown) + else: + truncated = False + + sommaire: Optional[dict[str, dict[str, str]]] = None + if params.include_sommaire: + flat = _parse_sommaire(html, s.insee_base_url) + sommaire = _format_sommaire(flat) if flat else None + + results.append( + DocumentResult( + id=str(url), + status="success", + markdown_content=markdown, + sommaire=sommaire, + truncated=truncated, + error=None, + ) + ) + except Exception as exc: + results.append( + DocumentResult( + id=str(url), + status="error", + markdown_content=None, + sommaire=None, + truncated=False, + error=f"{type(exc).__name__}: {str(exc)[:500]}", + ) + ) + + return GetInseeDocumentOutput(results=results, count=len(results)) diff --git a/src/mcpdiffusion/helpers/es_search.py b/src/mcpdiffusion/services/insee_search.py similarity index 52% rename from src/mcpdiffusion/helpers/es_search.py rename to src/mcpdiffusion/services/insee_search.py index fc9745d..325ad8c 100644 --- a/src/mcpdiffusion/helpers/es_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -1,27 +1,7 @@ -"""Shared Elasticsearch query-building helpers for the produit index. - -Both `insee_search_documents` and `insee_search_conjoncture` run multi_match -searches against the same index shape. This module centralizes: - -- `build_text_clauses` -- query + year + keyword clauses -- `apply_collection_filters` -- common must_not / theme / chiffre_clef logic -- `execute_search` -- run the assembled bool query and shape the response -- `DocumentHit` -- the whitelisted output record returned to models - -The two search tools differ in: -- Which collection they restrict to (publications vs Informations rapides). -- Whether year_of_reference is added as a hard filter (both, now -- see note - below). - -Design note on `year_of_reference` ----------------------------------- -Historically the two tools disagreed: one used year as a *hard filter*, the -other as a *soft should-boost*. That made identical queries return different -document sets depending on which tool was called. Both tools now apply the -year as a hard filter with `year_matches_title` -- a document titled -"Comptes nationaux 2020" is about 2020, period. The conjoncture use case -("show me the latest monthly release") is still served by leaving the year -unset; the search returns the newest match by score. +"""Business logic for INSEE.fr Elasticsearch search tools. + +Centralizes query building, collection filtering, and search execution +for search_insee_documents, search_insee_conjoncture, and search_insee_chiffrecle. """ from __future__ import annotations @@ -29,48 +9,15 @@ from elasticsearch import Elasticsearch from elasticsearch.dsl import Q, Search -from pydantic import BaseModel, Field - -from .es import INDEX_PRODUITS, get_es_client -from ..tools.env import KEYS_THEME_NIV1, DICT_GEO - - -class DocumentHit(BaseModel): - """Whitelisted publication record returned by INSEE.fr search tools. - - Fields are chosen so a model can: - - present the result to the user (titre, soustitre, chapo, anneediffusion), - - chain into `get_insee_document` via `url`, - - rank/filter by geography and theme. - """ - id: str = Field(description="Elasticsearch document id.") - score: float = Field(description="Relevance score from Elasticsearch.") - titre: Optional[str] = None - soustitre: Optional[str] = None - chapo: Optional[str] = None - anneediffusion: Optional[str] = Field( - default=None, description="Publication year as indexed." - ) - zone: Optional[str] = Field( - default=None, description="Geographic zone (e.g. 'France', 'Bretagne')." - ) - theme: Optional[str] = None - collection_libelle: Optional[str] = Field( - default=None, - description="Collection the publication belongs to " - "(e.g. 'Insee Premiere', 'Informations rapides').", - ) - idproduit: Optional[str] = Field( - default=None, - description="INSEE product identifier (often equal to the ES id).", - ) - url: str = Field( - description="Relative URL ready to feed into `get_insee_document`." - ) + +from ..config.settings import Settings, get_settings +from ..data.geography import DICT_GEO +from ..data.themes import KEYS_THEME_NIV1 +from ..infra.elasticsearch import get_es_client +from ..models.insee import DocumentHit def _coerce_hit_value(value) -> Optional[str]: - """ES can return lists or nested dicts for some fields; normalize to str|None.""" if value is None: return None if isinstance(value, list): @@ -83,14 +30,7 @@ def build_text_clauses( year_of_reference: Optional[int], keywords: Iterable[str] = (), ) -> tuple[list, list, list, list]: - """Return (must, filter, should, must_not) clause lists. - - - `query` drives the main multi_match (fuzzy) + a phrase-match should on titre. - - `year_of_reference` is applied as a *hard filter* on title/subtitle/chapo. - Rationale: a document titled "Bilan 2020" *is* about 2020; soft boosting - was inconsistent across tools and led to same-query-different-results bugs. - - `keywords` are optional extras that add should-clauses with a boost. - """ + """Return (must, filter, should, must_not) clause lists.""" must: list = [] filters: list = [] should: list = [] @@ -117,9 +57,6 @@ def build_text_clauses( ) if year_of_reference: - # Hard filter (see module docstring). Matches the year appearing in - # title, subtitle or chapo. Documents without the year in any of - # these fields are excluded -- this is the intended strictness. filters.append( Q( "multi_match", @@ -152,20 +89,12 @@ def apply_collection_filters( geo_niveau: Optional[str] = None, geo_keyword: Optional[str] = None, ) -> tuple[list, list]: - """Apply INSEE-specific filters to the running clause lists. - - Returns the updated (filters, should) pair. `must_not_rapides` and - `must_only_rapides` are mutually exclusive callers: one restricts to - publications (documents), the other to rapid releases (conjoncture). - """ + """Apply INSEE-specific filters. Returns updated (filters, should).""" should: list = [] - # Collection gating -- one or the other, never both. if must_only_rapides: filters.append(Q("term", collection_libelle="Informations rapides")) elif must_not_rapides: - # Excludes rapid releases from the general publications search. - # They have their own dedicated tool (search_insee_conjoncture). filters.append( Q("bool", must_not=[Q("term", collection_libelle="Informations rapides")]) ) @@ -207,11 +136,13 @@ def execute_search( must_not: list, number_of_results: int, client: Optional[Elasticsearch] = None, + settings: Settings | None = None, ) -> list[DocumentHit]: """Run the assembled bool query and return whitelisted DocumentHit records.""" - client = client or get_es_client() + s = settings or get_settings() + client = client or get_es_client(s) - s = Search(using=client, index=INDEX_PRODUITS).query( + search = Search(using=client, index=s.es_index_produits).query( Q( "function_score", query=Q( @@ -225,8 +156,8 @@ def execute_search( boost_mode="sum", ) ) - s = s[: max(1, number_of_results)] - res = s.execute() + search = search[: max(1, number_of_results)] + res = search.execute() hits: list[DocumentHit] = [] for hit in res: diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py new file mode 100644 index 0000000..0d197e1 --- /dev/null +++ b/src/mcpdiffusion/services/melodi.py @@ -0,0 +1,329 @@ +"""Business logic for Melodi tools (observations, datasets, modalities).""" +from __future__ import annotations + +from typing import Any + +import httpx +from elasticsearch import ConnectionError as ESConnectionError +from elasticsearch import TransportError + +from ..config.settings import Settings, get_settings +from ..core.errors import fail +from ..infra.elasticsearch import get_es_client +from ..infra.http import create_async_client +from ..models.melodi import ( + ColumnResult, + DatasetSearchResult, + GetMelodiObservationsInput, + GetMelodiObservationsOutput, + Modality, + SearchMelodiDatasetsInput, + SearchMelodiDatasetsOutput, + SearchMelodiModalitiesInput, + SearchMelodiModalitiesOutput, +) + + +async def get_melodi_observations( + params: GetMelodiObservationsInput, + *, + settings: Settings | None = None, +) -> GetMelodiObservationsOutput: + s = settings or get_settings() + url = f"{s.melodi_data_base_url}/{params.dataset_id}" + try: + async with create_async_client(settings=s) as client: + response = await client.get( + url, + params=params.dict_of_columns_and_values or None, + ) + response.raise_for_status() + except httpx.TimeoutException as exc: + fail( + "BACKEND_UNAVAILABLE", + f"Melodi API timed out calling {url}: {exc}. Try again or narrow the query.", + retryable=True, + ) + raise + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + body_excerpt = (exc.response.text or "")[:500] + if status == 400: + fail( + "INVALID_INPUT", + f"Melodi API rejected the query (HTTP 400). " + f"Columns/values passed: {params.dict_of_columns_and_values}. " + f"Upstream detail: {body_excerpt}. " + "Verify modality codes with `search_melodi_modalities`.", + ) + elif status == 404: + fail( + "NOT_FOUND", + f"Melodi dataset {params.dataset_id!r} not found (HTTP 404). " + "Check the dataset_id with `search_melodi_datasets`.", + ) + else: + fail( + "UPSTREAM_ERROR", + f"Melodi API returned HTTP {status}: {body_excerpt}", + retryable=(500 <= status < 600), + ) + raise + except httpx.HTTPError as exc: + fail( + "BACKEND_UNAVAILABLE", + f"Could not reach Melodi API at {url}: {exc}", + retryable=True, + ) + raise + + try: + payload = response.json() + except ValueError as exc: + fail( + "PARSE_ERROR", + f"Melodi API returned non-JSON response: {exc}", + ) + raise + + observations = payload.get("observations") if isinstance(payload, dict) else None + if not isinstance(observations, list): + fail( + "PARSE_ERROR", + "Melodi API response did not contain an 'observations' list.", + ) + raise # pragma: no cover + + if params.list_of_year: + years_str = {str(y) for y in params.list_of_year} + observations = [ + obs + for obs in observations + if (obs.get("dimensions", {}) + .get("TIME_PERIOD", "") + .split("-")[0]) in years_str + ] + + sliced = observations[: params.number_of_results] + return GetMelodiObservationsOutput( + dataset_id=params.dataset_id, + observations=sliced, + count=len(sliced), + ) + + +async def search_melodi_datasets( + params: SearchMelodiDatasetsInput, + *, + settings: Settings | None = None, +) -> SearchMelodiDatasetsOutput: + s = settings or get_settings() + es = get_es_client(s) + filters: list[dict[str, Any]] = [] + if params.start_year: + filters.append({ + "range": { + "metadata.temporal.endPeriod": { + "gte": f"{params.start_year}-01-01" + } + } + }) + if params.end_year: + filters.append({ + "range": { + "metadata.temporal.startPeriod": { + "lte": f"{params.end_year}-12-31" + } + } + }) + + body = { + "size": params.number_of_results, + "query": { + "bool": { + "should": [ + { + "nested": { + "path": "metadata.title", + "query": { + "match": { + "metadata.title.content": { + "query": params.french_query, + "boost": 10, + } + } + }, + } + }, + { + "nested": { + "path": "metadata.abstract", + "query": { + "match": { + "metadata.abstract.content": { + "query": params.french_query, + "boost": 6, + } + } + }, + } + }, + { + "nested": { + "path": "metadata.description", + "query": { + "match": { + "metadata.description.content": { + "query": params.french_query, + "boost": 3, + } + } + }, + } + }, + { + "match": { + "variables_text": { + "query": params.french_query, + "boost": 5, + } + } + }, + ], + "filter": filters, + } + }, + } + + try: + ds_res = es.search(index=s.es_index_melodi_datasets, body=body) + except (ESConnectionError, TransportError) as exc: + fail( + "BACKEND_UNAVAILABLE", + f"Melodi datasets search backend unreachable: {exc}. " + "Verify ES_HOST and try again.", + retryable=True, + ) + raise + + results: list[DatasetSearchResult] = [] + for hit in ds_res.get("hits", {}).get("hits", []): + source = hit.get("_source", {}) + description = source.get("metadata", {}).get("description") + if isinstance(description, list) and description: + description = description[0] + elif isinstance(description, dict): + description = description + else: + description = {"content": "", "lang": "fr"} + results.append( + DatasetSearchResult( + dataset_id=hit.get("_id", ""), + dataset_columns=source.get("columns", ""), + dataset_description=description, + dataset_score=float(hit.get("_score") or 0.0), + ) + ) + return SearchMelodiDatasetsOutput(results=results) + + +async def search_melodi_modalities( + params: SearchMelodiModalitiesInput, + *, + settings: Settings | None = None, +) -> SearchMelodiModalitiesOutput: + s = settings or get_settings() + es = get_es_client(s) + + filters: list[dict[str, Any]] = [{"term": {"dataset_id": params.dataset_id}}] + if params.columns_id: + filters.append({"terms": {"code": params.columns_id}}) + + try: + ds_column = es.search( + index=s.es_index_melodi_columns, + size=20, + query={ + "bool": { + "filter": filters, + "should": [ + { + "match": { + "text": { + "query": params.french_query, + "boost": 2, + } + } + }, + { + "nested": { + "path": "modalities", + "score_mode": "max", + "query": { + "multi_match": { + "query": params.french_query, + "fields": [ + "modalities.code^5", + "modalities.label.en^3", + "modalities.label.fr^3", + ], + "fuzziness": "AUTO", + } + }, + "inner_hits": { + "size": params.number_of_results, + "sort": [{"_score": "desc"}], + }, + } + }, + ], + } + }, + ) + except (ESConnectionError, TransportError) as exc: + fail( + "BACKEND_UNAVAILABLE", + f"Melodi columns search backend unreachable: {exc}. " + "Verify ES_HOST and try again.", + retryable=True, + ) + raise + + results: list[ColumnResult] = [] + for hit in ds_column.get("hits", {}).get("hits", []): + modalities: list[Modality] = [] + inner_hits = ( + hit.get("inner_hits", {}) + .get("modalities", {}) + .get("hits", {}) + .get("hits", []) + ) + for m in inner_hits: + src = m.get("_source", {}) + label = src.get("label", {}) or {} + modalities.append( + Modality( + code=str(src.get("code", "")), + label_en=str(label.get("en", "")), + label_fr=str(label.get("fr", "")), + score=float(m.get("_score") or 0.0), + ) + ) + results.append( + ColumnResult( + column_code=str(hit.get("_source", {}).get("code", "")), + metadata_columns=str(hit.get("_source", {}).get("text", "")), + matching_modalities=modalities, + ) + ) + + if not results: + fail( + "EMPTY_RESULT", + f"No modalities matched for dataset_id={params.dataset_id!r}, " + f"columns_id={params.columns_id!r}, " + f"french_query={params.french_query!r}. " + "Verify the dataset_id and column ids with `search_melodi_datasets`, " + "then try a broader French query.", + ) + return SearchMelodiModalitiesOutput(results=results) diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py new file mode 100644 index 0000000..61a3005 --- /dev/null +++ b/src/mcpdiffusion/services/rmes.py @@ -0,0 +1,524 @@ +"""Business logic for RMES (SPARQL) tools. + +Contains: taxonomy, categorization, SPARQL execution, graph cache, +and high-level operations for the three RMES tools. +""" +from __future__ import annotations + +import logging +import re +import time +from typing import Any + +import httpx + +from ..config.settings import Settings, get_settings +from ..infra.sparql import get_sparql_client +from ..models.rmes import ( + GRAPH_BASE, + MAX_ROW_LIMIT, + MAX_TIMEOUT, + CategoryBucket, + DescribeResourceOutput, + GraphCategoryChoice, + GraphRow, + ListGraphsOutput, + ResourceProperty, + RunSparqlOutput, + SparqlError, + SparqlErrorType, + RunSparqlInput, + DescribeResourceInput, + ListGraphsInput, +) + +logger = logging.getLogger("mcp.rmes") + +# Cache for raw graph rows (expensive COUNT query) +_GRAPH_CACHE: dict[str, Any] = {"data": None, "ts": 0.0} +_GRAPH_CACHE_TTL = 3600.0 # 1h + + +# --- Known vocabularies note (injected in run_sparql description) --- + +KNOWN_VOCABULARIES_NOTE = """ +Vocabulaires principaux rencontres dans cette base (au-dela de skos/xkos/dcterms) : +- sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualite. Un sdmx-mm:MetadataReport + a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des + sdmx-mm:ReportedAttribute rattaches via sdmx-mm:metadataReport. +- rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, StatisticalOperationSeries, + StatisticalOperationFamily (graphe "operations"), StatisticalIndicator (graphe "produits"), + StatutDiffusion... +- org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes + "organisations" et "organisations/insee"). +- dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). +Utilise RMES_list_graphs pour voir les grandes categories de graphes avant de creuser +avec ce tool. +""".strip() + + +# --------------------------------------------------------------------------- +# Graph taxonomy +# --------------------------------------------------------------------------- + +CategoryMatcher = Any # Callable[[str], bool] + + +class _CategoryRule: + __slots__ = ("key", "label", "description", "match") + + def __init__(self, key: str, label: str, description: str, match: CategoryMatcher): + self.key = key + self.label = label + self.description = description + self.match = match + + +def _exact(*paths: str) -> CategoryMatcher: + allowed = set(paths) + return lambda path: path in allowed + + +def _prefix(prefix: str) -> CategoryMatcher: + return lambda path: path.startswith(prefix) + + +CATEGORY_DEFS: list[_CategoryRule] = [ + _CategoryRule( + key="qualite_rapports", + label="Rapports qualite", + description=( + "Un graphe par operation statistique documentee (sdmx-mm:MetadataReport), " + "structure selon le standard europeen SIMS. Contient les dimensions qualite " + "(pertinence, precision, actualite, coherence...) sous forme de " + "sdmx-mm:ReportedAttribute. Tous ces graphes ont un schema identique." + ), + match=_prefix("qualite/rapport/"), + ), + _CategoryRule( + key="qualite_referentiels", + label="Referentiels qualite", + description=( + "Vocabulaire SIMS-FR (simsv2fr), documents annexes (documents) et referentiel " + "territorial (territoires) associes aux rapports qualite." + ), + match=_exact("qualite/documents", "qualite/simsv2fr", "qualite/territoires"), + ), + _CategoryRule( + key="codes_concepts_generiques", + label="Concepts generiques de codification", + description=( + "Concepts transverses qualifiant des operations ou nomenclatures (Frequence, " + "Langue, ModeCollecte, UniteEnquetee, CategorieSource, StatutEnquete...) et " + "notes explicatives xkos. Ce n'est PAS une nomenclature metier -- voir " + "'nomenclatures' pour NAF/PCS/COICOP/etc." + ), + match=_exact("codes", "codes/nomenclatures"), + ), + _CategoryRule( + key="nomenclatures", + label="Nomenclatures (classifications officielles)", + description=( + "Nomenclatures statistiques officielles et leurs versions successives : " + "activites (NAF/NAFR), produits (CPF), professions et categories " + "socioprofessionnelles (PCS/PCSESE), consommation (COICOP), categories " + "juridiques (CJ), emplois (EAP/EMB par annee), tables de correspondance entre " + "versions (ex: nafr2-cpfr21)." + ), + match=_prefix("codes/"), + ), + _CategoryRule( + key="operations_statistiques", + label="Operations statistiques", + description=( + "Catalogue des operations (StatisticalOperation), series et familles " + "d'enquetes/collectes de l'Insee. C'est la cible (sdmx-mm:target) de chaque " + "rapport qualite." + ), + match=_exact("operations"), + ), + _CategoryRule( + key="demographie", + label="Demographie", + description="Populations legales par annee (popleg).", + match=_prefix("demo/"), + ), + _CategoryRule( + key="geographie", + label="Geographie", + description="Code officiel geographique (COG) : communes, decoupages administratifs.", + match=_prefix("geo/"), + ), + _CategoryRule( + key="organisations", + label="Organisations", + description=( + "Organismes producteurs de statistiques (services statistiques ministeriels...) " + "et unites organisationnelles internes de l'Insee." + ), + match=_prefix("organisations"), + ), + _CategoryRule( + key="concepts", + label="Concepts et definitions statistiques", + description="Themes statistiques et definitions de notions utilisees dans les publications.", + match=_prefix("concepts"), + ), + _CategoryRule( + key="produits", + label="Produits / indicateurs statistiques", + description="Indicateurs statistiques publies (StatisticalIndicator).", + match=_exact("produits"), + ), + _CategoryRule( + key="catalogue", + label="Catalogue DCAT", + description="Metadonnees de catalogage (dcat:Dataset, dcat:CatalogRecord).", + match=_exact("catalogue"), + ), + _CategoryRule( + key="ontologies", + label="Ontologies / schema RDF", + description=( + "Definitions de classes et proprietes OWL/RDFS (def/base, def/geo, def/demo) " + "qui structurent les autres graphes. A consulter pour comprendre le schema " + "d'un graphe de donnees, pas pour y chercher des donnees elles-memes." + ), + match=_prefix("def/"), + ), +] + +_CATEGORY_AUTRE = _CategoryRule( + key="autre", + label="Autre / non categorise", + description=( + "Graphes ne correspondant a aucune famille connue ci-dessus. Categorie de secours : " + "si l'INSEE ajoute de nouveaux graphes sans mise a jour de ce serveur, ils " + "apparaissent ici plutot que d'etre mal classes." + ), + match=lambda path: True, +) + +_ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] +_RULES_BY_KEY = {r.key: r for r in _ALL_RULES} + + +def _relative_path(graph_uri: str) -> str: + if graph_uri.startswith(GRAPH_BASE): + return graph_uri[len(GRAPH_BASE):] + return graph_uri + + +def _categorize(graph_uri: str) -> _CategoryRule: + path = _relative_path(graph_uri) + for cat in CATEGORY_DEFS: + if cat.match(path): + return cat + return _CATEGORY_AUTRE + + +# --------------------------------------------------------------------------- +# SPARQL query helpers +# --------------------------------------------------------------------------- + +_STRIP_PREFIX_RE = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) +_QUERY_FORM_RE = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") +_LIMIT_RE = re.compile(r"(?i)\bLIMIT\s+\d+\b") + + +def _detect_query_form(query: str) -> str: + body = _STRIP_PREFIX_RE.sub("", query) + match = _QUERY_FORM_RE.search(body) + return match.group(1).upper() if match else "UNKNOWN" + + +def _ensure_limit(query: str, query_form: str, max_rows: int) -> tuple[str, bool]: + if query_form not in ("SELECT", "CONSTRUCT"): + return query, False + if _LIMIT_RE.search(query): + return query, False + return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True + + +def _accept_header(query_form: str) -> str: + if query_form in ("SELECT", "ASK"): + return "application/sparql-results+json" + return "text/turtle" + + +def _error_payload(error_type: SparqlErrorType, message: str, query: str, **extra: Any) -> dict[str, Any]: + payload = {"type": error_type, "message": message, "query": query} + payload.update(extra) + return {"error": payload} + + +# --------------------------------------------------------------------------- +# Low-level SPARQL execution +# --------------------------------------------------------------------------- + +async def _execute_sparql( + query: str, + timeout: float, + max_rows: int, + *, + sparql_client_factory=None, + settings: Settings | None = None, +) -> dict[str, Any]: + s = settings or get_settings() + query_form = _detect_query_form(query) + + if query_form == "UNKNOWN": + return _error_payload( + SparqlErrorType.INVALID_QUERY_FORM, + "Impossible de detecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requete. " + "Verifie la syntaxe SPARQL (pas GraphQL).", + query, + ) + + effective_query, limit_added = _ensure_limit(query, query_form, max_rows) + accept = _accept_header(query_form) + + try: + client = sparql_client_factory() if sparql_client_factory else get_sparql_client(s) + response = await client.post( + s.rmes_endpoint, + data={"query": effective_query}, + headers={"Accept": accept}, + timeout=min(timeout, MAX_TIMEOUT), + ) + response.raise_for_status() + + except httpx.TimeoutException: + return _error_payload( + SparqlErrorType.TIMEOUT, + f"Le endpoint n'a pas repondu en moins de {timeout}s. " + "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " + "evite les scans sans filtre sur tous les graphes).", + query, + ) + + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + body = exc.response.text[:2000] + if status == 400: + return _error_payload( + SparqlErrorType.SYNTAX_ERROR, + "Le endpoint a rejete la requete (erreur de syntaxe SPARQL probable).", + query, + endpoint_message=body, + ) + return _error_payload( + SparqlErrorType.HTTP_ERROR, + f"Le endpoint a repondu {status}.", + query, + endpoint_message=body, + ) + + except httpx.RequestError as exc: + logger.warning("Erreur reseau vers %s: %s", s.rmes_endpoint, exc) + return _error_payload( + SparqlErrorType.NETWORK_ERROR, + f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", + query, + ) + + if accept == "text/turtle": + return {"format": "turtle", "limit_added": limit_added, "data": response.text} + + result = response.json() + if limit_added: + result.setdefault("_meta", {})["limit_added"] = max_rows + result["_meta"]["hint"] = ( + f"Aucune clause LIMIT trouvee : une limite de {max_rows} a ete ajoutee " + "automatiquement pour eviter une reponse trop volumineuse. " + "Passe max_rows pour l'augmenter si besoin." + ) + return result + + +async def _get_raw_graph_rows( + *, + sparql_client_factory=None, + settings: Settings | None = None, +) -> dict[str, Any]: + now = time.time() + if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: + query = ( + "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } " + "GROUP BY ?g ORDER BY DESC(?nbTriples)" + ) + result = await _execute_sparql( + query, timeout=45.0, max_rows=1000, + sparql_client_factory=sparql_client_factory, settings=settings, + ) + if "error" in result: + return result + rows = [ + {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} + for b in result["results"]["bindings"] + ] + _GRAPH_CACHE["data"] = rows + _GRAPH_CACHE["ts"] = now + + return {"rows": _GRAPH_CACHE["data"]} + + +# --------------------------------------------------------------------------- +# High-level tool operations +# --------------------------------------------------------------------------- + +def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: + buckets: dict[str, CategoryBucket] = {} + for row in rows: + cat = _categorize(row["graph"]) + bucket = buckets.get(cat.key) + if bucket is None: + bucket = CategoryBucket( + category=cat.key, + label=cat.label, + description=cat.description, + count=0, + total_triples=0, + examples=[], + ) + buckets[cat.key] = bucket + bucket.count += 1 + bucket.total_triples += row["triples"] + if len(bucket.examples) < 5: + bucket.examples.append(row["graph"]) + + ordered_keys = [c.key for c in CATEGORY_DEFS] + [_CATEGORY_AUTRE.key] + return [buckets[k] for k in ordered_keys if k in buckets] + + +async def list_graphs( + params: ListGraphsInput, + *, + sparql_client_factory=None, + settings: Settings | None = None, +) -> ListGraphsOutput: + raw = await _get_raw_graph_rows( + sparql_client_factory=sparql_client_factory, settings=settings, + ) + if "error" in raw: + return ListGraphsOutput( + total_graphs_matched=0, + categories=[], + error=SparqlError(**raw["error"]), + ) + rows = raw["rows"] + expand = params.expand + + if params.contains: + needle = params.contains.lower() + rows = [r for r in rows if needle in r["graph"].lower()] + expand = True + + if params.category != GraphCategoryChoice.ALL: + rows = [r for r in rows if _categorize(r["graph"]).key == params.category.value] + expand = True + + summary = _build_category_summary(rows) + + if expand: + rows_by_graph = {r["graph"]: r["triples"] for r in rows} + for bucket in summary: + bucket_rows = [ + GraphRow(graph=g, triples=t) + for g, t in rows_by_graph.items() + if _categorize(g).key == bucket.category + ] + bucket_rows.sort(key=lambda r: r.triples, reverse=True) + bucket.graphs = bucket_rows + + return ListGraphsOutput(total_graphs_matched=len(rows), categories=summary) + + +def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[ResourceProperty]: + props: list[ResourceProperty] = [] + for b in bindings: + props.append( + ResourceProperty( + graph=b["g"]["value"], + direction=b["direction"]["value"], + predicate=b["p"]["value"], + value=b["o"]["value"], + value_type=b["o"].get("type"), + lang=b["o"].get("xml:lang"), + ) + ) + return props + + +async def describe_resource( + params: DescribeResourceInput, + *, + sparql_client_factory=None, + settings: Settings | None = None, +) -> DescribeResourceOutput: + graph_clause = f"<{params.graph}>" if params.graph else "?g" + graph_values = f"VALUES ?g {{ <{params.graph}> }}" if params.graph else "" + query = f""" + SELECT ?g ?direction ?p ?o WHERE {{ + {graph_values} + {{ + GRAPH {graph_clause} {{ <{params.uri}> ?p ?o }} + BIND("outgoing" AS ?direction) + }} UNION {{ + GRAPH {graph_clause} {{ ?o ?p <{params.uri}> }} + BIND("incoming" AS ?direction) + }} + }} LIMIT {MAX_ROW_LIMIT} + """ + from ..models.rmes import DEFAULT_TIMEOUT + result = await _execute_sparql( + query, timeout=DEFAULT_TIMEOUT, max_rows=MAX_ROW_LIMIT, + sparql_client_factory=sparql_client_factory, settings=settings, + ) + + if "error" in result: + return DescribeResourceOutput( + uri=params.uri, properties=[], count=0, error=SparqlError(**result["error"]) + ) + + properties = _parse_bindings_to_properties(result["results"]["bindings"]) + return DescribeResourceOutput(uri=params.uri, properties=properties, count=len(properties)) + + +async def run_sparql( + params: RunSparqlInput, + *, + sparql_client_factory=None, + settings: Settings | None = None, +) -> RunSparqlOutput: + if not params.full_sparql_query or not params.full_sparql_query.strip(): + return RunSparqlOutput( + error=SparqlError( + type=SparqlErrorType.EMPTY_QUERY, + message="La requete est vide.", + query=params.full_sparql_query, + ) + ) + + max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) + result = await _execute_sparql( + params.full_sparql_query, timeout=params.timeout, max_rows=max_rows, + sparql_client_factory=sparql_client_factory, settings=settings, + ) + + if "error" in result: + return RunSparqlOutput(error=SparqlError(**result["error"])) + + if result.get("format") == "turtle": + return RunSparqlOutput( + format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"] + ) + + meta = result.get("_meta", {}) + return RunSparqlOutput( + format="json", + limit_added=meta.get("limit_added"), + hint=meta.get("hint"), + variables=result.get("head", {}).get("vars"), + bindings=result.get("results", {}).get("bindings"), + ) diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 7852b88..147ae07 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -6,7 +6,7 @@ """ from __future__ import annotations -from mcp.server.fastmcp import FastMCP +from fastmcp import FastMCP from .melodi_get_observations import register_get_melodi_observations from .melodi_search_datasets import register_search_melodi_datasets @@ -21,10 +21,10 @@ from .rmes_run_sparql import register_rmes_run_sparql from .extras_send_feedback import register_extras_send_feedback -def register_tools(mcp: FastMCP, toollist:str|None=None) -> None: +def register_tools(mcp: FastMCP, toollist: str | None = None) -> None: """Register all MCP tools with the given FastMCP instance.""" # INSEE.fr - if toollist==("insee"): + if toollist == ("insee"): register_search_insee_documents(mcp) register_get_insee_homepage(mcp) register_get_insee_document(mcp) @@ -32,13 +32,13 @@ def register_tools(mcp: FastMCP, toollist:str|None=None) -> None: register_search_insee_chiffreclef(mcp) # Melodi - if toollist==("melodi"): + if toollist == ("melodi"): register_search_melodi_datasets(mcp) register_search_melodi_modalities(mcp) register_get_melodi_observations(mcp) # RMES (SPARQL) - if toollist==("rmes"): + if toollist == ("rmes"): register_rmes_list_graphs(mcp) register_rmes_describe_resource(mcp) register_rmes_run_sparql(mcp) @@ -55,4 +55,4 @@ def register_tools(mcp: FastMCP, toollist:str|None=None) -> None: register_rmes_list_graphs(mcp) register_rmes_describe_resource(mcp) register_rmes_run_sparql(mcp) - register_extras_send_feedback(mcp) \ No newline at end of file + register_extras_send_feedback(mcp) diff --git a/src/mcpdiffusion/tools/env.py b/src/mcpdiffusion/tools/env.py deleted file mode 100644 index bd87bde..0000000 --- a/src/mcpdiffusion/tools/env.py +++ /dev/null @@ -1,686 +0,0 @@ -""" -Tool metadata (name, description, version) and shared enums. - -Design notes: -- Tool *names* are English snake_case; French is kept only where it is - actual data (enum literals that hit the ES index, user-supplied queries). -- `CURRENT_DATE` is computed lazily so long-running servers always report - today's date, not the day the process started. -- `ES_HOST` is the single source of truth for the Elasticsearch endpoint. -- Tool descriptions describe the *final* schemas; rewrite in lockstep - when schemas change. -""" -from datetime import date -from typing import Literal - - -def current_date_iso() -> str: - """Return today's date as ISO-8601. Called at description render time - so a long-running server doesn't ship stale dates to models.""" - return date.today().isoformat() - - -# --- MELODI tools ----------------------------------------------------------- - -GET_DATASET = { - "tool_name": "get_melodi_observations", - "tool_description": ( - "Retrieve a filtered set of observations from a Melodi dataset. " - "The Melodi API holds official, high-granularity statistics " - "(prices, mortality, names, etc.).\n" - "\n" - "WHEN TO USE\n" - "- You already know the exact `dataset_id` (from `search_melodi_datasets`) " - "AND the modality codes you want to filter on " - "(from `search_melodi_modalities`).\n" - "\n" - "WHEN NOT TO USE\n" - "- You are still looking for the right dataset. Use `search_melodi_datasets` first.\n" - "- You need concept definitions or code-list vocabularies. Use `query_insee_rmes`.\n" - "\n" - "WORKFLOW (chain with companion tools)\n" - "1. `search_melodi_datasets` -> dataset_id + column ids\n" - "2. `search_melodi_modalities` -> exact modality codes for filtering\n" - "3. THIS TOOL (`get_melodi_observations`) -> final observations\n" - "\n" - "OUTPUT\n" - "A list of observations with dimensions, attributes and the numeric " - "measure (with unit). Returns an empty list when no rows match; " - "a structured error when the upstream API fails or inputs are invalid.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_DATASET = { - "tool_name": "search_melodi_datasets", - "tool_description": ( - "Search the INSEE Melodi dataset catalogue by French-language natural " - "language query. Each dataset has a unique `dataset_id`; the tool maps " - "the query to internal metadata to return the most relevant matches.\n" - "\n" - "WHEN TO USE\n" - "- The user asks for a specific statistic (price of a product, " - "mortality by region, frequency of a name, etc.) and you need to " - "locate the right dataset before fetching rows.\n" - "\n" - "WHEN NOT TO USE\n" - "- Generic, up-to-date indicator questions (use `get_insee_homepage`).\n" - "- Full-text analysis of a published report (use `search_insee_documents`).\n" - "- Definition/ontology lookups (use `query_insee_rmes`).\n" - "\n" - "TIPS\n" - "- Matching is lexical. Make `french_query` explicit and rich in French " - "synonyms: e.g. `\"indice des prix a la consommation\"`, " - "`\"deces par departement\"`, `\"prenoms des nouveau-nes\"`.\n" - "- Use `start_year` / `end_year` to narrow the temporal range. Leaving " - "both at default covers all years.\n" - "\n" - "NEXT STEP\n" - "Pass the returned `dataset_id` and column ids to " - "`search_melodi_modalities`, then feed the resolved codes into " - "`get_melodi_observations`.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_MODALITIES = { - "tool_name": "search_melodi_modalities", - "tool_description": ( - "Given a Melodi dataset and one or more column identifiers, rank the " - "most relevant modalities (codes/labels) for a free-text French query. " - "The result is what you need to filter rows in `get_melodi_observations`.\n" - "\n" - "WHEN TO USE\n" - "- You have a `dataset_id` (from `search_melodi_datasets`) and want " - "to find the exact modality code for a concept like `cote de boeuf`, " - "`Ile-de-France`, or `female Maria`.\n" - "\n" - "WHEN NOT TO USE\n" - "- You don't yet know the dataset. Run `search_melodi_datasets` first.\n" - "\n" - "INPUT\n" - "- `dataset_id` -- from a previous search result.\n" - "- `columns_id` -- which columns to search (e.g. `[\"PRICES\", \"GEO\"]`).\n" - "- `french_query` -- natural-language query in French.\n" - "\n" - "OUTPUT\n" - "A list of matching columns, each containing its `code`, metadata text " - "and the top-scoring `matching_modalities` with `code`, `label_fr`, " - "`label_en` and `score`. Empty list when nothing matches.\n" - "\n" - "NEXT STEP\n" - "Use the modality `code` values as entries in " - "`get_melodi_observations.dict_of_columns_and_values`.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# --- INSEE.fr tools --------------------------------------------------------- - -GET_DOCUMENT = { - "tool_name": "get_insee_document", - "tool_description": ( - "Fetch and parse a single INSEE publication from a known URL and " - "return its full text in markdown. Use ONLY when you already have one " - "or more explicit URLs (e.g. from `search_insee_documents` or from " - "the `link` fields returned by `get_insee_homepage`).\n" - "\n" - "WHEN TO USE\n" - "- You have a concrete URL of the form `/fr/statistiques/` or " - "`/fr/statistiques/?sommaire=`.\n" - "\n" - "WHEN NOT TO USE\n" - "- You are still looking for the right publication. Use " - "`search_insee_documents` first.\n" - "- You need a quick, up-to-date indicator. Use `get_insee_homepage`.\n" - "\n" - "INPUT\n" - "- `list_of_url` -- list of relative URLs to fetch (e.g. " - "`[\"/fr/statistiques/4277658?sommaire=4318291\"]`).\n" - "- `include_sommaire` -- also parse the page's table-of-contents " - "section. Use once to discover the structure of a multi-section " - "publication, then turn it off for subsequent requests on the same page.\n" - "- `truncate_content` -- when True (default), long markdown bodies are " - "clipped to keep the response compact for the model; set to False only " - "when you genuinely need the full text.\n" - "\n" - "OUTPUT\n" - "A uniform envelope: `{ status, results: [ { id, status, " - "markdown_content, sommaire, error, truncated } ], count }`. Each " - "per-URL entry has the same keys whether it succeeded or failed, so " - "downstream code can iterate without type-sniffing.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_DOCUMENTS = { - "tool_name": "search_insee_documents", - "tool_description": ( - "Search the INSEE catalogue of official statistical publications " - "(Insee Premiere, Insee Analyses, Dossiers, References, Focus, ...). " - "Returns structured publication records; pass the URL of a record to " - "`get_insee_document` to fetch the full text.\n" - "\n" - "⚠️ ROUTING PRIORITY\n" - "- Simple statistics (population, inflation, chômage, PIB, salaires) " - "by region/department? → Use `search_chiffres_clefs_insee` FIRST.\n" - "- Granular product data (e.g., beef rib price 2000)? → Use " - "`search_melodi_datasets` FIRST.\n" - "- This tool is for ANALYSIS, CONTEXT, and COMPLEX NARRATIVES.\n" - "\n" - "WHEN TO USE THIS TOOL\n" - "- Impact analyses (e.g., 'covid effects on tourism').\n" - "- Historical evolution and trends (e.g., 'unemployment 1990-2026').\n" - "- Detailed methodological or definitional content.\n" - "- Regional/departmental profiles with socioeconomic context.\n" - "- Specific thematic deep-dives (demography, labour market, inequalities, " - "environment, housing, ...). \n" - "- Comparative studies or cross-cutting analyses.\n" - "\n" - "WHEN NOT TO USE THIS TOOL\n" - "- Simple factual questions ('What is X region's population?') → " - "`search_chiffres_clefs_insee`.\n" - "- Quick, up-to-date headline indicators → `get_insee_homepage`.\n" - "- Latest monthly/quarterly rapid releases → `search_insee_conjoncture`.\n" - "- Vocabulary / code definitions / classifications → `query_insee_rmes`.\n" - "- Granular historical time series (product prices, individual wages) → " - "`search_melodi_datasets`.\n" - "\n" - "HOW TO SEARCH WELL\n" - "- `query` -- rich natural-language query with synonyms, context, " - "and target year/geography if relevant.\n" - "- `chiffre_clef=False` (default) -- general publications. Set to True " - "ONLY for 'essentials sur...' publications (essentiel sur l'inflation, " - "etc.), but prefer `search_chiffres_clefs_insee` for those instead.\n" - "- `geo_niveau` + `geo_keyword` -- territorial filtering " - "(COM/DEP/REG/INTER/COMPRD/FRANCE).\n" - "- `theme` -- restrict to top-level theme (Demographie, " - "Marche du travail, Economie, etc.). Default ALL.\n" - "- `year_of_reference` -- hard filter on publication year; null = all years.\n" - "\n" - "EXAMPLES\n" - "✅ 'impacts du covid sur l'emploi en Île-de-France' → this tool\n" - "✅ 'inégalités de revenus régionales' → this tool\n" - "❌ 'population Loire-Atlantique 2025' → search_chiffres_clefs_insee\n" - "❌ 'prix côte de boeuf 2000' → search_melodi_datasets\n" - "\n" - "OUTPUT\n" - "List of publications: `{ id, score, titre, soustitre, chapo, " - "anneediffusion, zone, theme, url }`. Feed `url` to `get_insee_document`.\n" - f"\n" - f"Current date is {current_date_iso()}.\n" - ), - "tool_metadata": {"version": "6.0", "author": "mirlon"}, -} - -SEARCH_CHIFFRECLEF = { - "tool_name": "search_insee_chiffrecle", - "tool_description": "Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : données synthétiques, \n" - "comparaisons régionales/départementales et statistiques factuelles simples.\n" - "À utiliser EN PRIORITÉ pour : population, inflation, chômage, PIB, salaires, \n" - "prix par catégorie, comparaisons géographiques (région, département, commune).\n" - "À utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', 'Taux de chômage en 2024 ?', 'Inflation en juillet 2026 ?'\n" - "À NE PAS utiliser pour : analyses détaillées, impacts/contexte, tendances \n" - "complexes, données produit granulaires historiques (→ utiliser search_melodi_datasets \n" - "ou search_insee_documents selon le contexte).\n" - "Retourne directement les tableaux synthétiques prêts à l'emploi.\n", - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_CONJONCTURE = { - "tool_name": "search_insee_conjoncture", - "tool_description": ( - "Search INSEE Rapid Releases (Informations rapides): short, recurring " - "publications reporting the latest monthly/quarterly/annual results for " - "major economic and social indicators (prices, employment, production, " - "housing, wages, national accounts, ...).\n" - "\n" - "WHEN TO USE\n" - "- The user asks for the *latest* monthly/quarterly release of a " - "named indicator (e.g. last month's consumer confidence, " - "last quarter's GDP estimate). Prefer the most recent edition.\n" - "\n" - "WHEN NOT TO USE\n" - "- Generic up-to-date indicator on the homepage: `get_insee_homepage`.\n" - "- Deep, peer-reviewed analysis: `search_insee_documents`.\n" - "\n" - "HOW TO SEARCH WELL\n" - "- `query` -- provide several synonyms and related notions; the " - "search is lexical and rewards keyword breadth.\n" - "- `theme_conjoncture` -- optional broad category (Industrial " - "production and activity, Inflation and producer prices, " - "Employment, unemployment and labour market, ...). Leave null to " - "search across all categories.\n" - "- `year_of_reference` -- hard filter on publication year; leave null " - "to search all years.\n" - "\n" - "OUTPUT\n" - "A list of publications: `{ id, score, titre, soustitre, chapo, " - "anneediffusion, zone, theme, url }`.\n" - f"\n" - f"Current date is {current_date_iso()}.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -GET_HOMEPAGE = { - "tool_name": "get_insee_homepage", - "tool_description": ( - "Retrieve the INSEE home page with the latest key indicators at national level" - "published by the institute (population, inflation, unemployment, " - "GDP growth, ...).\n" - "\n" - "WHEN TO USE -- preferred FIRST step for any generic, up-to-date " - "statistical question. It gives the most recent official figure " - "instantly, without searching individual documents.\n" - "\n" - "WHEN NOT TO USE\n" - "- User asks for a previous year's figure. Use `search_insee_documents` " - "or `search_insee_conjoncture` with `year_of_reference`.\n" - "\n" - "OUTPUT\n" - "- `mainIndicators` -- each with name, value, description and a link " - "to the underlying official product (pass the link to `get_insee_document`).\n" - "- `lastArticles` -- recent short articles with title, date, " - "collection and link.\n" - "- `keyGraphics` -- selection of recent graphical publications.\n" - "\n" - "WORKFLOW\n" - "1. Call this tool.\n" - "2. Present the indicator value + description + link.\n" - "3. Follow up with `search_insee_documents` or `search_insee_conjoncture` " - "only if the user needs deeper tables or historic series.\n" - f"\n" - f"Current date is {current_date_iso()}.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# --- RMES (SPARQL) ---------------------------------------------------------- - -RMES_SPARQL = { - "tool_name": "query_insee_rmes", - "tool_description": ( - "Run a SPARQL query against the INSEE semantic graph (RMES). " - "RMES holds INSEE metadata, concepts, definitions and code lists " - "(SKOS / XKOS). It does NOT hold observations.\n" - "\n" - "WHEN TO USE\n" - "- Concept definitions (`\"what is inflation\"`).\n" - "- Code-list lookups (NAF 2025 activity codes, PCS 2020, CPFR 21, " - "COICOP 2018, ...).\n" - "\n" - "WHEN NOT TO USE\n" - "- Actual data points or observations (use Melodi tools).\n" - "- Published reports (use INSEE.fr tools).\n" - "\n" - "INPUT\n" - "- `sparql_query` -- a complete, valid SPARQL query. Do NOT wrap it " - "in quotes. Generate it from one of the two templates below.\n" - "\n" - "GRAPH 1: code lists (`/graphes/codes/xxx`)\n" - "Allowed `xxx`: naf2025, pcsese2017, emb2026, eap2025, cpfr21, " - "coicop2018, pcs2020. Template (find up to 10 codes whose text " - "contains a keyword):\n" - "\n" - " SELECT ?g ?s ?p ?o\n" - " WHERE {\n" - " VALUES ?g { }\n" - " GRAPH ?g {\n" - " ?s ?p ?o .\n" - " FILTER( CONTAINS(LCASE(STR(?o)), \"extraction\") )\n" - " }\n" - " }\n" - " LIMIT 10\n" - "\n" - "GRAPH 2: concept definitions (`/graphes/concepts/definitions`)\n" - "Enriched with SKOS + XKOS. Template (find definitions whose " - "French label contains a keyword):\n" - "\n" - " PREFIX skos: \n" - " PREFIX xkos: \n" - " SELECT ?concept ?label ?definitionText\n" - " WHERE {\n" - " GRAPH {\n" - " ?concept skos:prefLabel ?label ;\n" - " skos:definition ?definitionResource .\n" - " ?definitionResource xkos:plainText ?definitionText .\n" - " FILTER(lang(?label) = \"fr\")\n" - " FILTER(lang(?definitionText) = \"fr\")\n" - " FILTER( CONTAINS(LCASE(STR(?label)), LCASE(\"inflation\")) )\n" - " }\n" - " }\n" - " LIMIT 10\n" - "\n" - "OUTPUT\n" - "The raw SPARQL JSON response on success; a structured error when " - "the query is malformed (400) or the endpoint is unavailable.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -RMES_LIST_GRAPHS = { - "tool_name": "RMES_list_graphs", - "tool_description": ( - "Liste les graphes nommés disponibles dans la base RDF de l'INSEE (RMES). " - "Utilise ce tool EN PREMIER pour découvrir quels graphes existent avant " - "d'écrire une requête SPARQL avec RMES_run_sparql -- il y a plus de 700 graphes.\n" - "\n" - "Par défaut (`category=ALL`), le résultat est une vue CONDENSÉE par catégorie, " - "avec un compteur et quelques URIs d'exemple par catégorie -- pas la liste plate " - "des 700+ graphes. Choisis une catégorie précise dans le paramètre `category` " - "pour cibler une famille, ou utilise `contains` pour une recherche libre par " - "sous-chaîne. Une catégorie \"autre\" recueille tout graphe ne correspondant à " - "aucune famille connue." - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -RMES_DESCRIBE_RESOURCE = { - "tool_name": "RMES_describe_resource", - "tool_description": ( - "Récupère toutes les propriétés connues (prédicat -> valeur) d'une ressource RDF " - "identifiée par son URI complète. Combine automatiquement les propriétés où la " - "ressource est sujet ET celles où elle est objet (utile pour remonter des relations " - "skos:broader par exemple). Restreins avec `graph` si tu sais déjà où chercher -- " - "sinon la recherche se fait sur tous les graphes, ce qui est plus lent." - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -RMES_RUN_SPARQL = { - "tool_name": "RMES_run_sparql", - "tool_description": ( - "Exécute une requête SPARQL libre sur RMES, la base de métadonnées, nomenclatures " - "et définitions de l'INSEE (elle ne contient PAS les chiffres/données, voir " - "get_MELODI_datasets pour ça).\n" - "\n" - "AVANT d'écrire une requête complexe : appelle RMES_list_graphs pour connaître les " - "catégories de graphes disponibles.\n" - "\n" - "Bonnes pratiques :\n" - "- Toujours filtrer sur un ou plusieurs graphes précis avec GRAPH { ... } ou " - " VALUES ?g { } plutôt que de scanner tous les graphes.\n" - "- Toujours ajouter FILTER(lang(?label) = \"fr\") sur les littéraux SKOS pour éviter " - " les doublons multilingues.\n" - "- Une clause LIMIT est fortement recommandée ; si absente, `max_rows` est ajoutée " - " automatiquement (indiqué dans la réponse via `limit_added`/`hint`).\n" - "- Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures " - " statistiques : ClassificationLevel, ExplanatoryNote), dcterms (métadonnées), " - " rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE).\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - - -# --- Shared enums / constants ---------------------------------------------- - -INSEE_GEO = Literal[ - "COM", - "DEP", - "REG", - "INTER", - "COMPRD", - "FRANCE", -] - - -INSEE_THEME_NIV1 = Literal[ - "Demographie", - "Revenus - Pouvoir d'achat - Consommation", - "Conditions de vie - Societe", - "Marche du travail - Salaires", - "Economie - Conjoncture - Comptes nationaux", - "Developpement durable - Environnement", - "Entreprises", - "Secteurs d'activite", - "Territoires, villes et quartiers", -] - - -KEYS_THEME_NIV1 = { - "Demographie": 0, - "Conditions de vie - Societe": 6, - "Marche du travail - Salaires": 20, - "Economie - Conjoncture - Comptes nationaux": 27, - "Entreprises": 37, - "Secteurs d'activite": 44, - "Territoires, villes et quartiers": 68, - "Developpement durable - Environnement": 74, - "Revenus - Pouvoir d'achat - Consommation": 80, - "Methodes": 86, -} - - -DICT_GEO = { - "COMMUNE": "COM", - "DEPARTEMENT": "DEP", - "REGION": "REG", - "INTERNATIONAL": "INTER", - "INTER REGION": "COMPRD", - "FRANCE": "FRANCE", -} - - -DICT_THEME_CONJ = { - "Industrial production and activity": [ - "Indice de la production industrielle ", - "Enquete mensuelle de conjoncture dans l'industrie", - "Enquete trimestrielle de conjoncture dans l'industrie", - "Chiffre d'affaires dans l'industrie et la construction", - "Indices des commandes en valeur recues dans l'industrie", - "Enquete sur les investissements dans l'industrie", - "Enquete de tresorerie dans l'industrie", - ], - "Construction and building sector": [ - "Enquete mensuelle de conjoncture dans l'industrie du batiment", - "Enquete trimestrielle dans les travaux publics", - "Enquete trimestrielle dans l'artisanat du batiment", - "Construction de locaux", - "Index batiment, travaux publics et divers de la construction", - "Indices des couts de production dans la construction", - "Indice des prix d'entretien-amelioration des batiments", - "Indice du cout de la construction", - ], - "Housing and real estate": [ - "Enquete trimestrielle dans la promotion immobiliere", - "Indice de reference des loyers", - "Indice des loyers commerciaux", - "Indice des loyers des activites tertiaires", - "Indices des loyers d'habitation", - "Indice des prix des logements neufs et anciens", - "Indices des prix des logements anciens", - "Commercialisation de logements neufs - Ventes aux particuliers et ventes aux institutionnels", - ], - "Retail, wholesale and services": [ - "Enquete mensuelle de conjoncture dans le commerce de detail et le commerce et la reparation automobiles", - "Enquete mensuelle de conjoncture dans les services", - "Enquete bimestrielle de conjoncture dans le commerce de gros", - "Volume des ventes dans le commerce de detail et les services personnels ", - "Volume des ventes dans le commerce", - "Chiffre d'affaires dans le commerce de gros et divers services aux entreprises", - "Indice de production dans les services", - "Chiffre d'affaires des grandes surfaces alimentaires (parution arretee aux resultats de decembre 2022)", - ], - "Business demographics and confidence": [ - "Creations d'entreprises", - "Defaillances d'entreprises (parution arretee aux resultats de juillet 2012)", - "Climat des affaires", - "Notes et Points de conjoncture nationaux", - "Conjoncture regionale", - ], - "Employment, unemployment and labour market": [ - "Estimation flash de l'emploi salarie", - "Emploi salarie", - "Emploi et taux de chomage localises (par region et departement)", - "Emploi salarie, salaires de base et duree du travail (resultats definitifs)", - "Emploi salarie, salaires de base et duree du travail (resultats provisoires)", - "Chomage au sens du BIT et indicateurs sur le marche du travail (resultats de l'enquete Emploi)", - "Les inscrits a France Travail", - ], - "Wages and labour costs": [ - "Indice du cout horaire du travail revise - Tous salaries (ICHT, ICHTrev-TS) - Publication arretee depuis le 06/10/2023", - "Indice du cout du travail (ICT) - Resultats detailles", - "Indice du cout du travail (ICT) - Estimation flash", - "Salaires de base - Comparaison France-Allemagne", - ], - "Public sector employment and pay": [ - "L'emploi dans la fonction publique", - "Indice de traitement brut dans la fonction publique d'Etat - grille indiciaire", - "Les salaires dans la fonction publique", - ], - "Households, consumption and health": [ - "Consommation de soins et biens medicaux (CSBM)", - "Prestations et ressources de protection sociale", - "Depenses de consommation des menages en biens", - "Enquete mensuelle de conjoncture aupres des menages ", - ], - "Inflation and producer prices": [ - "Prix a la consommation - moyennes annuelles", - "Indice des prix a la consommation - resultats definitifs", - "Indice des prix a la consommation - resultats provisoires", - "Indices de prix de production et d'importation de l'industrie", - "Indices des prix de production des services ", - "Indices des prix agricoles", - "Prix des energies et des matieres premieres importees", - "Indice des prix dans la grande distribution (parution arretee aux resultats de decembre 2025)", - ], - "National accounts and public finance": [ - "Comptes nationaux trimestriels - premiere estimation", - "Comptes nationaux trimestriels - deuxieme estimation", - "Comptes nationaux trimestriels - resultats detailles", - "Comptes nationaux annuels - revision des principaux agregats", - "Comptes nationaux des administrations publiques - premiers resultats", - "Situation mensuelle budgetaire de l'Etat", - "Dette trimestrielle de Maastricht des administrations publiques", - "Recettes fiscales de l'Etat", - ], - "Transport and tourism": [ - "Immatriculations de vehicules neufs", - "Frequentation touristique dans les hotels, campings et autres hebergements collectifs touristiques", - ], - "Business financing": [ - "Enquete annuelle credit-bail", - ], -} - - -THEME_CONJ = Literal[ - "Industrial production and activity", - "Construction and building sector", - "Housing and real estate", - "Retail, wholesale and services", - "Business demographics and confidence", - "Employment, unemployment and labour market", - "Wages and labour costs", - "Public sector employment and pay", - "Households, consumption and health", - "Inflation and producer prices", - "National accounts and public finance", - "Transport and tourism", - "Business financing", -] - -# --- Extras ----------------------------------------------------------------- - -SEND_FEEDBACK = { - "tool_name": "send_feedback", - "tool_description": ( - "Submit structured feedback about the MCP tools, server behavior, or user experience. " - "This tool appends a timestamped Markdown entry to the feedback log for administrator review.\n" - "\n" - "WHEN TO USE\n" - "- The user reports a bug, error, or unexpected behavior in any tool.\n" - "- The user suggests an improvement, new feature, or enhancement.\n" - "- The assistant encounters an issue during tool execution that should be logged.\n" - "- After completing a complex workflow where feedback on tool quality would be valuable.\n" - "\n" - "WHEN NOT TO USE\n" - "- For transient debugging or one-off troubleshooting (use terminal/logs instead).\n" - "- For questions about tool usage (ask the user or consult documentation).\n" - "\n" - "INPUT\n" - "- `username` -- identifier for the feedback author (e.g., user name, role, or session ID).\n" - "- `feedback` -- clear, actionable Markdown describing the issue or suggestion. " - "Include context (which tool, what happened), expected vs actual behavior, and " - "proposed solutions if applicable. Write as if filing a GitHub issue.\n" - "\n" - "OUTPUT\n" - "Confirmation message with the timestamp and path where feedback was recorded.\n" - "\n" - "EXAMPLES\n" - "✅ User: 'The search_melodi_datasets tool returned no results for \"prix du pain\" even though " - "the dataset exists.' → Log this as a bug report.\n" - "✅ User: 'It would be helpful if RMES_list_graphs could filter by triple count range.' → " - "Log this as a feature request.\n" - "✅ Assistant: 'During execution of get_insee_document, the markdown parser failed on nested " - "tables. This should be fixed.' → Log this as a technical issue.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - - -# --- Dict homepage ----------------------------------------------------------------- - -DICT_KV = [ - {"cle": "clé", "alias": "alias", "valeur": "valeur"}, - {"cle": "estimation de population France", "alias": "", "valeur": "Au 1er janvier 2026, la population résidant en France est estimée à 69,1 millions d'habitants."}, - {"cle": "population légale France", "alias": "", "valeur": "Au 1er janvier 2023, la population de la France hors Mayotte s'établit officiellement à 68 094 000 habitants."}, - {"cle": "immigrés France", "alias": "", "valeur": "En 2025, 8,0 millions d'immigrés vivent en France, soit 11,6 % de la population totale."}, - {"cle": "population étrangère France", "alias": "", "valeur": "En 2025, la population étrangère vivant en France s'élève à 6,3 millions de personnes, soit 9,1 % de la population totale."}, - {"cle": "naissances France", "alias": "", "valeur": "En 2025, le nombre de naissances en France est estimé à 645 000, soit une baisse de -2,1 % par rapport à 2024."}, - {"cle": "indicateur conjoncturel de fécondité", "alias": "", "valeur": "En 2025, l'indicateur conjoncturel de fécondité (ICF) continue de diminuer. Il s'établit à 1,56 enfant par femme (1,53 en France métropolitaine), après 1,61 en 2024 (1,58 en France métropolitaine)."}, - {"cle": "décès France", "alias": "", "valeur": "En 2025, le nombre de décès en France est estimé à 651 000, en hausse de 1,5 % par rapport à 2024, après +0,3 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile)."}, - {"cle": "espérance de vie France", "alias": "", "valeur": "En 2025, l'espérance de vie à la naissance s'élève à 85,9 ans pour les femmes et à 80,3 ans pour les hommes. Elle augmente en 2025, de +0,1 an pour les femmes comme pour les hommes, pour atteindre un niveau historiquement élevé."}, - {"cle": "mariages France", "alias": "", "valeur": "En 2025, le nombre de mariages célébrés en France est estimé à 251 000, dont 244 000 entre personnes de sexe différent et 7 000 entre personnes de même sexe. Le nombre de mariages augmente de 1,4 % par rapport à 2024, après +2,7 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile), alors que la tendance était plutôt à la baisse avant la crise sanitaire."}, - {"cle": "ménages France", "alias": "", "valeur": "En 2023, la France hors Mayotte compte 31,3 millions de ménages."}, - {"cle": "divorces France", "alias": "", "valeur": "128 043 divorces en 2016. Note : jusqu'en 2016, les divorces étaient des décisions de justice prononcées par un juge ; depuis 2017, les divorces par consentement mutuel passent par un acte notarié et ne sont plus comptabilisés de la même façon."}, - {"cle": "inflation", "alias": "Indice des prix à la consommation – IPC ", "valeur": "En juin 2026, les prix à la consommation (IPC) augmentent de 1,8 % sur un an. Sur un mois, l’indice des prix à la consommation diminue de 0,3 %."}, - {"cle": "Chômage BIT ", "alias": "", "valeur": "Au premier trimestre 2026, le taux de chômage en France (hors Mayotte) augmente de 0,2 point et atteint 8,1 % . Le nombre de chômeurs est de 2,6 millions de personnes."}, - {"cle": "emploi BIT", "alias": "", "valeur": "En moyenne sur l'année 2025, parmi les personnes âgées de 15 à 64 ans vivant en France, 69,3 % sont en emploi au sens du Bureau international du travail (BIT)."}, - {"cle": "PIB trimestriel", "alias": "croissance trimestrielle", "valeur": "Au premier trimestre 2026, le produit intérieur brut (PIB) en volume se replie légèrement (-0,1 %)."}, - {"cle": "PIB annuel", "alias": "croissance annuelle", "valeur": "En 2025, le PIB croît de 0,8 % en volume aux prix de l'année précédente."}, - {"cle": "Dépenses de consommation des ménages en biens", "alias": "", "valeur": "En mai 2026, les dépenses de consommation des ménages en biens rebondissent sur un mois (+0,5 % en volume après -0,5 % en avril). Les volumes sont mesurés aux prix de l'année précédente chaînés (en milliards d'euros 2020) et corrigés des variations saisonnières et des effets des jours ouvrables (CVS-CJO)."}, - {"cle": "Climat des affaires", "alias": "", "valeur": "En juin 2026, l'indicateur synthétique du climat des affaires, calculé à partir des réponses des chefs d'entreprise des principaux secteurs d'activité marchands rebondit très légèrement, à 94, en deçà de son niveau moyen."}, - {"cle": "climat de l'emploi", "alias": "", "valeur": "En juin 2026, l'indicateur du climat de l'emploi perd de nouveau trois points (après arrondi) et s'établit à 89, son niveau le plus bas depuis juin 2013 (hors crise sanitaire)."}, - {"cle": "production manufacturière", "alias": "Indice de la production industrielle - IPI", "valeur": "En mai 2026, après deux mois de hausse, la production se replie nettement dans l'industrie manufacturière (-1,0 % après +0,6 % en avril 2026). Dans l'ensemble de l'industrie, elle se replie aussi mais plus légèrement (-0,1 % après +0,3 %)."}, - {"cle": "niveau de vie", "alias": "", "valeur": "En 2024, en France métropolitaine, le niveau de vie médian de la population s'élève à 26 740 euros annuels. Il correspond à un revenu disponible de 2 228 euros par mois pour une personne seule."}, - {"cle": "pouvoir d’achat", "alias": "", "valeur": "En 2025, le pouvoir d’achat du revenu disponible (RDB) des ménages se replie de 0,4 % après une hausse de 2,7 % en 2024. Ramené au niveau individuel et en tenant compte de l’évolution de la taille des ménages, le pouvoir d’achat baisse de 0,7 % après une hausse de 2,2 % en 2024"}, - {"cle": "balance commerciale", "alias": "", "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l’activité en 2025, à hauteur de -0,2 point de PIB, après l’avoir fortement soutenue en 2023 et 2024. "}, - {"cle": "pauvreté monétaire", "alias": "", "valeur": "En 2024, 9,8 millions de personnes vivent avec un niveau de vie inférieur au seuil de pauvreté monétaire, soit 15,4 % de la population vivant dans un logement ordinaire en France métropolitaine."}, - {"cle": "patrimoine", "alias": "", "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine. "}, - {"cle": "état santé", "alias": "", "valeur": "En 2024, deux tiers des personnes âgées de 16 ans ou plus se déclarent en bonne ou très bonne santé. À l'opposé, près de 10 % jugent leur état de santé mauvais voire très mauvais."}, - {"cle": "prestation handicap", "alias": "", "valeur": "Selon leur âge et leur situation, les personnes en situation de handicap ou de perte d'autonomie peuvent prétendre à différentes prestations. Fin 2023, 44 000 personnes ont un droit ouvert à l'allocation compensatrice pour tierce personne (ACTP) et 407 000 à la prestation de compensation du handicap (PCH). Par ailleurs, 1,4 million de personnes de 60 ans ou plus ont perçu l'allocation personnalisée d'autonomie (APA) au titre du mois de décembre 2023."}, - {"cle": "dépenses liées à la culture", "alias": "", "valeur": "En 2025, les dépenses liées à la culture, au sport et aux loisirs s'élèvent à 108 milliards d'euros. Les services récréatifs, sportifs et culturels rassemblent 45 % de ces dépenses."}, - {"cle": "Parc de logements", "alias": "", "valeur": "Au 1er janvier 2025, la France hors Mayotte compte 38,4 millions de logements. 82,5 % des logements sont des résidences principales et 54,4 % des logements individuels (maisons)."}, - {"cle": "logements vacants", "alias": "", "valeur": "Après avoir fortement augmenté entre 2005 et 2019, la part des logements vacants diminue, passant de 8,1 % en 2019 à 7,7 % en 2025 ; en 2025, 3,0 millions de logements sont vacants."}, - {"cle": "résidences secondaires ou logements occasionnels", "alias": "", "valeur": "Au 1er janvier 2025, 3,8 millions de logements sont des résidences secondaires ou des logements occasionnels ; après avoir augmenté entre 2011 et 2017, leur part dans l'ensemble du parc est stable."}, - {"cle": "ménages sont propriétaires de leur résidence principale", "alias": "", "valeur": "Au 1er janvier 2025, 57,4 % des ménages sont propriétaires de leur résidence principale."}, - {"cle": "smic", "alias": "Salaire minimum interprofessionnel de croissance", "valeur": "Depuis le 1er janvier 2026, le Smic brut s'élève à 12,02 euros par heure, soit 1 823,03 euros par mois pour 151,67 heures de travail."}, - {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur privé", "alias": "", "valeur": "En 2023, le salaire mensuel moyen en équivalent temps plein (EQTP) dans le secteur privé est de 2 730 euros, nets de cotisations et contributions sociales."}, - {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur public", "alias": "", "valeur": "Dans la fonction publique, tous statuts confondus, un salarié gagne en moyenne 2 650 euros nets par mois en EQTP en 2023."}, - {"cle": "revenus non salariés", "alias": "", "valeur": "En 2023, hors agriculture, les non-salariés classiques (micro-entrepreneurs exclus) retirent en moyenne 4 040 euros par mois de leur activité non salariée. Cette moyenne recouvre de fortes disparités selon la nature des emplois."}, - {"cle": "salaires horaires", "alias": "", "valeur": "Au premier trimestre 2026, les salaires horaires augmentent de 0,3 % sur le trimestre et de 2,0 % sur un an"}, - {"cle": "coût horaire du travail", "alias": "Indice du coût du travail – ICT", "valeur": "Au premier trimestre 2026, le coût horaire du travail (salaires, cotisations et taxes, déduction faite des exonérations et subventions) de l'ensemble du secteur marchand non agricole (hors services aux ménages) freine significativement, dans le sillage des salaires : +0,5 % sur le trimestre et + 2,3 % sur un an."}, - {"cle": "création entreprises", "alias": "", "valeur": "En 2025, 1 165 800 entreprises ont été créées en France, dont 758 500 sous forme d'entrepreneurs individuels ayant adopté le régime de la microentreprise (micro-entrepreneurs)."}, - {"cle": "défaillances d'entreprises", "alias": "", "valeur": "En 2025, 68 872 unités légales ont été en situation de défaillance."}, - {"cle": "entreprises marchandes non agricoles et non financières en France", "alias": "", "valeur": "En 2023, en France, les secteurs marchands non agricoles et non financiers (incluant toutefois les exploitations forestières, les auxiliaires de services financiers et d'assurance et les holdings) comptent 5,2 millions d'entreprises. Ces entreprises emploient 15,9 millions de salariés en équivalent temps plein (EQTP)."}, - {"cle": "exploitations agricoles", "alias": "", "valeur": "Dans le secteur agricole, l'usage est de compter plutôt des exploitations agricoles ; en 2023, la France métropolitaine en compte 349 600 et la main d'œuvre agricole s'élève à 663 200 EQTP."}, - {"cle": "commerce", "alias": "", "valeur": "En 2023, le commerce rassemble 739 128 entreprises. Elles réalisent un chiffre d'affaires de 1 728 milliards d'euros et dégagent une valeur ajoutée (VA) de 272 milliards d'euros. Fin 2024, 3,4 millions de personnes occupent un emploi salarié dans le commerce."}, - {"cle": "industrie", "alias": "", "valeur": "En 2023, l'industrie rassemble 322 386 entreprises. Elles réalisent un chiffre d'affaire de 1 544 milliards d'euros et dégagent une valeur ajoutée (VA) de 368 milliards d'euros. Fin 2024, 3,3 millions de personnes occupent un emploi salarié dans l'industrie."}, - {"cle": "construction", "alias": "", "valeur": "En 2023, la construction rassemble 587 898 entreprises. Elles réalisent un chiffre d'affaires de 405 milliards d'euros et dégagent une valeur ajoutée (VA) de 128 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans la construction."}, - {"cle": "services", "alias": "", "valeur": "En 2023, les services principalement marchands non financiers comptent plus de 2,3 millions d'entreprises. Ces entreprises réalisent un chiffre d'affaires de 995 milliards d'euros et dégagent une valeur ajoutée (VA) de 475 milliards d'euros. Fin 2024, 7,5 millions de personnes (y compris les intérimaires) occupent un emploi salarié dans les services principalement marchands non financiers."}, - {"cle": "transports", "alias": "", "valeur": "En 2023, les transports et l'entreposage rassemblent 193 101 entreprises. Elles réalisent un chiffre d'affaires de 267 milliards d'euros et dégagent une valeur ajoutée (VA) de 102 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans les transports et l'entreposage."}, - {"cle": "entreprises de l'économie sociale", "alias": "", "valeur": "Les entreprises de l'économie sociale se caractérisent par leur famille de l'économie sociale, à la fois privé et à caractère essentiellement non lucratif. En 2022, elles représentent 9,8 % de l'emploi salarié total en équivalent temps plein. Les associations emploient 73 % de ce volume de travail salarié ; 14 % est employé par les coopératives, 6 % par les mutuelles, 5 % par les fondations et 3 % par les autres organismes privés à but non-lucratif."}, - {"cle": "Population quartiers prioritaires de la politique de la ville", "alias": "QPV", "valeur": "Les quartiers prioritaires de la politique de la ville (QPV) tels que définis par le décret n° 2015-1138 du 14 septembre 2015 regroupent 7,9 % de la population en 2020."}, - {"cle": "Population unités urbaines", "alias": "", "valeur": "Les unités urbaines rassemblent toujours plus d'habitants. En 2022, en France métropolitaine, elles représentent 78,8 % de la population, soit 51,9 millions d'habitants. À l'exception de l'unité urbaine de Paris qui concentre près de 11 millions d'habitants, les 10 plus grandes unités urbaines françaises comptent chacune entre 0,5 et 2 millions d'habitants."}, - {"cle": "mode déplacement domicile travail", "alias": "", "valeur": "Pour se rendre au travail, les personnes en emploi se déplacent majoritairement en voiture ou en deux-roues motorisés (71 % en 2022). 15 % des personnes en emploi empruntent les transports en commun."}, - {"cle": "dépense nationale protection de l'environnement", "alias": "", "valeur": "En 2022, la dépense nationale en faveur de la protection de l'environnement s'élève à 63,7 milliards d'euros (Md€). Elle est dédiée à la protection de l'air, de la biodiversité et des paysages, la collecte et traitement des déchets, la protection et dépollution des sols et des eaux, la lutte contre le bruit et d'autres activités de protection de l'environnement (frais de fonctionnement de l'administration publique et des opérateurs chargés des questions environnementales notamment). Les entreprises sont les principaux financeurs des dépenses de protection de l'environnement (22,6 Md€, soit 35 %), devant les administrations publiques (État et ses ministères, collectivités locales, organismes publics) (22,2 Md€, soit 35 %) et les ménages (18,1 Md€, soit 28 %)."}, - {"cle": "indice de référence des loyers", "alias": "IRL", "valeur": "Au deuxième trimestre 2026, l'indice de référence des loyers s'établit à 148,37. Sur un an, il augmente de 1,15 % après +0,78 % au trimestre précédent."}, - {"cle": "indice des loyers commerciaux", "alias": "ILC", "valeur": "Au premier trimestre 2026, l'indice des loyers commerciaux s'établit à 135,26. Sur un an, il baisse de 0,45 % (après -0,50 % au trimestre précédent)."}, - {"cle": "indice des loyers des activités tertiaires", "alias": "ILAT", "valeur": "Au premier trimestre 2026, l'indice des loyers des activités tertiaires s'établit à 137,42. Sur un an, il augmente de 0,09 % (après -0,06 % au trimestre précédent)."}, - {"cle": "indice du coût de la construction", "alias": "ICC", "valeur": "L'indice du coût de la construction (ICC) s'établit à 2 084 au premier trimestre 2026. Il est en hausse de 1,26 % sur un trimestre (après +0,10 % au trimestre précédent). Sur un an, il baisse de 2,89 % (après -2,37 % au trimestre précédent)."}, - {"cle": "index du bâtiment tous corps d'état", "alias": "BT01 ; index bâtiment BT01", "valeur": "En mai 2026, l'index Bâtiment BT01 « Tous corps d'état » s'établit à 137,9, en référence 100 en 2010."}, - {"cle": "index général des travaux publics", "alias": "TP01 ; index travaux publics TP01", "valeur": "En mai 2026, l’index Travaux publics TP01 « Index général tous travaux » s’établit à 140,4, en référence 100 en 2010."}, - {"cle": "index ingénierie", "alias": "ING ; indice ING", "valeur": "En mai 2026, l’index divers de la construction ING « Ingénierie » s’établit à 138,3, en référence 100 en 2010."}, -] diff --git a/src/mcpdiffusion/tools/extras_send_feedback.py b/src/mcpdiffusion/tools/extras_send_feedback.py index 4af43e2..cc5d18c 100644 --- a/src/mcpdiffusion/tools/extras_send_feedback.py +++ b/src/mcpdiffusion/tools/extras_send_feedback.py @@ -1,103 +1,20 @@ -"""Tool: send_feedback - -Submit structured feedback about the MCP tools, server behavior, or user experience. -Feedback is appended to a timestamped Markdown file for administrator review. -""" +"""Tool: send_feedback -- thin registration layer.""" from __future__ import annotations -from datetime import datetime -from pathlib import Path - from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from .env import SEND_FEEDBACK - - -# --------------------------------------------------------------------------- -# Schémas Pydantic -- send_feedback -# --------------------------------------------------------------------------- - -class SendFeedbackInput(BaseModel): - username: str = Field( - description="Identifier for the feedback author (e.g., user name, role, or session ID).", - examples=["alice", "data_analyst", "session_abc123"], - ) - feedback: str = Field( - description=( - "Clear, actionable Markdown describing the issue or suggestion. Include context " - "(which tool, what happened), expected vs actual behavior, and proposed solutions " - "if applicable. Write as if filing a GitHub issue." - ), - examples=[ - "## Bug Report\n\n**Tool:** search_melodi_datasets\n\n**Issue:** No results returned " - "for 'prix du pain' even though dataset DS_PRIX exists.\n\n**Expected:** Should find " - "at least one matching dataset.\n\n**Proposed fix:** Check if the Elasticsearch index " - "includes this dataset.", - ], - ) - - -class SendFeedbackOutput(BaseModel): - status: str = "success" - message: str - timestamp: str - #path: str +from ..config.tool_metadata import SEND_FEEDBACK +from ..core.logging import log_tool +from ..models.feedback import SendFeedbackInput, SendFeedbackOutput +from ..services.feedback import send_feedback -# --------------------------------------------------------------------------- -# Path resolution -# --------------------------------------------------------------------------- - -# Resolve feedback file path relative to this module's location, not CWD. -# Structure: mcpdiffusion/tools/extras_send_feedback.py -> mcpdiffusion/feedback/feedback.md -_FEEDBACK_DIR = Path(__file__).resolve().parent.parent / "feedback" -_FEEDBACK_FILE = _FEEDBACK_DIR / "feedback.md" - - -def _ensure_feedback_file() -> Path: - """Create feedback directory and seed file if they don't exist.""" - _FEEDBACK_DIR.mkdir(parents=True, exist_ok=True) - if not _FEEDBACK_FILE.exists(): - _FEEDBACK_FILE.write_text( - "# Feedback Log\n\n" - "This file collects feedback from users and the assistant about MCP tools, " - "server behavior, and suggestions for improvement. Each entry is timestamped " - "and formatted as Markdown for easy review.\n\n---\n\n", - encoding="utf-8", - ) - return _FEEDBACK_FILE - - -# --------------------------------------------------------------------------- -# Enregistrement du tool MCP -# --------------------------------------------------------------------------- def register_extras_send_feedback(mcp: FastMCP) -> None: - @mcp.tool( name=SEND_FEEDBACK["tool_name"], description=SEND_FEEDBACK["tool_description"], meta=SEND_FEEDBACK["tool_metadata"], ) @log_tool - async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: - feedback_path = _ensure_feedback_file() - timestamp = datetime.now().isoformat(timespec="seconds") - - # Format: ## heading with timestamp and username, then feedback body, then separator - entry = ( - f"## {timestamp} — {params.username}\n\n" - f"{params.feedback}\n\n" - "---\n\n" - ) - - with feedback_path.open("a", encoding="utf-8") as f: - f.write(entry) - - return SendFeedbackOutput( - message=f"Feedback recorded successfully.", - timestamp=timestamp, - #path=str(feedback_path), - ) + async def send_feedback_tool(params: SendFeedbackInput) -> SendFeedbackOutput: + return await send_feedback(params) diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index f4c2c16..82092f7 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -1,215 +1,12 @@ -"""Tool: get_insee_document - -Fetch a single INSEE publication from its URL and return the full text -as markdown, optionally with the parsed sommaire (table of contents). -""" +"""Tool: get_insee_document -- thin registration layer.""" from __future__ import annotations -import os -from collections import defaultdict -from typing import Any, Optional -from urllib.parse import urljoin, urlparse - -import httpx -from bs4 import BeautifulSoup, Tag from fastmcp import FastMCP -from pydantic import BaseModel, Field -from trafilatura import extract -from trafilatura.settings import Extractor - -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import GET_DOCUMENT - - -BASE_URL = "https://www.insee.fr" -_USER_AGENT = ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" -) -_DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) - -_TRAFILATURA_OPTIONS = Extractor( - output_format="markdown", - links=True, - formatting=True, - source="insee.fr", - with_metadata=True, -) - -# When truncate_content=True, the markdown body is clipped to this size. -# Head + tail are kept so the model sees the leading context (key figures, -# abstract) AND the trailing context (methodology, references). -_MAX_MARKDOWN_CHARS = 30_000 - - -def _tls_verify() -> bool: - return os.getenv("TLS_VERIFY", "true").strip().lower() != "false" - - -class GetInseeDocumentInput(BaseModel): - list_of_url: list[str] = Field( - description=( - "List of relative URLs to retrieve (e.g. " - "'/fr/statistiques/4277658?sommaire=4318291')." - ), - examples=[["/fr/statistiques/4277658?sommaire=4318291"]], - ) - include_sommaire: bool = Field( - default=True, - description=( - "If True, parse the page's table-of-contents section alongside " - "the main content. Use once to discover structure, then False " - "for subsequent requests on the same page." - ), - ) - truncate_content: bool = Field( - default=True, - description=( - "If True (default), long markdown bodies are clipped to keep the " - "response compact for the model. Set to False only when the full " - "text is required." - ), - ) - - -class DocumentResult(BaseModel): - """Uniform per-URL result: same keys whether the fetch succeeded or failed.""" - id: str = Field(description="The input URL that produced this entry.") - status: str = Field(description="'success' or 'error'.") - markdown_content: Optional[str] = None - sommaire: Optional[dict[str, dict[str, str]]] = Field( - default=None, - description=( - "Parsed table of contents as " - "{category: {title: url}}. None when include_sommaire=False " - "or when the page has no sommaire." - ), - ) - truncated: bool = Field( - default=False, - description="True if markdown_content was clipped due to size.", - ) - error: Optional[str] = Field( - default=None, - description="Human-readable error message when status == 'error'.", - ) - -class GetInseeDocumentOutput(BaseModel): - results: list[DocumentResult] - count: int - - -def _as_relative(url: str) -> str: - p = urlparse(url) - return f"{p.path}?{p.query}" if p.query else p.path - - -def _parse_sommaire(html: str, base_url: str = BASE_URL) -> list[dict[str, str]]: - """Extract entries from the 'Sommaire' block, tolerant of both - multi-category and flat layouts.""" - soup = BeautifulSoup(html, "lxml") - results: list[dict[str, str]] = [] - - sommaire_section = soup.find( - lambda t: t.has_attr("class") and any("sommaire" in c for c in t["class"]) - ) - if not sommaire_section: - return [] - - outer_ul = sommaire_section.find("ul", class_="sommaire") - if not outer_ul: - return [] - - for top_li in outer_ul.find_all("li", recursive=False): - heading_tag = top_li.find("h2") - if heading_tag: - category_name = heading_tag.get_text(strip=True) - inner_ul = top_li.find("ul", class_="sommaire") - if not inner_ul: - continue - for link_li in inner_ul.find_all("li", class_="lien-produit"): - a = link_li.find("a") - if not a: - continue - title = a.get_text(strip=True) - absolute = urljoin(base_url, a.get("href", "")) - rel_url = _as_relative(absolute) - results.append( - {"category": category_name, "title": title, "url": rel_url} - ) - else: - a = top_li.find("a") - if not a: - continue - title = a.get_text(strip=True) - absolute = urljoin(base_url, a.get("href", "")) - rel_url = _as_relative(absolute) - results.append({"category": "", "title": title, "url": rel_url}) - return results - - -def _format_sommaire(flat_items: list[dict[str, str]]) -> dict[str, dict[str, str]]: - grouped: dict[str, dict[str, str]] = defaultdict(dict) - for entry in flat_items: - grouped[entry["category"]][entry["title"]] = entry["url"] - return dict(grouped) - - -def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: - """Return (text, truncated_flag). Keeps head + tail when clipping.""" - if len(text) <= limit: - return text, False - head_size = (limit * 2) // 3 - tail_size = limit - head_size - 200 - marker = ( - "\n\n\n\n" - ) - return text[:head_size] + marker + text[-tail_size:], True - - -async def _fetch_html(url: str) -> str: - full_url = BASE_URL + url if not url.startswith(("http://", "https://")) else url - try: - async with httpx.AsyncClient( - timeout=_DEFAULT_TIMEOUT, - verify=_tls_verify(), - headers={"User-Agent": _USER_AGENT}, - follow_redirects=True, - ) as client: - response = await client.get(full_url) - response.raise_for_status() - return response.text - except httpx.TimeoutException as exc: - fail( - "BACKEND_UNAVAILABLE", - f"insee.fr timed out fetching {full_url}: {exc}", - retryable=True, - ) - raise - except httpx.HTTPStatusError as exc: - if exc.response.status_code == 404: - fail( - "NOT_FOUND", - f"INSEE document not found at {full_url} (HTTP 404). " - "Verify the URL with `search_insee_documents`.", - ) - else: - fail( - "UPSTREAM_ERROR", - f"insee.fr returned HTTP {exc.response.status_code} for {full_url}.", - retryable=(500 <= exc.response.status_code < 600), - ) - raise - except httpx.HTTPError as exc: - fail( - "BACKEND_UNAVAILABLE", - f"Network error fetching {full_url}: {exc}", - retryable=True, - ) - raise +from ..config.tool_metadata import GET_DOCUMENT +from ..core.logging import log_tool +from ..models.insee import GetInseeDocumentInput, GetInseeDocumentOutput +from ..services.insee_document import get_insee_document def register_get_insee_document(mcp: FastMCP) -> None: @@ -222,50 +19,4 @@ def register_get_insee_document(mcp: FastMCP) -> None: async def get_insee_documents( params: GetInseeDocumentInput, ) -> GetInseeDocumentOutput: - if not params.list_of_url: - fail( - "INVALID_INPUT", - "list_of_url must contain at least one URL. " - "Use `search_insee_documents` to find URLs first.", - ) - - results: list[DocumentResult] = [] - for url in params.list_of_url: - try: - html = await _fetch_html(str(url)) - markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" - if params.truncate_content: - markdown, truncated = _truncate(markdown) - else: - truncated = False - - sommaire: Optional[dict[str, dict[str, str]]] = None - if params.include_sommaire: - flat = _parse_sommaire(html) - sommaire = _format_sommaire(flat) if flat else None - - results.append( - DocumentResult( - id=str(url), - status="success", - markdown_content=markdown, - sommaire=sommaire, - truncated=truncated, - error=None, - ) - ) - except Exception as exc: - # Failures per URL don't abort the batch -- callers need - # every result to know which URLs worked and which didn't. - results.append( - DocumentResult( - id=str(url), - status="error", - markdown_content=None, - sommaire=None, - truncated=False, - error=f"{type(exc).__name__}: {str(exc)[:500]}", - ) - ) - - return GetInseeDocumentOutput(results=results, count=len(results)) + return await get_insee_document(params) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index c222d16..7989333 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -1,38 +1,12 @@ -"""Tool: get_insee_homepage - -Return the curated set of INSEE key indicators (``DICT_KV`` from -``tools.env``) instead of scraping the INSEE homepage. -""" +"""Tool: get_insee_homepage -- thin registration layer.""" from __future__ import annotations from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from .env import DICT_KV, GET_HOMEPAGE - - -class KeyValueIndicator(BaseModel): - """A single INSEE key indicator with its pre-computed textual value.""" - - key: str = Field(description="Indicator name (e.g. 'smic', 'PIB annuel').") - alias: str = Field( - default="", - description="Optional alias / alternative name for the indicator.", - ) - value: str = Field( - description="Pre-computed textual description of the latest figure." - ) - -class KeyIndicatorsOutput(BaseModel): - """The curated list of INSEE key indicators (replaces the homepage - scraping output).""" - - indicators: list[KeyValueIndicator] = Field( - description="Curated key indicators: name, alias and latest value.", - ) - count: int = Field(description="Number of indicators returned.") +from ..config.tool_metadata import GET_HOMEPAGE +from ..core.logging import log_tool +from ..data.indicators import DICT_KV +from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator def register_get_insee_homepage(mcp: FastMCP) -> None: @@ -50,7 +24,6 @@ async def get_insee_homepage() -> KeyIndicatorsOutput: value=entry["valeur"].strip(), ) for entry in DICT_KV - # Skip the placeholder/header row shipped in DICT_KV. if not ( entry["cle"].strip() == "clé" and entry["alias"].strip() == "alias" diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index 4e720ab..a0b628f 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -1,89 +1,22 @@ -"""Tool: search_insee_documents - -Full-text search of INSEE publications (Insee Premiere, Insee Analyses, -Dossiers, References, Chiffres-cles, ...) backed by the produit index. -""" +"""Tool: search_insee_chiffrecle -- thin registration layer.""" from __future__ import annotations -from enum import StrEnum -from typing import Optional - from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError from fastmcp import FastMCP -from pydantic import BaseModel, Field -from ..helpers.es_search import ( - DocumentHit, +from ..config.tool_metadata import SEARCH_CHIFFRECLEF +from ..core.errors import fail +from ..core.logging import log_tool +from ..models.insee import ( + SearchInseeChiffrecleInput, + SearchInseeChiffrecleOutput, +) +from ..services.insee_search import ( apply_collection_filters, build_text_clauses, execute_search, ) -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import SEARCH_CHIFFRECLEF - - -class _INSEETheme(StrEnum): - ALL = "ALL" - METHODES = "Methodes" - DEMOGRAPHIE = "Demographie" - REVENUS = "Revenus - Pouvoir d'achat - Consommation" - CONDITIONS = "Conditions de vie - Societe" - TRAVAIL = "Marche du travail - Salaires" - ECONOMIE = "Economie - Conjoncture - Comptes nationaux" - DD = "Developpement durable - Environnement" - ENTREPRISES = "Entreprises" - SECTEURS = "Secteurs d'activite" - TERRITOIRES = "Territoires, villes et quartiers" - - -class _INSEEGeo(StrEnum): - COM = "COM" - DEP = "DEP" - REG = "REG" - INTER = "INTER" - COMPRD = "COMPRD" - FRANCE = "FRANCE" - - -class SearchInseeChiffrecleInput(BaseModel): - query: str = Field( - description="Natural-language search query describing the statistics to retrieve.", - examples=["population de Lyon", "taux de chomage 2024", "PIB France"], - ) - - year_of_reference: Optional[int] = Field( - default=None, - description=( - "Hard filter on publication year (e.g. 2024). Leave null to " - "search all years." - ), - ) - - geo_niveau: _INSEEGeo = Field( - default=_INSEEGeo.FRANCE, - description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", - ) - geo_keyword: Optional[str] = Field( - default=None, - description=( - "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " - "'Bouches-du-Rhone'). Leave null to skip geographic filtering." - ), - ) - number_of_results: int = Field( - default=10, - description="Maximum number of results to return.", - ge=1, - le=20, - ) - - -class SearchInseeDocumentsOutput(BaseModel): - results: list[DocumentHit] - count: int - def register_search_insee_chiffreclef(mcp: FastMCP) -> None: @@ -93,9 +26,9 @@ def register_search_insee_chiffreclef(mcp: FastMCP) -> None: meta=SEARCH_CHIFFRECLEF["tool_metadata"], ) @log_tool - async def search_insee_documents( + async def search_insee_chiffrecle( params: SearchInseeChiffrecleInput, - ) -> SearchInseeDocumentsOutput: + ) -> SearchInseeChiffrecleOutput: must, filters, should, must_not = build_text_clauses( query=params.query, year_of_reference=params.year_of_reference, @@ -125,4 +58,4 @@ async def search_insee_documents( retryable=True, ) raise - return SearchInseeDocumentsOutput(results=hits, count=len(hits)) + return SearchInseeChiffrecleOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index d1665f4..1a00c8c 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -1,80 +1,24 @@ -"""Tool: search_insee_conjoncture - -Search INSEE Rapid Releases (Informations rapides) -- short, recurring -publications reporting the latest monthly/quarterly/annual results for -major economic and social indicators. -""" +"""Tool: search_insee_conjoncture -- thin registration layer.""" from __future__ import annotations -from enum import StrEnum -from typing import Optional - from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError +from elasticsearch.dsl import Q from fastmcp import FastMCP -from pydantic import BaseModel, Field -from ..helpers.es_search import ( - DocumentHit, +from ..config.tool_metadata import SEARCH_CONJONCTURE +from ..core.errors import fail +from ..core.logging import log_tool +from ..data.themes import DICT_THEME_CONJ +from ..models.insee import ( + SearchInseeConjonctureInput, + SearchInseeConjonctureOutput, +) +from ..services.insee_search import ( apply_collection_filters, build_text_clauses, execute_search, ) -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import DICT_THEME_CONJ, SEARCH_CONJONCTURE - - -class _ThemeConjoncture(StrEnum): - INDUSTRY = "Industrial production and activity" - BUILDING = "Construction and building sector" - HOUSING = "Housing and real estate" - RETAIL = "Retail, wholesale and services" - BUSINESS = "Business demographics and confidence" - EMPLOYMENT = "Employment, unemployment and labour market" - WAGES = "Wages and labour costs" - PUBLIC_SECTOR = "Public sector employment and pay" - CONSUMPTION = "Households, consumption and health" - PRICES = "Inflation and producer prices" - ACCOUNTING = "National accounts and public finance" - TRANSPORT = "Transport and tourism" - FINANCE = "Business financing" - - -class SearchInseeConjonctureInput(BaseModel): - query: str = Field( - description=( - "Natural-language query. The search is lexical and rewards " - "keyword breadth -- provide several synonyms and related notions." - ), - examples=["consommation", "hotel", "PIB"], - ) - theme_conjoncture: Optional[_ThemeConjoncture] = Field( - default=None, - description=( - "Optional broad category to restrict the search. Each category " - "contains multiple sub-themes. Leave null to search across all." - ), - ) - year_of_reference: Optional[int] = Field( - default=None, - description=( - "Hard filter on publication year (e.g. 2024). Leave null to " - "search all years; for 'latest release' use cases, prefer " - "leaving null so the freshest match wins by score." - ), - ) - number_of_results: int = Field( - default=10, - description="Maximum number of results to return.", - ge=1, - le=20, - ) - - -class SearchInseeConjonctureOutput(BaseModel): - results: list[DocumentHit] - count: int def register_search_insee_conjoncture(mcp: FastMCP) -> None: @@ -96,12 +40,9 @@ async def search_insee_conjoncture( must_not_rapides=False, must_only_rapides=True, ) - # Theme filter applied after the shared collection filters so the - # deux are not conflated with the generic theme (top-level INSEE). if params.theme_conjoncture: subthemes = DICT_THEME_CONJ.get(params.theme_conjoncture) if subthemes: - from elasticsearch.dsl import Q filters.append(Q("terms", conjoncture_libelle=subthemes)) try: diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index 7f1d82c..febbc0c 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -1,94 +1,22 @@ -"""Tool: search_insee_documents - -Full-text search of INSEE publications (Insee Premiere, Insee Analyses, -Dossiers, References, Chiffres-cles, ...) backed by the produit index. -""" +"""Tool: search_insee_documents -- thin registration layer.""" from __future__ import annotations -from enum import StrEnum -from typing import Optional - from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError from fastmcp import FastMCP -from pydantic import BaseModel, Field -from ..helpers.es_search import ( - DocumentHit, +from ..config.tool_metadata import SEARCH_DOCUMENTS +from ..core.errors import fail +from ..core.logging import log_tool +from ..models.insee import ( + SearchInseeDocumentsInput, + SearchInseeDocumentsOutput, +) +from ..services.insee_search import ( apply_collection_filters, build_text_clauses, execute_search, ) -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import SEARCH_DOCUMENTS - - -class _INSEETheme(StrEnum): - ALL = "ALL" - METHODES = "Methodes" - DEMOGRAPHIE = "Demographie" - REVENUS = "Revenus - Pouvoir d'achat - Consommation" - CONDITIONS = "Conditions de vie - Societe" - TRAVAIL = "Marche du travail - Salaires" - ECONOMIE = "Economie - Conjoncture - Comptes nationaux" - DD = "Developpement durable - Environnement" - ENTREPRISES = "Entreprises" - SECTEURS = "Secteurs d'activite" - TERRITOIRES = "Territoires, villes et quartiers" - - -class _INSEEGeo(StrEnum): - COM = "COM" - DEP = "DEP" - REG = "REG" - INTER = "INTER" - COMPRD = "COMPRD" - FRANCE = "FRANCE" - - -class SearchInseeDocumentsInput(BaseModel): - query: str = Field( - description="Natural-language search query describing the statistics to retrieve.", - examples=["population de Lyon", "taux de chomage 2024", "PIB France"], - ) - theme: _INSEETheme = Field( - default=_INSEETheme.ALL, - description="Optional top-level INSEE theme used to restrict the search. Default: ALL.", - ) - year_of_reference: Optional[int] = Field( - default=None, - description=( - "Hard filter on publication year (e.g. 2024). Leave null to " - "search all years." - ), - ) - #chiffre_clef: bool = Field( - # default=False, - # description="If True, restrict to 'Chiffres-cles' (key figures).", - #) - geo_niveau: _INSEEGeo = Field( - default=_INSEEGeo.FRANCE, - description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", - ) - geo_keyword: Optional[str] = Field( - default=None, - description=( - "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " - "'Bouches-du-Rhone'). Leave null to skip geographic filtering." - ), - ) - number_of_results: int = Field( - default=10, - description="Maximum number of results to return.", - ge=1, - le=20, - ) - - -class SearchInseeDocumentsOutput(BaseModel): - results: list[DocumentHit] - count: int def register_search_insee_documents(mcp: FastMCP) -> None: diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index 99650e6..e349ac7 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -1,68 +1,12 @@ -"""Tool: get_melodi_observations - -Retrieve filtered observations from a Melodi dataset. -""" +"""Tool: get_melodi_observations -- thin registration layer.""" from __future__ import annotations -from typing import Any - -import httpx from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import GET_DATASET - - -MELODI_DATA_BASE_URL = "https://api.insee.fr/melodi/data" -_DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) - - -def _tls_verify() -> bool: - import os - return os.getenv("TLS_VERIFY", "true").strip().lower() != "false" - - -class GetMelodiObservationsInput(BaseModel): - dataset_id: str = Field( - description="Identifier of the Melodi dataset (from search_melodi_datasets).", - examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], - ) - list_of_year: list[int] = Field( - default_factory=list, - description=( - "Years to keep in the result set. Leave empty (the default) to " - "return all available years. Pass e.g. [2020, 2021, 2022] to keep " - "only those years." - ), - examples=[[], [2020, 2021, 2022]], - ) - dict_of_columns_and_values: dict[str, str] = Field( - default_factory=dict, - description=( - "Filters based on modality codes of columns. Leave empty to " - "return all rows. Keys are column ids (e.g. 'PRICES', 'GEO'); " - "values are the exact modality codes returned by " - "`search_melodi_modalities`." - ), - examples=[ - {"PRICES": "D"}, - {"PCS": "6", "GEO": "2025-FRANCE-FM"}, - ], - ) - number_of_results: int = Field( - default=100, - description="Maximum number of observations to return.", - ge=1, - le=1000, - ) - -class GetMelodiObservationsOutput(BaseModel): - dataset_id: str - observations: list[dict[str, Any]] - count: int +from ..config.tool_metadata import GET_DATASET +from ..core.logging import log_tool +from ..models.melodi import GetMelodiObservationsInput, GetMelodiObservationsOutput +from ..services.melodi import get_melodi_observations def register_get_melodi_observations(mcp: FastMCP) -> None: @@ -72,92 +16,7 @@ def register_get_melodi_observations(mcp: FastMCP) -> None: meta=GET_DATASET["tool_metadata"], ) @log_tool - async def get_melodi_observations( + async def get_melodi_observations_tool( params: GetMelodiObservationsInput, ) -> GetMelodiObservationsOutput: - url = f"{MELODI_DATA_BASE_URL}/{params.dataset_id}" - try: - async with httpx.AsyncClient( - timeout=_DEFAULT_TIMEOUT, - verify=_tls_verify(), - ) as client: - response = await client.get( - url, - params=params.dict_of_columns_and_values or None, - ) - response.raise_for_status() - except httpx.TimeoutException as exc: - fail( - "BACKEND_UNAVAILABLE", - f"Melodi API timed out after {_DEFAULT_TIMEOUT.read}s " - f"calling {url}: {exc}. Try again or narrow the query.", - retryable=True, - ) - raise - except httpx.HTTPStatusError as exc: - status = exc.response.status_code - body_excerpt = (exc.response.text or "")[:500] - if status == 400: - fail( - "INVALID_INPUT", - f"Melodi API rejected the query (HTTP 400). " - f"Columns/values passed: {params.dict_of_columns_and_values}. " - f"Upstream detail: {body_excerpt}. " - "Verify modality codes with `search_melodi_modalities`.", - ) - elif status == 404: - fail( - "NOT_FOUND", - f"Melodi dataset {params.dataset_id!r} not found (HTTP 404). " - "Check the dataset_id with `search_melodi_datasets`.", - ) - else: - fail( - "UPSTREAM_ERROR", - f"Melodi API returned HTTP {status}: {body_excerpt}", - retryable=(500 <= status < 600), - ) - raise - except httpx.HTTPError as exc: - fail( - "BACKEND_UNAVAILABLE", - f"Could not reach Melodi API at {url}: {exc}", - retryable=True, - ) - raise - - try: - payload = response.json() - except ValueError as exc: - fail( - "PARSE_ERROR", - f"Melodi API returned non-JSON response: {exc}", - ) - raise - - observations = payload.get("observations") if isinstance(payload, dict) else None - if not isinstance(observations, list): - fail( - "PARSE_ERROR", - "Melodi API response did not contain an 'observations' list.", - ) - raise - - # Year filter (post-fetch, since the upstream API doesn't expose a - # dedicated year param -- kept consistent with the previous behavior). - if params.list_of_year: - years_str = {str(y) for y in params.list_of_year} - observations = [ - obs - for obs in observations - if (obs.get("dimensions", {}) - .get("TIME_PERIOD", "") - .split("-")[0]) in years_str - ] - - sliced = observations[: params.number_of_results] - return GetMelodiObservationsOutput( - dataset_id=params.dataset_id, - observations=sliced, - count=len(sliced), - ) + return await get_melodi_observations(params) diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py index f3ed662..edc351a 100644 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ b/src/mcpdiffusion/tools/melodi_search_datasets.py @@ -1,73 +1,12 @@ -"""Tool: search_melodi_datasets - -Search the INSEE Melodi dataset catalogue by French-language query. -""" +"""Tool: search_melodi_datasets -- thin registration layer.""" from __future__ import annotations -from typing import Optional - -from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.es import INDEX_MELODI_DATASETS, get_es_client -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import SEARCH_DATASET - - -class SearchMelodiDatasetsInput(BaseModel): - french_query: str = Field( - description=( - "Explicit French description of the statistical dataset to search. " - "Mention the phenomenon (inflation, births, unemployment), " - "geographic level, population or product if known. " - "Do NOT provide codes." - ), - examples=[ - "indice des prix a la consommation", - "deces par departement", - "prenoms des nouveau-nes", - "population communale", - "salaires des enseignants", - ], - ) - start_year: int = Field( - default=1900, - description="Dataset must contain data from at least this year.", - ) - end_year: int = Field( - default=2100, - description="Dataset must contain data up to at least this year.", - ) - number_of_results: int = Field( - default=5, - description="Maximum number of datasets to return, ordered by relevance.", - ge=1, - le=20, - ) - -class DatasetDescription(BaseModel): - content: str - lang: str - - -class DatasetSearchResult(BaseModel): - dataset_id: str - dataset_columns: str = Field( - description=( - "Pipe-separated list of available columns formatted as " - "'COLUMN_ID Label'." - ) - ) - dataset_description: DatasetDescription - dataset_score: float - - -class SearchMelodiDatasetsOutput(BaseModel): - results: list[DatasetSearchResult] +from ..config.tool_metadata import SEARCH_DATASET +from ..core.logging import log_tool +from ..models.melodi import SearchMelodiDatasetsInput, SearchMelodiDatasetsOutput +from ..services.melodi import search_melodi_datasets def register_search_melodi_datasets(mcp: FastMCP) -> None: @@ -77,114 +16,7 @@ def register_search_melodi_datasets(mcp: FastMCP) -> None: meta=SEARCH_DATASET["tool_metadata"], ) @log_tool - async def search_melodi_datasets( + async def search_melodi_datasets_tool( params: SearchMelodiDatasetsInput, ) -> SearchMelodiDatasetsOutput: - es = get_es_client() - filters = [] - if params.start_year: - filters.append({ - "range": { - "metadata.temporal.endPeriod": { - "gte": f"{params.start_year}-01-01" - } - } - }) - if params.end_year: - filters.append({ - "range": { - "metadata.temporal.startPeriod": { - "lte": f"{params.end_year}-12-31" - } - } - }) - - body = { - "size": params.number_of_results, - "query": { - "bool": { - "should": [ - { - "nested": { - "path": "metadata.title", - "query": { - "match": { - "metadata.title.content": { - "query": params.french_query, - "boost": 10, - } - } - }, - } - }, - { - "nested": { - "path": "metadata.abstract", - "query": { - "match": { - "metadata.abstract.content": { - "query": params.french_query, - "boost": 6, - } - } - }, - } - }, - { - "nested": { - "path": "metadata.description", - "query": { - "match": { - "metadata.description.content": { - "query": params.french_query, - "boost": 3, - } - } - }, - } - }, - { - "match": { - "variables_text": { - "query": params.french_query, - "boost": 5, - } - } - }, - ], - "filter": filters, - } - }, - } - - try: - ds_res = es.search(index=INDEX_MELODI_DATASETS, body=body) - except (ESConnectionError, TransportError) as exc: - fail( - "BACKEND_UNAVAILABLE", - f"Melodi datasets search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", - retryable=True, - ) - raise # pragma: no cover -- fail() raises - - results: list[DatasetSearchResult] = [] - for hit in ds_res.get("hits", {}).get("hits", []): - source = hit.get("_source", {}) - description = source.get("metadata", {}).get("description") - # Defensive: index sometimes stores a dict, sometimes a list. - if isinstance(description, list) and description: - description = description[0] - elif isinstance(description, dict): - description = description - else: - description = {"content": "", "lang": "fr"} - results.append( - DatasetSearchResult( - dataset_id=hit.get("_id", ""), - dataset_columns=source.get("columns", ""), - dataset_description=description, - dataset_score=float(hit.get("_score") or 0.0), - ) - ) - return SearchMelodiDatasetsOutput(results=results) + return await search_melodi_datasets(params) diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py index 9295dbf..e2d9398 100644 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ b/src/mcpdiffusion/tools/melodi_search_modalities.py @@ -1,60 +1,12 @@ -"""Tool: search_melodi_modalities - -Rank modality codes/labels for a free-text query on one or more columns -of a Melodi dataset. Returns what `get_melodi_observations` needs. -""" +"""Tool: search_melodi_modalities -- thin registration layer.""" from __future__ import annotations -from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.es import INDEX_MELODI_COLUMNS, get_es_client -from ..helpers.logging import log_tool -from ..helpers.schemas import fail -from .env import SEARCH_MODALITIES - - -class SearchMelodiModalitiesInput(BaseModel): - dataset_id: str = Field( - description="Identifier of the Melodi dataset (from search_melodi_datasets).", - examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], - ) - columns_id: list[str] = Field( - description="Identifiers of the columns within the dataset to search.", - examples=[["PRICES"], ["PRICES", "GEO"]], - ) - french_query: str = Field( - description=( - "Natural-language French query describing the modalities to " - "retrieve (e.g. 'cote de boeuf', 'Ile-de-France', 'female Maria')." - ), - examples=["prix", "boissons non alcoolisees"], - ) - number_of_results: int = Field( - default=10, - description="Maximum number of modalities to return per column.", - ge=1, - le=50, - ) - -class Modality(BaseModel): - code: str - label_en: str - label_fr: str - score: float - - -class ColumnResult(BaseModel): - column_code: str - metadata_columns: str - matching_modalities: list[Modality] - - -class SearchMelodiModalitiesOutput(BaseModel): - results: list[ColumnResult] +from ..config.tool_metadata import SEARCH_MODALITIES +from ..core.logging import log_tool +from ..models.melodi import SearchMelodiModalitiesInput, SearchMelodiModalitiesOutput +from ..services.melodi import search_melodi_modalities def register_search_melodi_modalities(mcp: FastMCP) -> None: @@ -64,100 +16,7 @@ def register_search_melodi_modalities(mcp: FastMCP) -> None: meta=SEARCH_MODALITIES["tool_metadata"], ) @log_tool - async def search_melodi_modalities( + async def search_melodi_modalities_tool( params: SearchMelodiModalitiesInput, ) -> SearchMelodiModalitiesOutput: - es = get_es_client() - - filters = [{"term": {"dataset_id": params.dataset_id}}] - if params.columns_id: - filters.append({"terms": {"code": params.columns_id}}) - - try: - ds_column = es.search( - index=INDEX_MELODI_COLUMNS, - size=20, - query={ - "bool": { - "filter": filters, - "should": [ - { - "match": { - "text": { - "query": params.french_query, - "boost": 2, - } - } - }, - { - "nested": { - "path": "modalities", - "score_mode": "max", - "query": { - "multi_match": { - "query": params.french_query, - "fields": [ - "modalities.code^5", - "modalities.label.en^3", - "modalities.label.fr^3", - ], - "fuzziness": "AUTO", - } - }, - "inner_hits": { - "size": params.number_of_results, - "sort": [{"_score": "desc"}], - }, - } - }, - ], - } - }, - ) - except (ESConnectionError, TransportError) as exc: - fail( - "BACKEND_UNAVAILABLE", - f"Melodi columns search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", - retryable=True, - ) - raise - - results: list[ColumnResult] = [] - for hit in ds_column.get("hits", {}).get("hits", []): - modalities: list[Modality] = [] - inner_hits = ( - hit.get("inner_hits", {}) - .get("modalities", {}) - .get("hits", {}) - .get("hits", []) - ) - for m in inner_hits: - src = m.get("_source", {}) - label = src.get("label", {}) or {} - modalities.append( - Modality( - code=str(src.get("code", "")), - label_en=str(label.get("en", "")), - label_fr=str(label.get("fr", "")), - score=float(m.get("_score") or 0.0), - ) - ) - results.append( - ColumnResult( - column_code=str(hit.get("_source", {}).get("code", "")), - metadata_columns=str(hit.get("_source", {}).get("text", "")), - matching_modalities=modalities, - ) - ) - - if not results: - fail( - "EMPTY_RESULT", - f"No modalities matched for dataset_id={params.dataset_id!r}, " - f"columns_id={params.columns_id!r}, " - f"french_query={params.french_query!r}. " - "Verify the dataset_id and column ids with `search_melodi_datasets`, " - "then try a broader French query.", - ) - return SearchMelodiModalitiesOutput(results=results) + return await search_melodi_modalities(params) diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index 6df1aa3..f68de2f 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -1,108 +1,20 @@ -"""Tool: RMES_describe_resource - -Retrieve all known properties (predicate -> value) of an RDF resource -identified by its full URI, across all graphs or restricted to one. -""" +"""Tool: RMES_describe_resource -- thin registration layer.""" from __future__ import annotations -from typing import Any, Literal, Optional - from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from ..helpers.rmes import ( - DEFAULT_TIMEOUT, - MAX_ROW_LIMIT, - SparqlError, - _execute_sparql, -) -from .env import RMES_DESCRIBE_RESOURCE - - -# --------------------------------------------------------------------------- -# Schémas Pydantic -- RMES_describe_resource -# --------------------------------------------------------------------------- - -class DescribeResourceInput(BaseModel): - uri: str = Field( - description="URI complète de la ressource RDF à décrire.", - examples=["http://id.insee.fr/codes/naf2025/section/A"], - ) - graph: str | None = Field( - default=None, - description=( - "URI d'un graphe nommé pour restreindre la recherche. Sans cette valeur (None par défaut), " - "la recherche se fait sur tous les graphes (plus lent)." - ), - ) - -class ResourceProperty(BaseModel): - graph: str - direction: Literal["outgoing", "incoming"] - predicate: str - value: str - value_type: Optional[str] = None - lang: Optional[str] = None +from ..config.tool_metadata import RMES_DESCRIBE_RESOURCE +from ..core.logging import log_tool +from ..models.rmes import DescribeResourceInput, DescribeResourceOutput +from ..services.rmes import describe_resource -class DescribeResourceOutput(BaseModel): - uri: str - properties: list[ResourceProperty] - count: int - error: Optional[SparqlError] = None - - -def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[ResourceProperty]: - props: list[ResourceProperty] = [] - for b in bindings: - props.append( - ResourceProperty( - graph=b["g"]["value"], - direction=b["direction"]["value"], - predicate=b["p"]["value"], - value=b["o"]["value"], - value_type=b["o"].get("type"), - lang=b["o"].get("xml:lang"), - ) - ) - return props - - -# --------------------------------------------------------------------------- -# Enregistrement du tool MCP -# --------------------------------------------------------------------------- - def register_rmes_describe_resource(mcp: FastMCP) -> None: - @mcp.tool( name=RMES_DESCRIBE_RESOURCE["tool_name"], description=RMES_DESCRIBE_RESOURCE["tool_description"], meta=RMES_DESCRIBE_RESOURCE["tool_metadata"], ) @log_tool - async def describe_resource(params: DescribeResourceInput) -> DescribeResourceOutput: - graph_clause = f"<{params.graph}>" if params.graph else "?g" - graph_values = f"VALUES ?g {{ <{params.graph}> }}" if params.graph else "" - query = f""" - SELECT ?g ?direction ?p ?o WHERE {{ - {graph_values} - {{ - GRAPH {graph_clause} {{ <{params.uri}> ?p ?o }} - BIND("outgoing" AS ?direction) - }} UNION {{ - GRAPH {graph_clause} {{ ?o ?p <{params.uri}> }} - BIND("incoming" AS ?direction) - }} - }} LIMIT {MAX_ROW_LIMIT} - """ - result = await _execute_sparql(query, timeout=DEFAULT_TIMEOUT, max_rows=MAX_ROW_LIMIT) - - if "error" in result: - return DescribeResourceOutput( - uri=params.uri, properties=[], count=0, error=SparqlError(**result["error"]) - ) - - properties = _parse_bindings_to_properties(result["results"]["bindings"]) - return DescribeResourceOutput(uri=params.uri, properties=properties, count=len(properties)) + async def describe_resource_tool(params: DescribeResourceInput) -> DescribeResourceOutput: + return await describe_resource(params) diff --git a/src/mcpdiffusion/tools/rmes_list_graphs.py b/src/mcpdiffusion/tools/rmes_list_graphs.py index 50ff8c1..11d48a1 100644 --- a/src/mcpdiffusion/tools/rmes_list_graphs.py +++ b/src/mcpdiffusion/tools/rmes_list_graphs.py @@ -1,141 +1,20 @@ -"""Tool: RMES_list_graphs - -List named graphs available in the INSEE RDF database (RMES), grouped -by category with counts and example URIs. -""" +"""Tool: RMES_list_graphs -- thin registration layer.""" from __future__ import annotations -from typing import Any, Optional - from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from ..helpers.rmes import ( - CATEGORY_DEFS, - GraphCategoryChoice, - GraphRow, - SparqlError, - _CATEGORY_AUTRE, - _CategoryRule, - _CATEGORY_FIELD_DESCRIPTION, - _categorize, - _get_raw_graph_rows, -) -from .env import RMES_LIST_GRAPHS - - -# --------------------------------------------------------------------------- -# Schémas Pydantic -- RMES_list_graphs -# --------------------------------------------------------------------------- - -class ListGraphsInput(BaseModel): - contains: Optional[str] = Field( - default=None, - description=( - "Filtre les graphes dont l'URI contient cette sous-chaîne (insensible à la " - "casse), ex. 'naf' ou 'qualite/rapport'. Active automatiquement le détail " - "complet (`graphs`) dans les catégories retenues." - ), - examples=["naf", "qualite/rapport", "geo"], - ) - category: GraphCategoryChoice = Field( - default=GraphCategoryChoice.ALL, - description=_CATEGORY_FIELD_DESCRIPTION, - ) - expand: bool = Field( - default=False, - description=( - "Si True, inclut la liste complète des graphes (URI + nb de triplets) pour " - "chaque catégorie retenue, au lieu de seulement quelques exemples. Se " - "déclenche automatiquement si `contains` est fourni ou `category != ALL`." - ), - ) - - -class CategoryBucket(BaseModel): - category: str - label: str - description: str - count: int - total_triples: int - examples: list[str] - graphs: Optional[list[GraphRow]] = None +from ..config.tool_metadata import RMES_LIST_GRAPHS +from ..core.logging import log_tool +from ..models.rmes import ListGraphsInput, ListGraphsOutput +from ..services.rmes import list_graphs -class ListGraphsOutput(BaseModel): - total_graphs_matched: int - categories: list[CategoryBucket] - error: Optional[SparqlError] = None - - -def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: - buckets: dict[str, CategoryBucket] = {} - for row in rows: - cat = _categorize(row["graph"]) - bucket = buckets.get(cat.key) - if bucket is None: - bucket = CategoryBucket( - category=cat.key, - label=cat.label, - description=cat.description, - count=0, - total_triples=0, - examples=[], - ) - buckets[cat.key] = bucket - bucket.count += 1 - bucket.total_triples += row["triples"] - if len(bucket.examples) < 5: - bucket.examples.append(row["graph"]) - - ordered_keys = [c.key for c in CATEGORY_DEFS] + [_CATEGORY_AUTRE.key] - return [buckets[k] for k in ordered_keys if k in buckets] - - -# --------------------------------------------------------------------------- -# Enregistrement du tool MCP -# --------------------------------------------------------------------------- def register_rmes_list_graphs(mcp: FastMCP) -> None: - @mcp.tool( name=RMES_LIST_GRAPHS["tool_name"], description=RMES_LIST_GRAPHS["tool_description"], meta=RMES_LIST_GRAPHS["tool_metadata"], ) @log_tool - async def list_graphs(params: ListGraphsInput) -> ListGraphsOutput: - raw = await _get_raw_graph_rows() - if "error" in raw: - return ListGraphsOutput( - total_graphs_matched=0, - categories=[], - error=SparqlError(**raw["error"]), - ) - rows = raw["rows"] - expand = params.expand - - if params.contains: - needle = params.contains.lower() - rows = [r for r in rows if needle in r["graph"].lower()] - expand = True - - if params.category != GraphCategoryChoice.ALL: - rows = [r for r in rows if _categorize(r["graph"]).key == params.category.value] - expand = True - - summary = _build_category_summary(rows) - - if expand: - rows_by_graph = {r["graph"]: r["triples"] for r in rows} - for bucket in summary: - bucket_rows = [ - GraphRow(graph=g, triples=t) - for g, t in rows_by_graph.items() - if _categorize(g).key == bucket.category - ] - bucket_rows.sort(key=lambda r: r.triples, reverse=True) - bucket.graphs = bucket_rows - - return ListGraphsOutput(total_graphs_matched=len(rows), categories=summary) + async def list_graphs_tool(params: ListGraphsInput) -> ListGraphsOutput: + return await list_graphs(params) diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index 70fe94f..175a2ae 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -1,72 +1,15 @@ -"""Tool: RMES_run_sparql - -Execute arbitrary SPARQL queries against the INSEE semantic graph (RMES). -Supports SELECT, ASK, CONSTRUCT, and DESCRIBE forms. -""" +"""Tool: RMES_run_sparql -- thin registration layer.""" from __future__ import annotations -from typing import Any, Literal, Optional - from fastmcp import FastMCP -from pydantic import BaseModel, Field - -from ..helpers.logging import log_tool -from ..helpers.rmes import ( - DEFAULT_ROW_LIMIT, - DEFAULT_TIMEOUT, - KNOWN_VOCABULARIES_NOTE, - MAX_ROW_LIMIT, - MAX_TIMEOUT, - SparqlError, - SparqlErrorType, - _execute_sparql, -) -from .env import RMES_RUN_SPARQL - - -# --------------------------------------------------------------------------- -# Schémas Pydantic -- RMES_run_sparql -# --------------------------------------------------------------------------- - -class RunSparqlInput(BaseModel): - full_sparql_query: str = Field( - description="Requête SPARQL complète (SELECT / ASK / CONSTRUCT / DESCRIBE).", - ) - timeout: float = Field( - default=DEFAULT_TIMEOUT, - description=f"Timeout en secondes (plafonné à {MAX_TIMEOUT}s).", - gt=0, - ) - max_rows: int = Field( - default=DEFAULT_ROW_LIMIT, - description=f"Limite de lignes ajoutée si absente de la requête (plafonnée à {MAX_ROW_LIMIT}).", - ge=1, - le=MAX_ROW_LIMIT, - ) - -class RunSparqlOutput(BaseModel): - format: Literal["json", "turtle"] = "json" - limit_added: Optional[int] = None - hint: Optional[str] = None - # Résultats SELECT/ASK : variables déclarées + lignes brutes (bindings SPARQL JSON). - # On garde les lignes en dict libre plutôt que de les typer entièrement : les - # variables retournées dépendent entièrement de la requête SPARQL de l'appelant, - # les figer dans un schéma fixe serait soit incomplet, soit un schéma générique - # sans valeur ajoutée par rapport à un dict. - variables: Optional[list[str]] = None - bindings: Optional[list[dict[str, Any]]] = None - # Résultat CONSTRUCT/DESCRIBE - turtle: Optional[str] = None - error: Optional[SparqlError] = None +from ..config.tool_metadata import RMES_RUN_SPARQL +from ..core.logging import log_tool +from ..models.rmes import RunSparqlInput, RunSparqlOutput +from ..services.rmes import KNOWN_VOCABULARIES_NOTE, run_sparql -# --------------------------------------------------------------------------- -# Enregistrement du tool MCP -# --------------------------------------------------------------------------- - def register_rmes_run_sparql(mcp: FastMCP) -> None: - @mcp.tool( name=RMES_RUN_SPARQL["tool_name"], description=RMES_RUN_SPARQL["tool_description"] + "\n" + KNOWN_VOCABULARIES_NOTE + "\n\n" @@ -80,37 +23,10 @@ def register_rmes_run_sparql(mcp: FastMCP) -> None: " }\n" "} LIMIT 10\n" "\n" - "Les requêtes CONSTRUCT/DESCRIBE renvoient du Turtle (`format=\"turtle\"`, champ `turtle`) " - "plutôt que des lignes (`format=\"json\"`, champs `variables`/`bindings`).", + "Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format=\"turtle\"`, champ `turtle`) " + "plutot que des lignes (`format=\"json\"`, champs `variables`/`bindings`).", meta=RMES_RUN_SPARQL["tool_metadata"], ) @log_tool - async def run_sparql(params: RunSparqlInput) -> RunSparqlOutput: - if not params.full_sparql_query or not params.full_sparql_query.strip(): - return RunSparqlOutput( - error=SparqlError( - type=SparqlErrorType.EMPTY_QUERY, - message="La requête est vide.", - query=params.full_sparql_query, - ) - ) - - max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) - result = await _execute_sparql(params.full_sparql_query, timeout=params.timeout, max_rows=max_rows) - - if "error" in result: - return RunSparqlOutput(error=SparqlError(**result["error"])) - - if result.get("format") == "turtle": - return RunSparqlOutput( - format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"] - ) - - meta = result.get("_meta", {}) - return RunSparqlOutput( - format="json", - limit_added=meta.get("limit_added"), - hint=meta.get("hint"), - variables=result.get("head", {}).get("vars"), - bindings=result.get("results", {}).get("bindings"), - ) + async def run_sparql_tool(params: RunSparqlInput) -> RunSparqlOutput: + return await run_sparql(params) diff --git a/tests/conftest.py b/tests/conftest.py index 7b1330c..eba2b23 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,11 +8,13 @@ import pytest from fastmcp import Client, FastMCP -from mcpdiffusion.helpers import rmes as rmes_module +from mcpdiffusion.config.settings import get_settings +from mcpdiffusion.services import rmes as rmes_service from mcpdiffusion.tools.rmes_describe_resource import register_rmes_describe_resource from mcpdiffusion.tools.rmes_list_graphs import register_rmes_list_graphs from mcpdiffusion.tools.rmes_run_sparql import register_rmes_run_sparql +_ENDPOINT = get_settings().rmes_endpoint # --------------------------------------------------------------------------- # Helpers: fake httpx responses @@ -23,7 +25,7 @@ def _json_response(body: dict[str, Any], status: int = 200) -> httpx.Response: status_code=status, content=json.dumps(body).encode(), headers={"content-type": "application/sparql-results+json"}, - request=httpx.Request("POST", rmes_module.ENDPOINT), + request=httpx.Request("POST", _ENDPOINT), ) @@ -32,7 +34,7 @@ def _text_response(text: str, status: int = 200) -> httpx.Response: status_code=status, content=text.encode(), headers={"content-type": "text/turtle"}, - request=httpx.Request("POST", rmes_module.ENDPOINT), + request=httpx.Request("POST", _ENDPOINT), ) @@ -41,7 +43,7 @@ def _error_response(status: int, body: str = "Bad Request") -> httpx.Response: status_code=status, content=body.encode(), headers={"content-type": "text/plain"}, - request=httpx.Request("POST", rmes_module.ENDPOINT), + request=httpx.Request("POST", _ENDPOINT), ) @@ -55,7 +57,7 @@ def _out(call_tool_result) -> dict[str, Any]: # --------------------------------------------------------------------------- class FakeAsyncClient: - """Drop-in replacement for httpx.AsyncClient used by rmes._get_client().""" + """Drop-in replacement for httpx.AsyncClient used by sparql.get_sparql_client().""" def __init__(self, handler): self.handler = handler @@ -95,11 +97,11 @@ def rmes_client(rmes_mcp: FastMCP) -> Client: @pytest.fixture def mock_sparql(monkeypatch): """Return a callable that sets up the fake SPARQL endpoint.""" - rmes_module._GRAPH_CACHE["data"] = None - rmes_module._GRAPH_CACHE["ts"] = 0.0 + rmes_service._GRAPH_CACHE["data"] = None + rmes_service._GRAPH_CACHE["ts"] = 0.0 def _setup(handler): fake = FakeAsyncClient(handler) - monkeypatch.setattr(rmes_module, "_get_client", lambda: fake) + monkeypatch.setattr(rmes_service, "get_sparql_client", lambda *a, **kw: fake) return _setup diff --git a/tests/test_middleware.py b/tests/test_middleware.py index ca72d7c..bf7b864 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,8 +1,6 @@ -"""Unit tests for mcpdiffusion.middleware (RateLimitMiddleware).""" +"""Unit tests for mcpdiffusion.core.middleware (RateLimitMiddleware).""" from __future__ import annotations -from unittest.mock import patch - import pytest from starlette.applications import Starlette from starlette.requests import Request @@ -10,8 +8,8 @@ from starlette.routing import Route from starlette.testclient import TestClient -from mcpdiffusion import middleware as mw -from mcpdiffusion.middleware import RateLimitMiddleware +from mcpdiffusion.config.settings import Settings +from mcpdiffusion.core.middleware import RateLimitMiddleware # --------------------------------------------------------------------------- @@ -19,22 +17,21 @@ # --------------------------------------------------------------------------- def _make_app(rate_limit: int = 3) -> Starlette: - """Create a minimal Starlette app with the RateLimitMiddleware.""" + """Create a minimal Starlette app with a DI-configured RateLimitMiddleware.""" async def homepage(request: Request) -> PlainTextResponse: return PlainTextResponse("ok") + settings = Settings( + GLOBAL_REQUEST_MIN=rate_limit, + TZ="Europe/Paris", + _env_file=None, + ) app = Starlette(routes=[Route("/", homepage)]) - app.add_middleware(RateLimitMiddleware) + app.add_middleware(RateLimitMiddleware, settings=settings) return app -@pytest.fixture(autouse=True) -def _reset_limiter(): - """Reset the module-level limiter storage before each test.""" - mw._limits_storage.reset() - - # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- @@ -43,19 +40,15 @@ class TestRateLimitAllowed: """Requests within the limit should pass through normally.""" def test_single_request_returns_200(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 10), \ - patch.object(mw, "_rate", mw.parse("10/minute")): - client = TestClient(_make_app()) - resp = client.get("/") + client = TestClient(_make_app(rate_limit=10)) + resp = client.get("/") assert resp.status_code == 200 assert resp.text == "ok" def test_response_contains_rate_limit_headers(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 10), \ - patch.object(mw, "_rate", mw.parse("10/minute")): - client = TestClient(_make_app()) - resp = client.get("/") + client = TestClient(_make_app(rate_limit=10)) + resp = client.get("/") assert "X-RateLimit-Limit" in resp.headers assert "X-RateLimit-Remaining" in resp.headers @@ -63,11 +56,9 @@ def test_response_contains_rate_limit_headers(self): assert resp.headers["X-RateLimit-Limit"] == "10" def test_remaining_decreases(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 5), \ - patch.object(mw, "_rate", mw.parse("5/minute")): - client = TestClient(_make_app()) - r1 = client.get("/") - r2 = client.get("/") + client = TestClient(_make_app(rate_limit=5)) + r1 = client.get("/") + r2 = client.get("/") remaining1 = int(r1.headers["X-RateLimit-Remaining"]) remaining2 = int(r2.headers["X-RateLimit-Remaining"]) @@ -78,21 +69,17 @@ class TestRateLimitExceeded: """Requests over the limit should be rejected with 429.""" def test_returns_429_when_limit_exceeded(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 2), \ - patch.object(mw, "_rate", mw.parse("2/minute")): - client = TestClient(_make_app()) - client.get("/") - client.get("/") - resp = client.get("/") + client = TestClient(_make_app(rate_limit=2)) + client.get("/") + client.get("/") + resp = client.get("/") assert resp.status_code == 429 def test_429_body_contains_detail_and_retry_after(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 1), \ - patch.object(mw, "_rate", mw.parse("1/minute")): - client = TestClient(_make_app()) - client.get("/") - resp = client.get("/") + client = TestClient(_make_app(rate_limit=1)) + client.get("/") + resp = client.get("/") body = resp.json() assert "detail" in body @@ -104,34 +91,34 @@ def test_429_body_contains_detail_and_retry_after(self): assert len(parts) == 3 def test_429_headers(self): - with patch.object(mw, "GLOBAL_REQUEST_MIN", 1), \ - patch.object(mw, "_rate", mw.parse("1/minute")): - client = TestClient(_make_app()) - client.get("/") - resp = client.get("/") + client = TestClient(_make_app(rate_limit=1)) + client.get("/") + resp = client.get("/") assert resp.headers["X-RateLimit-Remaining"] == "0" assert "Retry-After" in resp.headers -class TestRateLimitPerPath: - """Rate limits should be tracked independently per path.""" - - def test_different_paths_have_separate_counters(self): - async def other(request: Request) -> PlainTextResponse: - return PlainTextResponse("other") +class TestRateLimitPerIP: + """Rate limits should be tracked per IP.""" - with patch.object(mw, "GLOBAL_REQUEST_MIN", 1), \ - patch.object(mw, "_rate", mw.parse("1/minute")): - app = Starlette(routes=[ - Route("/a", lambda r: PlainTextResponse("a")), - Route("/b", lambda r: PlainTextResponse("b")), - ]) - app.add_middleware(RateLimitMiddleware) - client = TestClient(app) + def test_different_paths_share_same_counter(self): + """With per-IP limiting, different paths share the same counter.""" + settings = Settings( + GLOBAL_REQUEST_MIN=1, + TZ="Europe/Paris", + _env_file=None, + ) + app = Starlette(routes=[ + Route("/a", lambda r: PlainTextResponse("a")), + Route("/b", lambda r: PlainTextResponse("b")), + ]) + app.add_middleware(RateLimitMiddleware, settings=settings) + client = TestClient(app) - resp_a = client.get("/a") - resp_b = client.get("/b") + resp_a = client.get("/a") + resp_b = client.get("/b") + # Same IP, so second request is rate-limited assert resp_a.status_code == 200 - assert resp_b.status_code == 200 + assert resp_b.status_code == 429 diff --git a/tests/test_rmes_helpers.py b/tests/test_rmes_helpers.py index c037888..cb8f794 100644 --- a/tests/test_rmes_helpers.py +++ b/tests/test_rmes_helpers.py @@ -1,30 +1,27 @@ -"""Unit tests for mcpdiffusion.helpers.rmes (pure logic, no MCP layer).""" +"""Unit tests for mcpdiffusion.services.rmes (pure logic, no MCP layer).""" from __future__ import annotations -import json import time -from typing import Any import httpx import pytest -from mcpdiffusion.helpers.rmes import ( - GRAPH_BASE, - SparqlErrorType, +from mcpdiffusion.models.rmes import GRAPH_BASE, SparqlErrorType +from mcpdiffusion.services import rmes as rmes_service +from mcpdiffusion.services.rmes import ( _CATEGORY_AUTRE, + _GRAPH_CACHE, + _GRAPH_CACHE_TTL, _accept_header, _categorize, _detect_query_form, _ensure_limit, _error_payload, + _execute_sparql, _get_raw_graph_rows, _relative_path, - _execute_sparql, - _GRAPH_CACHE, - _GRAPH_CACHE_TTL, ) from tests.conftest import FakeAsyncClient, _json_response -from mcpdiffusion.helpers import rmes as rmes_module # =================================================================== @@ -219,10 +216,8 @@ def test_external_uri_falls_back_to_autre(self): assert cat.key == "autre" def test_specific_rules_take_precedence(self): - # "codes" exact → codes_concepts_generiques, not nomenclatures (prefix "codes/") cat = _categorize(f"{GRAPH_BASE}codes") assert cat.key == "codes_concepts_generiques" - # "codes/naf2025" → nomenclatures (prefix "codes/"), not codes_concepts_generiques cat = _categorize(f"{GRAPH_BASE}codes/naf2025") assert cat.key == "nomenclatures" @@ -248,15 +243,15 @@ def test_extra_fields(self): # =================================================================== -# _execute_sparql (async, mocked HTTP) +# _execute_sparql (async, mocked HTTP via DI) # =================================================================== @pytest.fixture def mock_http(monkeypatch): - """Patch _get_client to return a FakeAsyncClient.""" + """Patch get_sparql_client where it is used (services.rmes namespace).""" def _setup(handler): fake = FakeAsyncClient(handler) - monkeypatch.setattr(rmes_module, "_get_client", lambda: fake) + monkeypatch.setattr(rmes_service, "get_sparql_client", lambda *a, **kw: fake) return _setup @@ -269,7 +264,7 @@ async def test_unknown_form_returns_error_without_http_call(self, mock_http): assert "error" in result assert result["error"]["type"] == SparqlErrorType.INVALID_QUERY_FORM - assert len(called) == 0 # no HTTP call made + assert len(called) == 0 async def test_select_success(self, mock_http): body = {"head": {"vars": ["x"]}, "results": {"bindings": []}} @@ -403,7 +398,7 @@ def handler(url, **kw): result2 = await _get_raw_graph_rows() assert result1 == result2 - assert len(call_count) == 1 # HTTP called only once + assert len(call_count) == 1 async def test_cache_expires_after_ttl(self, mock_http, monkeypatch): body = { @@ -423,7 +418,6 @@ def handler(url, **kw): await _get_raw_graph_rows() assert len(call_count) == 1 - # Simulate cache expiry _GRAPH_CACHE["ts"] = time.time() - _GRAPH_CACHE_TTL - 1 await _get_raw_graph_rows() diff --git a/tests/test_rmes_tools.py b/tests/test_rmes_tools.py index b6ab785..d212908 100644 --- a/tests/test_rmes_tools.py +++ b/tests/test_rmes_tools.py @@ -1,7 +1,7 @@ """Unit tests for the three RMES tools (list_graphs, describe_resource, run_sparql). All HTTP calls to the real SPARQL endpoint are mocked via monkeypatch on -`mcpdiffusion.helpers.rmes._get_client`, so these tests run offline. +`mcpdiffusion.infra.sparql.get_sparql_client`, so these tests run offline. """ from __future__ import annotations diff --git a/uv.lock b/uv.lock index a96a9c4..addeb15 100644 --- a/uv.lock +++ b/uv.lock @@ -1034,6 +1034,7 @@ dependencies = [ { name = "httpx" }, { name = "limits" }, { name = "lxml" }, + { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "requests" }, { name = "starlette" }, @@ -1055,6 +1056,7 @@ requires-dist = [ { name = "httpx", specifier = "==0.28.1" }, { name = "limits", specifier = ">=5.8.0" }, { name = "lxml", specifier = "==6.1.2" }, + { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = "==1.2.3" }, { name = "requests", specifier = "==2.34.2" }, { name = "starlette", specifier = "==1.6.0" }, From c8c8b4e39c0d48d05842d5f582abc46a479ea7a7 Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Fri, 21 Aug 2026 08:56:20 +0200 Subject: [PATCH 02/55] refacto es to attach to lifespan --- pyproject.toml | 2 +- src/mcpdiffusion/infra/elasticsearch.py | 55 +++----- src/mcpdiffusion/server.py | 3 +- src/mcpdiffusion/services/insee_search.py | 7 +- src/mcpdiffusion/services/melodi.py | 9 +- .../tools/insee_search_chiffrecle.py | 5 +- .../tools/insee_search_conjoncture.py | 5 +- .../tools/insee_search_documents.py | 5 +- .../tools/melodi_search_datasets.py | 6 +- .../tools/melodi_search_modalities.py | 6 +- uv.lock | 132 +++++++++++++++++- 11 files changed, 180 insertions(+), 55 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bb3bf2..54673d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "MCP server for INSEE data diffusion" requires-python = ">=3.12" dependencies = [ - "fastmcp==3.4.2", + "fastmcp[tasks]==3.4.2", "elasticsearch==9.5.0", "uvicorn==0.52.4", "requests==2.34.2", diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py index 6e1b9dd..b52cbf7 100644 --- a/src/mcpdiffusion/infra/elasticsearch.py +++ b/src/mcpdiffusion/infra/elasticsearch.py @@ -1,39 +1,28 @@ """Centralized Elasticsearch client singleton with injected settings.""" -from __future__ import annotations - -import logging +from __future__ import annotations +from fastmcp.server.lifespan import lifespan +from fastmcp import Context from elasticsearch import Elasticsearch - -from ..config.settings import Settings, get_settings +from ..config.settings import get_settings +import logging logger = logging.getLogger("mcp.main") -_client: Elasticsearch | None = None - - -def get_es_client(settings: Settings | None = None) -> Elasticsearch: - """Return the shared Elasticsearch client, building it on first call.""" - global _client - if _client is None: - s = settings or get_settings() - if not s.es_host: - raise RuntimeError( - "ES_HOST environment variable is not set. " - "See .env.example for the expected value." - ) - _client = Elasticsearch( - s.es_host, - verify_certs=s.tls_verify, - request_timeout=30, - max_retries=2, - retry_on_timeout=True, - ) - logger.info("Elasticsearch client initialized for %s", s.es_host) - return _client - - -def reset_es_client() -> None: - """Drop the cached client. Used by tests / long-running reconfiguration.""" - global _client - _client = None +@lifespan +async def build_es_client(server): + s = get_settings() + if not s.es_host: + raise RuntimeError("ES_HOST is not set. See .env.example.") + client = Elasticsearch( + s.es_host, verify_certs=s.tls_verify, + request_timeout=30, max_retries=2, retry_on_timeout=True, + ) + logger.info("Elasticsearch client initialized for %s", s.es_host) + yield {"es_client": client} + + await client.close() + + +def get_client_es(ctx : Context) : + return ctx.lifespan_context["es_client"] \ No newline at end of file diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 12931d0..bc2c9b0 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -16,13 +16,14 @@ from .core.logging import MAIN_LOGGER_NAME, UVICORN_LOGGING_CONFIG from .core.middleware import RateLimitMiddleware from .tools import register_tools +from .infra.elasticsearch import build_es_client load_dotenv() settings = get_settings() logger = logging.getLogger(MAIN_LOGGER_NAME) -mcp = FastMCP("INSEE-mcp-diffusion") +mcp = FastMCP("INSEE-mcp-diffusion", lifespan = build_es_client ) register_tools(mcp, toollist=settings.toollist) diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index 325ad8c..83b1f8d 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -13,7 +13,6 @@ from ..config.settings import Settings, get_settings from ..data.geography import DICT_GEO from ..data.themes import KEYS_THEME_NIV1 -from ..infra.elasticsearch import get_es_client from ..models.insee import DocumentHit @@ -135,14 +134,12 @@ def execute_search( should: list, must_not: list, number_of_results: int, - client: Optional[Elasticsearch] = None, + es: Elasticsearch, settings: Settings | None = None, ) -> list[DocumentHit]: """Run the assembled bool query and return whitelisted DocumentHit records.""" s = settings or get_settings() - client = client or get_es_client(s) - - search = Search(using=client, index=s.es_index_produits).query( + search = Search(using=es, index=s.es_index_produits).query( Q( "function_score", query=Q( diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 0d197e1..8f77cee 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -5,11 +5,9 @@ import httpx from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError - +from elasticsearch import TransportError, Elasticsearch from ..config.settings import Settings, get_settings from ..core.errors import fail -from ..infra.elasticsearch import get_es_client from ..infra.http import create_async_client from ..models.melodi import ( ColumnResult, @@ -116,9 +114,9 @@ async def search_melodi_datasets( params: SearchMelodiDatasetsInput, *, settings: Settings | None = None, + es: Elasticsearch, ) -> SearchMelodiDatasetsOutput: s = settings or get_settings() - es = get_es_client(s) filters: list[dict[str, Any]] = [] if params.start_year: filters.append({ @@ -231,10 +229,9 @@ async def search_melodi_modalities( params: SearchMelodiModalitiesInput, *, settings: Settings | None = None, + es: Elasticsearch, ) -> SearchMelodiModalitiesOutput: s = settings or get_settings() - es = get_es_client(s) - filters: list[dict[str, Any]] = [{"term": {"dataset_id": params.dataset_id}}] if params.columns_id: filters.append({"terms": {"code": params.columns_id}}) diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index a0b628f..402d24e 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -3,11 +3,12 @@ from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_CHIFFRECLEF from ..core.errors import fail from ..core.logging import log_tool +from ..infra.elasticsearch import get_client_es from ..models.insee import ( SearchInseeChiffrecleInput, SearchInseeChiffrecleOutput, @@ -28,6 +29,7 @@ def register_search_insee_chiffreclef(mcp: FastMCP) -> None: @log_tool async def search_insee_chiffrecle( params: SearchInseeChiffrecleInput, + ctx: Context, ) -> SearchInseeChiffrecleOutput: must, filters, should, must_not = build_text_clauses( query=params.query, @@ -49,6 +51,7 @@ async def search_insee_chiffrecle( should=should, must_not=must_not, number_of_results=params.number_of_results, + es=get_client_es(ctx), ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index 1a00c8c..011e6fe 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -4,12 +4,13 @@ from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError from elasticsearch.dsl import Q -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_CONJONCTURE from ..core.errors import fail from ..core.logging import log_tool from ..data.themes import DICT_THEME_CONJ +from ..infra.elasticsearch import get_client_es from ..models.insee import ( SearchInseeConjonctureInput, SearchInseeConjonctureOutput, @@ -30,6 +31,7 @@ def register_search_insee_conjoncture(mcp: FastMCP) -> None: @log_tool async def search_insee_conjoncture( params: SearchInseeConjonctureInput, + ctx: Context, ) -> SearchInseeConjonctureOutput: must, filters, should, must_not = build_text_clauses( query=params.query, @@ -52,6 +54,7 @@ async def search_insee_conjoncture( should=should, must_not=must_not, number_of_results=params.number_of_results, + es=get_client_es(ctx), ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index febbc0c..211934d 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -3,11 +3,12 @@ from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_DOCUMENTS from ..core.errors import fail from ..core.logging import log_tool +from ..infra.elasticsearch import get_client_es from ..models.insee import ( SearchInseeDocumentsInput, SearchInseeDocumentsOutput, @@ -28,6 +29,7 @@ def register_search_insee_documents(mcp: FastMCP) -> None: @log_tool async def search_insee_documents( params: SearchInseeDocumentsInput, + ctx: Context, ) -> SearchInseeDocumentsOutput: must, filters, should, must_not = build_text_clauses( query=params.query, @@ -49,6 +51,7 @@ async def search_insee_documents( should=should, must_not=must_not, number_of_results=params.number_of_results, + es=get_client_es(ctx), ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py index edc351a..7a0c590 100644 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ b/src/mcpdiffusion/tools/melodi_search_datasets.py @@ -1,10 +1,11 @@ """Tool: search_melodi_datasets -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_DATASET from ..core.logging import log_tool +from ..infra.elasticsearch import get_client_es from ..models.melodi import SearchMelodiDatasetsInput, SearchMelodiDatasetsOutput from ..services.melodi import search_melodi_datasets @@ -18,5 +19,6 @@ def register_search_melodi_datasets(mcp: FastMCP) -> None: @log_tool async def search_melodi_datasets_tool( params: SearchMelodiDatasetsInput, + ctx: Context, ) -> SearchMelodiDatasetsOutput: - return await search_melodi_datasets(params) + return await search_melodi_datasets(params, es=get_client_es(ctx)) diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py index e2d9398..c01a48b 100644 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ b/src/mcpdiffusion/tools/melodi_search_modalities.py @@ -1,10 +1,11 @@ """Tool: search_melodi_modalities -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_MODALITIES from ..core.logging import log_tool +from ..infra.elasticsearch import get_client_es from ..models.melodi import SearchMelodiModalitiesInput, SearchMelodiModalitiesOutput from ..services.melodi import search_melodi_modalities @@ -18,5 +19,6 @@ def register_search_melodi_modalities(mcp: FastMCP) -> None: @log_tool async def search_melodi_modalities_tool( params: SearchMelodiModalitiesInput, + ctx: Context, ) -> SearchMelodiModalitiesOutput: - return await search_melodi_modalities(params) + return await search_melodi_modalities(params, es=get_client_es(ctx)) diff --git a/uv.lock b/uv.lock index addeb15..c40b373 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -95,6 +104,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, ] +[[package]] +name = "burner-redis" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/89/54706febafc135095b2a9d797cfbd4eed2ab1ad7819808b99b587020471b/burner_redis-0.1.7.tar.gz", hash = "sha256:7474ff092669fd11ef765411572cdafcc3d89b8054aef4ca0617be6d6be4c680", size = 638644, upload-time = "2026-05-08T15:01:42.961Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/5d/198bd1d22e504b3034353430703afbdb3efe6e25cb90bf52d896e1d266a7/burner_redis-0.1.7-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f80c866996e0455d584eb3c0f3b067e411c632fb0519eab454e0968edf01e62c", size = 1288888, upload-time = "2026-05-08T15:01:26.103Z" }, + { url = "https://files.pythonhosted.org/packages/2f/4e/ce5c91b884ac37fcd380756402536f8810964014097950900517ce8bd30c/burner_redis-0.1.7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a3d9569a376b690fb5876d454e4904443332dc3ad5c0057e149fc2ad220bf599", size = 1234282, upload-time = "2026-05-08T15:01:28.286Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/31c25cc88143eac2dddcc394151a0db627923d44c94376a83768552c9f13/burner_redis-0.1.7-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20eba1917e3bca9eea5957d5700ff8defcb5a209e57a7841d005549aa0151f44", size = 1337341, upload-time = "2026-05-08T15:01:30.397Z" }, + { url = "https://files.pythonhosted.org/packages/e1/32/95cfa1833316ca2b6b2e58150a4900bc1ad256043cdd36198f1887618ccc/burner_redis-0.1.7-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39111467059b8a28f15ea061d2414ec25c3e57c65759983f90f4d358e7d6a72d", size = 1366800, upload-time = "2026-05-08T15:01:32.891Z" }, + { url = "https://files.pythonhosted.org/packages/34/ad/93c3916f053f89b7b5760da5bf855cd78b7885d480f9cfcc64f3732c1dc2/burner_redis-0.1.7-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9b5adfe99aeb8407f468078f3769b2a63e9168fea12f7709df5d2a3b152706e4", size = 1538160, upload-time = "2026-05-08T15:01:34.667Z" }, + { url = "https://files.pythonhosted.org/packages/5c/b9/19bae42cb124932d71168bc8e5bcb1da33aa62b908e5e632b3d298d7cb15/burner_redis-0.1.7-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:591a9d20685f9d6d22bf0c863b50b12dfcf328b06111b3f62c33cd3185d48ce0", size = 1591491, upload-time = "2026-05-08T15:01:36.708Z" }, + { url = "https://files.pythonhosted.org/packages/f5/30/207f47f406619a5b564355d2946c3171f84231a28b800709b5645b06a5ae/burner_redis-0.1.7-cp310-abi3-win_amd64.whl", hash = "sha256:f6cf4ac666766b32fd63940aad0c120847905fd3102c17e5b6b305f91a21d079", size = 1117564, upload-time = "2026-05-08T15:01:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/76/6f/e9beaf46c5e9fd10dfcdb889ebf7d3aa85142c650c0ab17ab284194f58e1/burner_redis-0.1.7-cp310-abi3-win_arm64.whl", hash = "sha256:458f88feeddfb40a586cc3fcbd8e9384bbdfd2a4512a695af4900e06052570d4", size = 1040407, upload-time = "2026-05-08T15:01:41.235Z" }, +] + [[package]] name = "cachetools" version = "7.1.7" @@ -371,6 +396,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -394,6 +428,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/38/ce65091ff20a16e06d17418c4353af5f56d3190821b1a06983c79ae79274/courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e", size = 34193, upload-time = "2026-06-01T17:30:14.984Z" }, ] +[[package]] +name = "cronsim" +version = "2.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/1a/02f105147f7f2e06ed4f734ff5a6439590bb275a53dd91fc73df6312298a/cronsim-2.7-py3-none-any.whl", hash = "sha256:1e1431fa08c51dc7f72e67e571c7c7a09af26420169b607badd4ca9677ffad1e", size = 14213, upload-time = "2025-10-21T16:38:20.431Z" }, +] + [[package]] name = "cryptography" version = "50.0.0" @@ -571,6 +613,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/4d/8b1ba42251160e11ca34686344572121432c23a082d56ef6bbdec5888fc1/fastmcp-3.4.2-py3-none-any.whl", hash = "sha256:c87a62b029f0c5400ada85f683629345d2466c39169f0cb853e487b2f7308c08", size = 8018, upload-time = "2026-06-06T01:30:38.118Z" }, ] +[package.optional-dependencies] +tasks = [ + { name = "fastmcp-slim", extra = ["tasks"] }, +] + [[package]] name = "fastmcp-slim" version = "3.4.2" @@ -621,6 +668,9 @@ server = [ { name = "watchfiles" }, { name = "websockets" }, ] +tasks = [ + { name = "pydocket" }, +] [[package]] name = "griffelib" @@ -1030,7 +1080,7 @@ source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, { name = "elasticsearch" }, - { name = "fastmcp" }, + { name = "fastmcp", extra = ["tasks"] }, { name = "httpx" }, { name = "limits" }, { name = "lxml" }, @@ -1052,7 +1102,7 @@ dev = [ requires-dist = [ { name = "beautifulsoup4", specifier = "==4.15.0" }, { name = "elasticsearch", specifier = "==9.5.0" }, - { name = "fastmcp", specifier = "==3.4.2" }, + { name = "fastmcp", extras = ["tasks"], specifier = "==3.4.2" }, { name = "httpx", specifier = "==0.28.1" }, { name = "limits", specifier = ">=5.8.0" }, { name = "lxml", specifier = "==6.1.2" }, @@ -1148,6 +1198,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.5" @@ -1172,6 +1231,9 @@ keyring = [ memory = [ { name = "cachetools" }, ] +redis = [ + { name = "redis" }, +] [[package]] name = "pycparser" @@ -1291,6 +1353,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] +[[package]] +name = "pydocket" +version = "0.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "burner-redis" }, + { name = "cloudpickle" }, + { name = "cronsim" }, + { name = "opentelemetry-api" }, + { name = "prometheus-client" }, + { name = "py-key-value-aio", extra = ["memory", "redis"] }, + { name = "python-json-logger" }, + { name = "redis" }, + { name = "rich" }, + { name = "typer" }, + { name = "typing-extensions" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "uncalled-for" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/6b/a87c6e3fd197807f630af4270aa2ce8f4c1fca4e43bba783f273d298d646/pydocket-0.24.1.tar.gz", hash = "sha256:477d77be1fcfd10ee0c2d0b8aa8c6e97851b9c7f39bb6f2b4e6d42e9b4d6e95a", size = 430759, upload-time = "2026-08-10T19:50:03.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/05/4e3b902bc0ca407188aa5fe38af49be634487aec242a77287103242e11b2/pydocket-0.24.1-py3-none-any.whl", hash = "sha256:1faa6c3d566f1f0431e35dfa12db4ccee515d945b913b516ee3e6279afb9e789", size = 130249, upload-time = "2026-08-10T19:50:01.627Z" }, +] + [[package]] name = "pygments" version = "2.21.0" @@ -1373,6 +1459,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] +[[package]] +name = "python-json-logger" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/25/5473e46b179f8e8b4ad3aeeb36773d1701b7770eaf5e5bc2025c7303b598/python_json_logger-4.2.0.tar.gz", hash = "sha256:e371ebe22ec01e289850102091a2b1f6fc9e655c7f1f5f29073936756c290afa", size = 18211, upload-time = "2026-08-15T11:36:38.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/55/6467fde553886cb293e41538f3a8b4e4fd4688c6df242cf982162d8367fb/python_json_logger-4.2.0-py3-none-any.whl", hash = "sha256:158a52126fcd6869e09574d2b66272666f3dc8f468c62637ef9a1fa883719cb9", size = 14988, upload-time = "2026-08-15T11:36:36.821Z" }, +] + [[package]] name = "python-multipart" version = "0.0.32" @@ -1465,6 +1560,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "redis" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -1717,6 +1821,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1797,6 +1910,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/01/af18878398102a5a5afa0811f4f8f2a8a94a60cc16e8e9cf54bc95f96808/trafilatura-2.2.0-py3-none-any.whl", hash = "sha256:ac43592a6201264dfc4f9c361cbe3eb3fea96e54437010a159d5e7365360ed98", size = 151906, upload-time = "2026-07-31T16:06:46.485Z" }, ] +[[package]] +name = "typer" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/40/4a3db7990d1f62a53182aa96eaef57aeb2886a27f90a195bc66713565d31/typer-0.27.1.tar.gz", hash = "sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df", size = 203994, upload-time = "2026-08-03T14:41:03.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From ab4e9b916667bc04eea6f12ef87850afc1bd1429 Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Fri, 21 Aug 2026 09:19:20 +0200 Subject: [PATCH 03/55] refacto http clients using lifespan and getter into tools --- src/mcpdiffusion/infra/elasticsearch.py | 28 +----- src/mcpdiffusion/infra/http.py | 33 +------ src/mcpdiffusion/infra/lifespan.py | 60 ++++++++++++ src/mcpdiffusion/infra/sparql.py | 24 +---- src/mcpdiffusion/server.py | 4 +- src/mcpdiffusion/services/insee_document.py | 18 ++-- src/mcpdiffusion/services/melodi.py | 13 ++- src/mcpdiffusion/services/rmes.py | 21 ++-- src/mcpdiffusion/tools/insee_get_document.py | 6 +- .../tools/melodi_get_observations.py | 6 +- .../tools/rmes_describe_resource.py | 7 +- src/mcpdiffusion/tools/rmes_list_graphs.py | 7 +- src/mcpdiffusion/tools/rmes_run_sparql.py | 7 +- tests/conftest.py | 26 +++-- tests/test_rmes_helpers.py | 97 ++++++++++--------- 15 files changed, 187 insertions(+), 170 deletions(-) create mode 100644 src/mcpdiffusion/infra/lifespan.py diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py index b52cbf7..974a829 100644 --- a/src/mcpdiffusion/infra/elasticsearch.py +++ b/src/mcpdiffusion/infra/elasticsearch.py @@ -1,28 +1,8 @@ -"""Centralized Elasticsearch client singleton with injected settings.""" - +"""Elasticsearch client accessor from FastMCP lifespan context.""" from __future__ import annotations -from fastmcp.server.lifespan import lifespan -from fastmcp import Context -from elasticsearch import Elasticsearch -from ..config.settings import get_settings -import logging -logger = logging.getLogger("mcp.main") - -@lifespan -async def build_es_client(server): - s = get_settings() - if not s.es_host: - raise RuntimeError("ES_HOST is not set. See .env.example.") - client = Elasticsearch( - s.es_host, verify_certs=s.tls_verify, - request_timeout=30, max_retries=2, retry_on_timeout=True, - ) - logger.info("Elasticsearch client initialized for %s", s.es_host) - yield {"es_client": client} - - await client.close() +from fastmcp import Context -def get_client_es(ctx : Context) : - return ctx.lifespan_context["es_client"] \ No newline at end of file +def get_client_es(ctx: Context): + return ctx.lifespan_context["es_client"] diff --git a/src/mcpdiffusion/infra/http.py b/src/mcpdiffusion/infra/http.py index 7e893be..890ee76 100644 --- a/src/mcpdiffusion/infra/http.py +++ b/src/mcpdiffusion/infra/http.py @@ -1,33 +1,8 @@ -"""Shared HTTP client factory with centralized TLS settings.""" +"""HTTP client accessor from FastMCP lifespan context.""" from __future__ import annotations -import httpx +from fastmcp import Context -from ..config.settings import Settings, get_settings - -_USER_AGENT = ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" -) -_DEFAULT_TIMEOUT = httpx.Timeout(30.0, connect=10.0) - - -def create_async_client( - *, - settings: Settings | None = None, - headers: dict[str, str] | None = None, - follow_redirects: bool = False, - timeout: httpx.Timeout | None = None, -) -> httpx.AsyncClient: - """Create an httpx.AsyncClient with centralized TLS and timeout settings.""" - s = settings or get_settings() - merged_headers = {"User-Agent": _USER_AGENT} - if headers: - merged_headers.update(headers) - return httpx.AsyncClient( - verify=s.tls_verify, - headers=merged_headers, - follow_redirects=follow_redirects, - timeout=timeout or _DEFAULT_TIMEOUT, - ) +def get_http_client(ctx: Context): + return ctx.lifespan_context["http_client"] diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/infra/lifespan.py new file mode 100644 index 0000000..b4cfc14 --- /dev/null +++ b/src/mcpdiffusion/infra/lifespan.py @@ -0,0 +1,60 @@ +"""Combined application lifespan: creates and tears down shared clients.""" +from __future__ import annotations + +import logging + +import httpx +from elasticsearch import Elasticsearch +from fastmcp.server.lifespan import lifespan + +from ..config.settings import get_settings + +logger = logging.getLogger("mcp.main") + +_HTTP_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" +) +_HTTP_TIMEOUT = httpx.Timeout(30.0, connect=10.0) +_SPARQL_USER_AGENT = "MCP-RMeS/2.0" + + +@lifespan +async def app_lifespan(server): + s = get_settings() + + # Elasticsearch + if not s.es_host: + raise RuntimeError("ES_HOST is not set. See .env.example.") + es_client = Elasticsearch( + s.es_host, + verify_certs=s.tls_verify, + request_timeout=30, + max_retries=2, + retry_on_timeout=True, + ) + logger.info("Elasticsearch client initialized for %s", s.es_host) + + # HTTP client (insee.fr, melodi API) + http_client = httpx.AsyncClient( + verify=s.tls_verify, + headers={"User-Agent": _HTTP_USER_AGENT}, + timeout=_HTTP_TIMEOUT, + ) + logger.info("HTTP client initialized") + + # SPARQL client + sparql_client = httpx.AsyncClient( + headers={"User-Agent": _SPARQL_USER_AGENT}, + ) + logger.info("SPARQL client initialized") + + yield { + "es_client": es_client, + "http_client": http_client, + "sparql_client": sparql_client, + } + + await es_client.close() + await http_client.aclose() + await sparql_client.aclose() diff --git a/src/mcpdiffusion/infra/sparql.py b/src/mcpdiffusion/infra/sparql.py index 55bbe52..0e9ab72 100644 --- a/src/mcpdiffusion/infra/sparql.py +++ b/src/mcpdiffusion/infra/sparql.py @@ -1,24 +1,8 @@ -"""Low-level SPARQL HTTP client for RMES.""" +"""SPARQL client accessor from FastMCP lifespan context.""" from __future__ import annotations -import httpx +from fastmcp import Context -from ..config.settings import Settings, get_settings -HEADERS_BASE = {"User-Agent": "MCP-RMeS/2.0"} - -_client: httpx.AsyncClient | None = None - - -def get_sparql_client(settings: Settings | None = None) -> httpx.AsyncClient: - """Return a shared httpx.AsyncClient for SPARQL queries, recreated if closed.""" - global _client - if _client is None or _client.is_closed: - _client = httpx.AsyncClient(headers=HEADERS_BASE) - return _client - - -def reset_sparql_client() -> None: - """Drop the cached client. Used by tests.""" - global _client - _client = None +def get_sparql_client(ctx: Context): + return ctx.lifespan_context["sparql_client"] diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index bc2c9b0..2dc96b1 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -16,14 +16,14 @@ from .core.logging import MAIN_LOGGER_NAME, UVICORN_LOGGING_CONFIG from .core.middleware import RateLimitMiddleware from .tools import register_tools -from .infra.elasticsearch import build_es_client +from .infra.lifespan import app_lifespan load_dotenv() settings = get_settings() logger = logging.getLogger(MAIN_LOGGER_NAME) -mcp = FastMCP("INSEE-mcp-diffusion", lifespan = build_es_client ) +mcp = FastMCP("INSEE-mcp-diffusion", lifespan=app_lifespan) register_tools(mcp, toollist=settings.toollist) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 2096b4e..99cb087 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -9,9 +9,10 @@ from trafilatura import extract from trafilatura.settings import Extractor +import httpx + from ..config.settings import Settings, get_settings from ..core.errors import fail -from ..infra.http import create_async_client from ..models.insee import ( DocumentResult, GetInseeDocumentInput, @@ -95,16 +96,12 @@ def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: return text[:head_size] + marker + text[-tail_size:], True -async def _fetch_html(url: str, settings: Settings) -> str: - import httpx +async def _fetch_html(url: str, settings: Settings, http_client: httpx.AsyncClient) -> str: full_url = settings.insee_base_url + url if not url.startswith(("http://", "https://")) else url try: - async with create_async_client( - settings=settings, follow_redirects=True, - ) as client: - response = await client.get(full_url) - response.raise_for_status() - return response.text + response = await http_client.get(full_url, follow_redirects=True) + response.raise_for_status() + return response.text except httpx.TimeoutException as exc: fail( "BACKEND_UNAVAILABLE", @@ -138,6 +135,7 @@ async def _fetch_html(url: str, settings: Settings) -> str: async def get_insee_document( params: GetInseeDocumentInput, *, + http_client: httpx.AsyncClient, settings: Settings | None = None, ) -> GetInseeDocumentOutput: s = settings or get_settings() @@ -152,7 +150,7 @@ async def get_insee_document( results: list[DocumentResult] = [] for url in params.list_of_url: try: - html = await _fetch_html(str(url), s) + html = await _fetch_html(str(url), s, http_client) markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" if params.truncate_content: markdown, truncated = _truncate(markdown) diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 8f77cee..e90104e 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -8,7 +8,6 @@ from elasticsearch import TransportError, Elasticsearch from ..config.settings import Settings, get_settings from ..core.errors import fail -from ..infra.http import create_async_client from ..models.melodi import ( ColumnResult, DatasetSearchResult, @@ -25,17 +24,17 @@ async def get_melodi_observations( params: GetMelodiObservationsInput, *, + http_client: httpx.AsyncClient, settings: Settings | None = None, ) -> GetMelodiObservationsOutput: s = settings or get_settings() url = f"{s.melodi_data_base_url}/{params.dataset_id}" try: - async with create_async_client(settings=s) as client: - response = await client.get( - url, - params=params.dict_of_columns_and_values or None, - ) - response.raise_for_status() + response = await http_client.get( + url, + params=params.dict_of_columns_and_values or None, + ) + response.raise_for_status() except httpx.TimeoutException as exc: fail( "BACKEND_UNAVAILABLE", diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 61a3005..a04a41e 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -13,7 +13,6 @@ import httpx from ..config.settings import Settings, get_settings -from ..infra.sparql import get_sparql_client from ..models.rmes import ( GRAPH_BASE, MAX_ROW_LIMIT, @@ -261,7 +260,7 @@ async def _execute_sparql( timeout: float, max_rows: int, *, - sparql_client_factory=None, + sparql_client: httpx.AsyncClient, settings: Settings | None = None, ) -> dict[str, Any]: s = settings or get_settings() @@ -279,7 +278,7 @@ async def _execute_sparql( accept = _accept_header(query_form) try: - client = sparql_client_factory() if sparql_client_factory else get_sparql_client(s) + client = sparql_client response = await client.post( s.rmes_endpoint, data={"query": effective_query}, @@ -338,7 +337,7 @@ async def _execute_sparql( async def _get_raw_graph_rows( *, - sparql_client_factory=None, + sparql_client: httpx.AsyncClient, settings: Settings | None = None, ) -> dict[str, Any]: now = time.time() @@ -349,7 +348,7 @@ async def _get_raw_graph_rows( ) result = await _execute_sparql( query, timeout=45.0, max_rows=1000, - sparql_client_factory=sparql_client_factory, settings=settings, + sparql_client=sparql_client, settings=settings, ) if "error" in result: return result @@ -394,11 +393,11 @@ def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: async def list_graphs( params: ListGraphsInput, *, - sparql_client_factory=None, + sparql_client: httpx.AsyncClient, settings: Settings | None = None, ) -> ListGraphsOutput: raw = await _get_raw_graph_rows( - sparql_client_factory=sparql_client_factory, settings=settings, + sparql_client=sparql_client, settings=settings, ) if "error" in raw: return ListGraphsOutput( @@ -453,7 +452,7 @@ def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[Resour async def describe_resource( params: DescribeResourceInput, *, - sparql_client_factory=None, + sparql_client: httpx.AsyncClient, settings: Settings | None = None, ) -> DescribeResourceOutput: graph_clause = f"<{params.graph}>" if params.graph else "?g" @@ -473,7 +472,7 @@ async def describe_resource( from ..models.rmes import DEFAULT_TIMEOUT result = await _execute_sparql( query, timeout=DEFAULT_TIMEOUT, max_rows=MAX_ROW_LIMIT, - sparql_client_factory=sparql_client_factory, settings=settings, + sparql_client=sparql_client, settings=settings, ) if "error" in result: @@ -488,7 +487,7 @@ async def describe_resource( async def run_sparql( params: RunSparqlInput, *, - sparql_client_factory=None, + sparql_client: httpx.AsyncClient, settings: Settings | None = None, ) -> RunSparqlOutput: if not params.full_sparql_query or not params.full_sparql_query.strip(): @@ -503,7 +502,7 @@ async def run_sparql( max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) result = await _execute_sparql( params.full_sparql_query, timeout=params.timeout, max_rows=max_rows, - sparql_client_factory=sparql_client_factory, settings=settings, + sparql_client=sparql_client, settings=settings, ) if "error" in result: diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 82092f7..9673eff 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -1,10 +1,11 @@ """Tool: get_insee_document -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import GET_DOCUMENT from ..core.logging import log_tool +from ..infra.http import get_http_client from ..models.insee import GetInseeDocumentInput, GetInseeDocumentOutput from ..services.insee_document import get_insee_document @@ -18,5 +19,6 @@ def register_get_insee_document(mcp: FastMCP) -> None: @log_tool async def get_insee_documents( params: GetInseeDocumentInput, + ctx: Context, ) -> GetInseeDocumentOutput: - return await get_insee_document(params) + return await get_insee_document(params, http_client=get_http_client(ctx)) diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index e349ac7..a792205 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -1,10 +1,11 @@ """Tool: get_melodi_observations -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import GET_DATASET from ..core.logging import log_tool +from ..infra.http import get_http_client from ..models.melodi import GetMelodiObservationsInput, GetMelodiObservationsOutput from ..services.melodi import get_melodi_observations @@ -18,5 +19,6 @@ def register_get_melodi_observations(mcp: FastMCP) -> None: @log_tool async def get_melodi_observations_tool( params: GetMelodiObservationsInput, + ctx: Context, ) -> GetMelodiObservationsOutput: - return await get_melodi_observations(params) + return await get_melodi_observations(params, http_client=get_http_client(ctx)) diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index f68de2f..72c2710 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -1,10 +1,11 @@ """Tool: RMES_describe_resource -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_DESCRIBE_RESOURCE from ..core.logging import log_tool +from ..infra.sparql import get_sparql_client from ..models.rmes import DescribeResourceInput, DescribeResourceOutput from ..services.rmes import describe_resource @@ -16,5 +17,5 @@ def register_rmes_describe_resource(mcp: FastMCP) -> None: meta=RMES_DESCRIBE_RESOURCE["tool_metadata"], ) @log_tool - async def describe_resource_tool(params: DescribeResourceInput) -> DescribeResourceOutput: - return await describe_resource(params) + async def describe_resource_tool(params: DescribeResourceInput, ctx: Context) -> DescribeResourceOutput: + return await describe_resource(params, sparql_client=get_sparql_client(ctx)) diff --git a/src/mcpdiffusion/tools/rmes_list_graphs.py b/src/mcpdiffusion/tools/rmes_list_graphs.py index 11d48a1..ec4d6b6 100644 --- a/src/mcpdiffusion/tools/rmes_list_graphs.py +++ b/src/mcpdiffusion/tools/rmes_list_graphs.py @@ -1,10 +1,11 @@ """Tool: RMES_list_graphs -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_LIST_GRAPHS from ..core.logging import log_tool +from ..infra.sparql import get_sparql_client from ..models.rmes import ListGraphsInput, ListGraphsOutput from ..services.rmes import list_graphs @@ -16,5 +17,5 @@ def register_rmes_list_graphs(mcp: FastMCP) -> None: meta=RMES_LIST_GRAPHS["tool_metadata"], ) @log_tool - async def list_graphs_tool(params: ListGraphsInput) -> ListGraphsOutput: - return await list_graphs(params) + async def list_graphs_tool(params: ListGraphsInput, ctx: Context) -> ListGraphsOutput: + return await list_graphs(params, sparql_client=get_sparql_client(ctx)) diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index 175a2ae..8e9c534 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -1,10 +1,11 @@ """Tool: RMES_run_sparql -- thin registration layer.""" from __future__ import annotations -from fastmcp import FastMCP +from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_RUN_SPARQL from ..core.logging import log_tool +from ..infra.sparql import get_sparql_client from ..models.rmes import RunSparqlInput, RunSparqlOutput from ..services.rmes import KNOWN_VOCABULARIES_NOTE, run_sparql @@ -28,5 +29,5 @@ def register_rmes_run_sparql(mcp: FastMCP) -> None: meta=RMES_RUN_SPARQL["tool_metadata"], ) @log_tool - async def run_sparql_tool(params: RunSparqlInput) -> RunSparqlOutput: - return await run_sparql(params) + async def run_sparql_tool(params: RunSparqlInput, ctx: Context) -> RunSparqlOutput: + return await run_sparql(params, sparql_client=get_sparql_client(ctx)) diff --git a/tests/conftest.py b/tests/conftest.py index eba2b23..5a571b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import httpx import pytest from fastmcp import Client, FastMCP +from fastmcp.server.lifespan import lifespan from mcpdiffusion.config.settings import get_settings from mcpdiffusion.services import rmes as rmes_service @@ -57,9 +58,9 @@ def _out(call_tool_result) -> dict[str, Any]: # --------------------------------------------------------------------------- class FakeAsyncClient: - """Drop-in replacement for httpx.AsyncClient used by sparql.get_sparql_client().""" + """Drop-in replacement for httpx.AsyncClient.""" - def __init__(self, handler): + def __init__(self, handler=None): self.handler = handler self.is_closed = False @@ -75,9 +76,21 @@ async def post(self, url, **kwargs): # --------------------------------------------------------------------------- @pytest.fixture -def rmes_mcp() -> FastMCP: +def _fake_sparql_client(): + """Shared FakeAsyncClient whose handler is set by mock_sparql.""" + return FakeAsyncClient() + + +@pytest.fixture +def rmes_mcp(_fake_sparql_client) -> FastMCP: """Return a FastMCP instance with only the three RMES tools registered.""" - mcp = FastMCP("test-rmes") + client = _fake_sparql_client + + @lifespan + async def test_lifespan(server): + yield {"sparql_client": client} + + mcp = FastMCP("test-rmes", lifespan=test_lifespan) register_rmes_list_graphs(mcp) register_rmes_describe_resource(mcp) register_rmes_run_sparql(mcp) @@ -95,13 +108,12 @@ def rmes_client(rmes_mcp: FastMCP) -> Client: # --------------------------------------------------------------------------- @pytest.fixture -def mock_sparql(monkeypatch): +def mock_sparql(_fake_sparql_client): """Return a callable that sets up the fake SPARQL endpoint.""" rmes_service._GRAPH_CACHE["data"] = None rmes_service._GRAPH_CACHE["ts"] = 0.0 def _setup(handler): - fake = FakeAsyncClient(handler) - monkeypatch.setattr(rmes_service, "get_sparql_client", lambda *a, **kw: fake) + _fake_sparql_client.handler = handler return _setup diff --git a/tests/test_rmes_helpers.py b/tests/test_rmes_helpers.py index cb8f794..0ffb7f7 100644 --- a/tests/test_rmes_helpers.py +++ b/tests/test_rmes_helpers.py @@ -7,7 +7,6 @@ import pytest from mcpdiffusion.models.rmes import GRAPH_BASE, SparqlErrorType -from mcpdiffusion.services import rmes as rmes_service from mcpdiffusion.services.rmes import ( _CATEGORY_AUTRE, _GRAPH_CACHE, @@ -246,98 +245,102 @@ def test_extra_fields(self): # _execute_sparql (async, mocked HTTP via DI) # =================================================================== -@pytest.fixture -def mock_http(monkeypatch): - """Patch get_sparql_client where it is used (services.rmes namespace).""" - def _setup(handler): - fake = FakeAsyncClient(handler) - monkeypatch.setattr(rmes_service, "get_sparql_client", lambda *a, **kw: fake) - return _setup - - class TestExecuteSparql: - async def test_unknown_form_returns_error_without_http_call(self, mock_http): + async def test_unknown_form_returns_error_without_http_call(self): called = [] - mock_http(lambda url, **kw: called.append(1) or _json_response({})) + fake = FakeAsyncClient(lambda url, **kw: called.append(1) or _json_response({})) - result = await _execute_sparql("INSERT DATA {

}", timeout=10, max_rows=100) + result = await _execute_sparql( + "INSERT DATA {

}", timeout=10, max_rows=100, sparql_client=fake, + ) assert "error" in result assert result["error"]["type"] == SparqlErrorType.INVALID_QUERY_FORM assert len(called) == 0 - async def test_select_success(self, mock_http): + async def test_select_success(self): body = {"head": {"vars": ["x"]}, "results": {"bindings": []}} - mock_http(lambda url, **kw: _json_response(body)) + fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) - result = await _execute_sparql("SELECT ?x WHERE { ?x ?p ?o } LIMIT 1", timeout=10, max_rows=100) + result = await _execute_sparql( + "SELECT ?x WHERE { ?x ?p ?o } LIMIT 1", timeout=10, max_rows=100, sparql_client=fake, + ) assert "error" not in result assert result["head"]["vars"] == ["x"] - async def test_select_without_limit_adds_meta(self, mock_http): + async def test_select_without_limit_adds_meta(self): body = {"head": {"vars": ["x"]}, "results": {"bindings": []}} - mock_http(lambda url, **kw: _json_response(body)) + fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) - result = await _execute_sparql("SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=50) + result = await _execute_sparql( + "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=50, sparql_client=fake, + ) assert result["_meta"]["limit_added"] == 50 assert "hint" in result["_meta"] - async def test_construct_returns_turtle(self, mock_http): + async def test_construct_returns_turtle(self): turtle = " ." - mock_http(lambda url, **kw: httpx.Response( + fake = FakeAsyncClient(lambda url, **kw: httpx.Response( 200, content=turtle.encode(), headers={"content-type": "text/turtle"}, request=httpx.Request("POST", url), )) result = await _execute_sparql( "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1", timeout=10, max_rows=100, + sparql_client=fake, ) assert result["format"] == "turtle" assert result["data"] == turtle - async def test_timeout_returns_error(self, mock_http): + async def test_timeout_returns_error(self): def handler(url, **kw): raise httpx.TimeoutException("timed out") - mock_http(handler) + fake = FakeAsyncClient(handler) - result = await _execute_sparql("SELECT ?x WHERE { ?x ?p ?o }", timeout=5, max_rows=100) + result = await _execute_sparql( + "SELECT ?x WHERE { ?x ?p ?o }", timeout=5, max_rows=100, sparql_client=fake, + ) assert result["error"]["type"] == SparqlErrorType.TIMEOUT - async def test_http_400_returns_syntax_error(self, mock_http): + async def test_http_400_returns_syntax_error(self): def handler(url, **kw): resp = httpx.Response(400, content=b"Parse error", request=httpx.Request("POST", url)) raise httpx.HTTPStatusError("Bad Request", request=resp.request, response=resp) - mock_http(handler) + fake = FakeAsyncClient(handler) - result = await _execute_sparql("SELECT bad", timeout=10, max_rows=100) + result = await _execute_sparql("SELECT bad", timeout=10, max_rows=100, sparql_client=fake) assert result["error"]["type"] == SparqlErrorType.SYNTAX_ERROR assert "endpoint_message" in result["error"] - async def test_http_500_returns_http_error(self, mock_http): + async def test_http_500_returns_http_error(self): def handler(url, **kw): resp = httpx.Response(500, content=b"Internal error", request=httpx.Request("POST", url)) raise httpx.HTTPStatusError("Server Error", request=resp.request, response=resp) - mock_http(handler) + fake = FakeAsyncClient(handler) - result = await _execute_sparql("SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100) + result = await _execute_sparql( + "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100, sparql_client=fake, + ) assert result["error"]["type"] == SparqlErrorType.HTTP_ERROR - async def test_network_error_returns_network_error(self, mock_http): + async def test_network_error_returns_network_error(self): def handler(url, **kw): raise httpx.ConnectError("connection refused") - mock_http(handler) + fake = FakeAsyncClient(handler) - result = await _execute_sparql("SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100) + result = await _execute_sparql( + "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100, sparql_client=fake, + ) assert result["error"]["type"] == SparqlErrorType.NETWORK_ERROR @@ -353,33 +356,33 @@ def reset_cache(self): _GRAPH_CACHE["data"] = None _GRAPH_CACHE["ts"] = 0.0 - async def test_returns_rows_on_success(self, mock_http): + async def test_returns_rows_on_success(self): body = { "head": {"vars": ["g", "nbTriples"]}, "results": {"bindings": [ {"g": {"value": "http://rdf.insee.fr/graphes/codes/naf2025"}, "nbTriples": {"value": "100"}}, ]}, } - mock_http(lambda url, **kw: _json_response(body)) + fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) - result = await _get_raw_graph_rows() + result = await _get_raw_graph_rows(sparql_client=fake) assert "rows" in result assert len(result["rows"]) == 1 assert result["rows"][0]["graph"] == "http://rdf.insee.fr/graphes/codes/naf2025" assert result["rows"][0]["triples"] == 100 - async def test_returns_error_on_failure(self, mock_http): + async def test_returns_error_on_failure(self): def handler(url, **kw): raise httpx.TimeoutException("timed out") - mock_http(handler) + fake = FakeAsyncClient(handler) - result = await _get_raw_graph_rows() + result = await _get_raw_graph_rows(sparql_client=fake) assert "error" in result - async def test_uses_cache_on_second_call(self, mock_http): + async def test_uses_cache_on_second_call(self): call_count = [] body = { "head": {"vars": ["g", "nbTriples"]}, @@ -392,15 +395,15 @@ def handler(url, **kw): call_count.append(1) return _json_response(body) - mock_http(handler) + fake = FakeAsyncClient(handler) - result1 = await _get_raw_graph_rows() - result2 = await _get_raw_graph_rows() + result1 = await _get_raw_graph_rows(sparql_client=fake) + result2 = await _get_raw_graph_rows(sparql_client=fake) assert result1 == result2 assert len(call_count) == 1 - async def test_cache_expires_after_ttl(self, mock_http, monkeypatch): + async def test_cache_expires_after_ttl(self): body = { "head": {"vars": ["g", "nbTriples"]}, "results": {"bindings": [ @@ -413,12 +416,12 @@ def handler(url, **kw): call_count.append(1) return _json_response(body) - mock_http(handler) + fake = FakeAsyncClient(handler) - await _get_raw_graph_rows() + await _get_raw_graph_rows(sparql_client=fake) assert len(call_count) == 1 _GRAPH_CACHE["ts"] = time.time() - _GRAPH_CACHE_TTL - 1 - await _get_raw_graph_rows() + await _get_raw_graph_rows(sparql_client=fake) assert len(call_count) == 2 From daae83de302e97c0c442f1da3a87a52e35840066 Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Fri, 21 Aug 2026 10:13:42 +0200 Subject: [PATCH 04/55] feat/test on other services --- src/mcpdiffusion/services/insee_search.py | 3 + tests/conftest.py | 8 +- tests/test_feedback_service.py | 74 ++++ tests/test_insee_document_service.py | 281 ++++++++++++++ tests/test_insee_search_service.py | 179 +++++++++ tests/test_melodi_service.py | 349 ++++++++++++++++++ ...t_rmes_helpers.py => test_rmes_service.py} | 0 7 files changed, 893 insertions(+), 1 deletion(-) create mode 100644 tests/test_feedback_service.py create mode 100644 tests/test_insee_document_service.py create mode 100644 tests/test_insee_search_service.py create mode 100644 tests/test_melodi_service.py rename tests/{test_rmes_helpers.py => test_rmes_service.py} (100%) diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index 83b1f8d..e62d254 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -24,6 +24,8 @@ def _coerce_hit_value(value) -> Optional[str]: return str(value) +# Build query + def build_text_clauses( query: Optional[str], year_of_reference: Optional[int], @@ -126,6 +128,7 @@ def apply_collection_filters( return filters, should +# Execute search with built query def execute_search( *, diff --git a/tests/conftest.py b/tests/conftest.py index 5a571b0..1677541 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,12 +64,18 @@ def __init__(self, handler=None): self.handler = handler self.is_closed = False - async def post(self, url, **kwargs): + async def _request(self, url, **kwargs): resp = self.handler(url, **kwargs) if resp.status_code >= 400: resp.raise_for_status() return resp + async def get(self, url, **kwargs): + return await self._request(url, **kwargs) + + async def post(self, url, **kwargs): + return await self._request(url, **kwargs) + # --------------------------------------------------------------------------- # Fixtures: RMES server & client diff --git a/tests/test_feedback_service.py b/tests/test_feedback_service.py new file mode 100644 index 0000000..acd7668 --- /dev/null +++ b/tests/test_feedback_service.py @@ -0,0 +1,74 @@ +"""Unit tests for mcpdiffusion.services.feedback.""" +from __future__ import annotations + +from mcpdiffusion.models.feedback import SendFeedbackInput +from mcpdiffusion.services.feedback import _ensure_feedback_file, send_feedback + + +class TestEnsureFeedbackFile: + def test_creates_file_if_missing(self, tmp_path, monkeypatch): + feedback_dir = tmp_path / "feedback" + feedback_file = feedback_dir / "feedback.md" + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_DIR", feedback_dir) + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_FILE", feedback_file) + + result = _ensure_feedback_file() + + assert result == feedback_file + assert feedback_file.exists() + content = feedback_file.read_text(encoding="utf-8") + assert "# Feedback Log" in content + + def test_does_not_overwrite_existing_file(self, tmp_path, monkeypatch): + feedback_dir = tmp_path / "feedback" + feedback_dir.mkdir() + feedback_file = feedback_dir / "feedback.md" + feedback_file.write_text("existing content", encoding="utf-8") + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_DIR", feedback_dir) + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_FILE", feedback_file) + + _ensure_feedback_file() + + assert feedback_file.read_text(encoding="utf-8") == "existing content" + + +class TestSendFeedback: + async def test_returns_success(self, tmp_path, monkeypatch): + feedback_dir = tmp_path / "feedback" + feedback_file = feedback_dir / "feedback.md" + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_DIR", feedback_dir) + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_FILE", feedback_file) + + params = SendFeedbackInput(username="alice", feedback="Great tool!") + result = await send_feedback(params) + + assert result.status == "success" + assert result.message == "Feedback recorded successfully." + assert result.timestamp + + async def test_appends_entry_to_file(self, tmp_path, monkeypatch): + feedback_dir = tmp_path / "feedback" + feedback_file = feedback_dir / "feedback.md" + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_DIR", feedback_dir) + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_FILE", feedback_file) + + await send_feedback(SendFeedbackInput(username="alice", feedback="First")) + + content = feedback_file.read_text(encoding="utf-8") + assert "alice" in content + assert "First" in content + + async def test_multiple_entries_appended(self, tmp_path, monkeypatch): + feedback_dir = tmp_path / "feedback" + feedback_file = feedback_dir / "feedback.md" + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_DIR", feedback_dir) + monkeypatch.setattr("mcpdiffusion.services.feedback._FEEDBACK_FILE", feedback_file) + + await send_feedback(SendFeedbackInput(username="alice", feedback="First")) + await send_feedback(SendFeedbackInput(username="bob", feedback="Second")) + + content = feedback_file.read_text(encoding="utf-8") + assert "alice" in content + assert "bob" in content + assert "First" in content + assert "Second" in content diff --git a/tests/test_insee_document_service.py b/tests/test_insee_document_service.py new file mode 100644 index 0000000..e1a2955 --- /dev/null +++ b/tests/test_insee_document_service.py @@ -0,0 +1,281 @@ +"""Unit tests for mcpdiffusion.services.insee_document.""" +from __future__ import annotations + +import httpx +import pytest +from fastmcp.exceptions import ToolError +from mcpdiffusion.core.errors import fail + +from mcpdiffusion.config.settings import Settings +from mcpdiffusion.models.insee import GetInseeDocumentInput +from mcpdiffusion.services.insee_document import ( + _as_relative, + _format_sommaire, + _parse_sommaire, + _truncate, + _fetch_html, + get_insee_document, +) +from tests.conftest import FakeAsyncClient + +_SETTINGS = Settings(INSEE_BASE_URL="https://www.insee.fr", _env_file=None) + + +# =================================================================== +# _as_relative +# =================================================================== + +class TestAsRelative: + def test_path_only(self): + assert _as_relative("https://www.insee.fr/fr/statistiques/123") == "/fr/statistiques/123" + + def test_with_query_string(self): + result = _as_relative("https://www.insee.fr/fr/statistiques/123?sommaire=456") + assert result == "/fr/statistiques/123?sommaire=456" + + def test_already_relative(self): + assert _as_relative("/fr/statistiques/123") == "/fr/statistiques/123" + + +# =================================================================== +# _truncate +# =================================================================== + +class TestTruncate: + def test_short_text_not_truncated(self): + text, truncated = _truncate("Short text") + assert text == "Short text" + assert truncated is False + + def test_exact_limit_not_truncated(self): + text = "x" * 1000 + result, truncated = _truncate(text, limit=1000) + assert truncated is False + assert result == text + + def test_long_text_truncated(self): + text = "x" * 5000 + result, truncated = _truncate(text, limit=1000) + assert truncated is True + assert len(result) < len(text) + assert "CONTENT TRUNCATED" in result + + def test_preserves_head_and_tail(self): + text = "HEAD" + "x" * 5000 + "TAIL" + result, truncated = _truncate(text, limit=1000) + assert truncated is True + assert result.startswith("HEAD") + assert result.endswith("TAIL") + + +# =================================================================== +# _parse_sommaire +# =================================================================== + +class TestParseSommaire: + def test_empty_html_returns_empty(self): + assert _parse_sommaire("", "https://www.insee.fr") == [] + + def test_no_sommaire_section_returns_empty(self): + html = "

Content
" + assert _parse_sommaire(html, "https://www.insee.fr") == [] + + def test_parses_categorized_links(self): + html = """ + +
+ +
+ + """ + result = _parse_sommaire(html, "https://www.insee.fr") + assert len(result) == 2 + assert result[0]["category"] == "Category A" + assert result[0]["title"] == "Link 1" + assert result[0]["url"] == "/fr/stat/1" + assert result[1]["title"] == "Link 2" + + def test_parses_uncategorized_links(self): + html = """ + +
+ +
+ + """ + result = _parse_sommaire(html, "https://www.insee.fr") + assert len(result) == 1 + assert result[0]["category"] == "" + assert result[0]["title"] == "Direct Link" + + def test_no_ul_inside_sommaire_returns_empty(self): + html = """ + +

No list here

+ + """ + assert _parse_sommaire(html, "https://www.insee.fr") == [] + + +# =================================================================== +# _format_sommaire +# =================================================================== + +class TestFormatSommaire: + def test_groups_by_category(self): + items = [ + {"category": "A", "title": "T1", "url": "/1"}, + {"category": "A", "title": "T2", "url": "/2"}, + {"category": "B", "title": "T3", "url": "/3"}, + ] + result = _format_sommaire(items) + assert result == {"A": {"T1": "/1", "T2": "/2"}, "B": {"T3": "/3"}} + + def test_empty_list(self): + assert _format_sommaire([]) == {} + + def test_empty_category(self): + items = [{"category": "", "title": "T1", "url": "/1"}] + result = _format_sommaire(items) + assert result == {"": {"T1": "/1"}} + + +# =================================================================== +# _fetch_html +# =================================================================== + +class TestFetchHtml: + async def test_success_returns_html(self): + fake = FakeAsyncClient(lambda url, **kw: httpx.Response( + 200, content=b"OK", request=httpx.Request("GET", url), + )) + result = await _fetch_html("/fr/stat/1", _SETTINGS, fake) + assert result == "OK" + + async def test_prepends_base_url_for_relative_path(self): + captured = [] + + def handler(url, **kw): + captured.append(url) + return httpx.Response(200, content=b"ok", request=httpx.Request("GET", url)) + + await _fetch_html("/fr/stat/1", _SETTINGS, FakeAsyncClient(handler)) + assert captured[0] == "https://www.insee.fr/fr/stat/1" + + async def test_absolute_url_not_modified(self): + captured = [] + + def handler(url, **kw): + captured.append(url) + return httpx.Response(200, content=b"ok", request=httpx.Request("GET", url)) + + await _fetch_html("https://other.fr/page", _SETTINGS, FakeAsyncClient(handler)) + assert captured[0] == "https://other.fr/page" + + async def test_timeout_raises_tool_error(self): + def handler(url, **kw): + raise httpx.TimeoutException("timed out") + + with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): + await _fetch_html("/fr/stat/1", _SETTINGS, FakeAsyncClient(handler)) + + async def test_404_raises_tool_error(self): + def handler(url, **kw): + resp = httpx.Response(404, content=b"Not Found", request=httpx.Request("GET", url)) + raise httpx.HTTPStatusError("Not Found", request=resp.request, response=resp) + + with pytest.raises(ToolError, match="NOT_FOUND"): + await _fetch_html("/fr/stat/1", _SETTINGS, FakeAsyncClient(handler)) + + async def test_500_raises_tool_error(self): + def handler(url, **kw): + resp = httpx.Response(500, content=b"Error", request=httpx.Request("GET", url)) + raise httpx.HTTPStatusError("Error", request=resp.request, response=resp) + + with pytest.raises(ToolError, match="UPSTREAM_ERROR"): + await _fetch_html("/fr/stat/1", _SETTINGS, FakeAsyncClient(handler)) + + +# =================================================================== +# get_insee_document +# =================================================================== + +class TestGetInseeDocument: + async def test_empty_url_list_raises(self): + params = GetInseeDocumentInput(list_of_url=[]) + with pytest.raises(ToolError, match="INVALID_INPUT"): + await get_insee_document(params, http_client=FakeAsyncClient(), settings=_SETTINGS) + + async def test_fetch_error_returns_error_result(self): + def handler(url, **kw): + raise httpx.TimeoutException("timed out") + + params = GetInseeDocumentInput(list_of_url=["/fr/stat/1"]) + result = await get_insee_document( + params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + ) + + assert result.count == 1 + assert result.results[0].status == "error" + assert "ToolError" in result.results[0].error + + async def test_success_returns_markdown(self): + html = "

Important paragraph.

" + fake = FakeAsyncClient(lambda url, **kw: httpx.Response( + 200, content=html.encode(), request=httpx.Request("GET", url), + )) + params = GetInseeDocumentInput( + list_of_url=["/fr/stat/1"], + include_sommaire=False, + truncate_content=False, + ) + result = await get_insee_document(params, http_client=fake, settings=_SETTINGS) + + assert result.count == 1 + assert result.results[0].status == "success" + assert result.results[0].error is None + + async def test_multiple_urls(self): + fake = FakeAsyncClient(lambda url, **kw: httpx.Response( + 200, content=b"

Content

", + request=httpx.Request("GET", url), + )) + params = GetInseeDocumentInput( + list_of_url=["/fr/stat/1", "/fr/stat/2"], + include_sommaire=False, + ) + result = await get_insee_document(params, http_client=fake, settings=_SETTINGS) + assert result.count == 2 + + async def test_mixed_success_and_error(self): + call_count = [0] + + def handler(url, **kw): + call_count[0] += 1 + if call_count[0] == 1: + return httpx.Response( + 200, content=b"

OK

", + request=httpx.Request("GET", url), + ) + raise httpx.TimeoutException("timed out") + + params = GetInseeDocumentInput( + list_of_url=["/fr/stat/ok", "/fr/stat/fail"], + include_sommaire=False, + ) + result = await get_insee_document( + params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + ) + assert result.count == 2 + assert result.results[0].status == "success" + assert result.results[1].status == "error" diff --git a/tests/test_insee_search_service.py b/tests/test_insee_search_service.py new file mode 100644 index 0000000..5d82bbc --- /dev/null +++ b/tests/test_insee_search_service.py @@ -0,0 +1,179 @@ +"""Unit tests for mcpdiffusion.services.insee_search (pure logic, no ES).""" +from __future__ import annotations + +from mcpdiffusion.services.insee_search import ( + _coerce_hit_value, + apply_collection_filters, + build_text_clauses, +) + + +# =================================================================== +# _coerce_hit_value +# =================================================================== + +class TestCoerceHitValue: + def test_none_returns_none(self): + assert _coerce_hit_value(None) is None + + def test_string_passthrough(self): + assert _coerce_hit_value("hello") == "hello" + + def test_integer_coerced_to_string(self): + assert _coerce_hit_value(42) == "42" + + def test_list_joined(self): + assert _coerce_hit_value(["a", "b", "c"]) == "a, b, c" + + def test_empty_list_returns_none(self): + assert _coerce_hit_value([]) is None + + def test_single_element_list(self): + assert _coerce_hit_value(["only"]) == "only" + + +# =================================================================== +# build_text_clauses +# =================================================================== + +class TestBuildTextClauses: + def test_no_arguments_returns_empty_lists(self): + must, filters, should, must_not = build_text_clauses(None, None) + assert must == [] + assert filters == [] + assert should == [] + assert must_not == [] + + def test_query_adds_must_and_should(self): + must, filters, should, must_not = build_text_clauses("population", None) + assert len(must) == 1 + assert len(should) == 1 + + def test_year_adds_filter(self): + must, filters, should, must_not = build_text_clauses(None, 2024) + assert must == [] + assert len(filters) == 1 + + def test_query_and_year_combined(self): + must, filters, should, must_not = build_text_clauses("PIB", 2023) + assert len(must) == 1 + assert len(filters) == 1 + + def test_keywords_add_should_clauses(self): + must, filters, should, must_not = build_text_clauses( + None, None, keywords=["eco", "stats"], + ) + assert len(should) == 2 + + def test_empty_keywords_ignored(self): + must, filters, should, must_not = build_text_clauses(None, None, keywords=[]) + assert should == [] + + def test_query_with_keywords(self): + must, filters, should, must_not = build_text_clauses( + "chomage", None, keywords=["emploi"], + ) + assert len(must) == 1 + assert len(should) == 2 # match_phrase + keyword + + +# =================================================================== +# apply_collection_filters +# =================================================================== + +class TestApplyCollectionFilters: + def test_must_only_rapides(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=True, + ) + assert len(filters) == 1 + + def test_must_not_rapides(self): + filters, should = apply_collection_filters( + [], must_not_rapides=True, must_only_rapides=False, + ) + assert len(filters) == 1 + + def test_no_rapides_filter_when_both_false(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, + ) + assert filters == [] + assert should == [] + + def test_chiffre_clef_adds_filter(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, chiffre_clef=True, + ) + assert len(filters) == 1 + + def test_valid_theme_adds_filter(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, theme="Demographie", + ) + assert len(filters) == 1 + + def test_theme_all_ignored(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, theme="ALL", + ) + assert filters == [] + + def test_unknown_theme_ignored(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, theme="NotATheme", + ) + assert filters == [] + + def test_valid_geo_niveau(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, geo_niveau="COMMUNE", + ) + assert len(filters) == 1 + + def test_unknown_geo_niveau_ignored(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, geo_niveau="MARS", + ) + assert filters == [] + + def test_geo_keyword_adds_two_should_clauses(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, geo_keyword="Paris", + ) + assert len(should) == 2 + + def test_geo_keyword_all_ignored(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, geo_keyword="all", + ) + assert should == [] + + def test_geo_keyword_all_case_insensitive(self): + filters, should = apply_collection_filters( + [], must_not_rapides=False, must_only_rapides=False, geo_keyword="ALL", + ) + assert should == [] + + def test_preserves_existing_filters(self): + initial = [{"existing": True}] + filters, should = apply_collection_filters( + initial, + must_not_rapides=True, + must_only_rapides=False, + chiffre_clef=True, + ) + assert len(filters) == 3 # existing + not_rapides + chiffre_clef + + def test_combined_filters(self): + filters, should = apply_collection_filters( + [], + must_not_rapides=True, + must_only_rapides=False, + chiffre_clef=True, + theme="Demographie", + geo_niveau="DEPARTEMENT", + geo_keyword="Bretagne", + ) + assert len(filters) == 4 # not_rapides + chiffre_clef + theme + geo_niveau + assert len(should) == 2 # geo_keyword multi_match + match_phrase diff --git a/tests/test_melodi_service.py b/tests/test_melodi_service.py new file mode 100644 index 0000000..a27885c --- /dev/null +++ b/tests/test_melodi_service.py @@ -0,0 +1,349 @@ +"""Unit tests for mcpdiffusion.services.melodi.""" +from __future__ import annotations + +import json + +import httpx +import pytest +from elasticsearch import ConnectionError as ESConnectionError +from fastmcp.exceptions import ToolError + +from mcpdiffusion.config.settings import Settings +from mcpdiffusion.models.melodi import ( + GetMelodiObservationsInput, + SearchMelodiDatasetsInput, + SearchMelodiModalitiesInput, +) +from mcpdiffusion.services.melodi import ( + get_melodi_observations, + search_melodi_datasets, + search_melodi_modalities, +) +from tests.conftest import FakeAsyncClient + +_SETTINGS = Settings( + MELODI_DATA_BASE_URL="https://api.insee.fr/melodi/data", + ES_INDEX_MELODI_DATASETS="melodi_datasets", + ES_INDEX_MELODI_COLUMNS="melodi_columns", + _env_file=None, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _json_http_response(payload: dict, url: str = "https://api.test") -> httpx.Response: + return httpx.Response( + 200, + content=json.dumps(payload).encode(), + headers={"content-type": "application/json"}, + request=httpx.Request("GET", url), + ) + + +class FakeElasticsearch: + """Minimal mock for Elasticsearch.search().""" + + def __init__(self, response=None, error=None): + self.response = response + self.error = error + + def search(self, **kwargs): + if self.error: + raise self.error + return self.response + + +# =================================================================== +# get_melodi_observations +# =================================================================== + +class TestGetMelodiObservations: + async def test_success(self): + payload = {"observations": [{"v": 1}, {"v": 2}, {"v": 3}]} + fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) + params = GetMelodiObservationsInput(dataset_id="DS_TEST") + + result = await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + assert result.dataset_id == "DS_TEST" + assert result.count == 3 + + async def test_year_filtering(self): + payload = {"observations": [ + {"dimensions": {"TIME_PERIOD": "2020-01"}, "v": 1}, + {"dimensions": {"TIME_PERIOD": "2021-06"}, "v": 2}, + {"dimensions": {"TIME_PERIOD": "2022-12"}, "v": 3}, + ]} + fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) + params = GetMelodiObservationsInput( + dataset_id="DS_TEST", list_of_year=[2020, 2022], + ) + + result = await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + assert result.count == 2 + + async def test_number_of_results_limits_output(self): + payload = {"observations": [{"v": i} for i in range(50)]} + fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) + params = GetMelodiObservationsInput(dataset_id="DS_TEST", number_of_results=5) + + result = await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + assert result.count == 5 + + async def test_timeout_raises_tool_error(self): + def handler(url, **kw): + raise httpx.TimeoutException("timeout") + + params = GetMelodiObservationsInput(dataset_id="DS_TEST") + with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): + await get_melodi_observations( + params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + ) + + async def test_404_raises_tool_error(self): + def handler(url, **kw): + resp = httpx.Response(404, content=b"Not Found", request=httpx.Request("GET", url)) + raise httpx.HTTPStatusError("Not Found", request=resp.request, response=resp) + + params = GetMelodiObservationsInput(dataset_id="DS_NONEXIST") + with pytest.raises(ToolError, match="NOT_FOUND"): + await get_melodi_observations( + params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + ) + + async def test_400_raises_tool_error(self): + def handler(url, **kw): + resp = httpx.Response(400, content=b"Bad Request", request=httpx.Request("GET", url)) + raise httpx.HTTPStatusError("Bad", request=resp.request, response=resp) + + params = GetMelodiObservationsInput(dataset_id="DS_TEST") + with pytest.raises(ToolError, match="INVALID_INPUT"): + await get_melodi_observations( + params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + ) + + async def test_non_json_response_raises(self): + fake = FakeAsyncClient(lambda url, **kw: httpx.Response( + 200, content=b"not json", + headers={"content-type": "text/plain"}, + request=httpx.Request("GET", url), + )) + params = GetMelodiObservationsInput(dataset_id="DS_TEST") + with pytest.raises(ToolError, match="PARSE_ERROR"): + await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + async def test_missing_observations_key_raises(self): + payload = {"data": []} + fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) + params = GetMelodiObservationsInput(dataset_id="DS_TEST") + with pytest.raises(ToolError, match="PARSE_ERROR"): + await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + async def test_empty_year_filter_returns_all(self): + payload = {"observations": [{"v": 1}, {"v": 2}]} + fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) + params = GetMelodiObservationsInput(dataset_id="DS_TEST", list_of_year=[]) + + result = await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) + + assert result.count == 2 + + +# =================================================================== +# search_melodi_datasets +# =================================================================== + +class TestSearchMelodiDatasets: + async def test_success(self): + es_response = { + "hits": {"hits": [{ + "_id": "DS_IPC", + "_score": 10.5, + "_source": { + "columns": "COL1 Label1 | COL2 Label2", + "metadata": { + "description": {"content": "Price index", "lang": "fr"}, + }, + }, + }]}, + } + params = SearchMelodiDatasetsInput(french_query="prix") + + result = await search_melodi_datasets( + params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + ) + + assert len(result.results) == 1 + assert result.results[0].dataset_id == "DS_IPC" + assert result.results[0].dataset_score == 10.5 + + async def test_empty_results(self): + es = FakeElasticsearch(response={"hits": {"hits": []}}) + params = SearchMelodiDatasetsInput(french_query="nonexistent") + + result = await search_melodi_datasets(params, es=es, settings=_SETTINGS) + + assert result.results == [] + + async def test_es_connection_error_raises(self): + es = FakeElasticsearch(error=ESConnectionError("connection refused")) + params = SearchMelodiDatasetsInput(french_query="prix") + + with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): + await search_melodi_datasets(params, es=es, settings=_SETTINGS) + + async def test_description_list_takes_first(self): + es_response = { + "hits": {"hits": [{ + "_id": "DS_1", "_score": 1.0, + "_source": { + "columns": "", + "metadata": { + "description": [ + {"content": "First", "lang": "fr"}, + {"content": "Second", "lang": "en"}, + ], + }, + }, + }]}, + } + result = await search_melodi_datasets( + SearchMelodiDatasetsInput(french_query="test"), + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, + ) + assert result.results[0].dataset_description.content == "First" + + async def test_description_missing_defaults(self): + es_response = { + "hits": {"hits": [{ + "_id": "DS_1", "_score": 1.0, + "_source": {"columns": "", "metadata": {}}, + }]}, + } + result = await search_melodi_datasets( + SearchMelodiDatasetsInput(french_query="test"), + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, + ) + assert result.results[0].dataset_description.content == "" + assert result.results[0].dataset_description.lang == "fr" + + async def test_description_dict_kept_as_is(self): + es_response = { + "hits": {"hits": [{ + "_id": "DS_1", "_score": 1.0, + "_source": { + "columns": "", + "metadata": { + "description": {"content": "Direct dict", "lang": "en"}, + }, + }, + }]}, + } + result = await search_melodi_datasets( + SearchMelodiDatasetsInput(french_query="test"), + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, + ) + assert result.results[0].dataset_description.content == "Direct dict" + + +# =================================================================== +# search_melodi_modalities +# =================================================================== + +class TestSearchMelodiModalities: + async def test_success_with_inner_hits(self): + es_response = { + "hits": {"hits": [{ + "_source": {"code": "PRICES", "text": "Price types"}, + "inner_hits": { + "modalities": {"hits": {"hits": [{ + "_score": 5.0, + "_source": { + "code": "D", + "label": {"en": "Unit value", "fr": "Valeur unitaire"}, + }, + }]}}, + }, + }]}, + } + params = SearchMelodiModalitiesInput( + dataset_id="DS_IPC", columns_id=["PRICES"], french_query="prix", + ) + + result = await search_melodi_modalities( + params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + ) + + assert len(result.results) == 1 + assert result.results[0].column_code == "PRICES" + mod = result.results[0].matching_modalities[0] + assert mod.code == "D" + assert mod.label_fr == "Valeur unitaire" + assert mod.score == 5.0 + + async def test_empty_results_raises_tool_error(self): + es = FakeElasticsearch(response={"hits": {"hits": []}}) + params = SearchMelodiModalitiesInput( + dataset_id="DS_X", columns_id=["COL"], french_query="unknown", + ) + + with pytest.raises(ToolError, match="EMPTY_RESULT"): + await search_melodi_modalities(params, es=es, settings=_SETTINGS) + + async def test_es_error_raises(self): + es = FakeElasticsearch(error=ESConnectionError("down")) + params = SearchMelodiModalitiesInput( + dataset_id="DS_X", columns_id=["COL"], french_query="test", + ) + + with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): + await search_melodi_modalities(params, es=es, settings=_SETTINGS) + + async def test_no_inner_hits_returns_empty_modalities(self): + es_response = { + "hits": {"hits": [{ + "_source": {"code": "GEO", "text": "Geography"}, + "inner_hits": {"modalities": {"hits": {"hits": []}}}, + }]}, + } + params = SearchMelodiModalitiesInput( + dataset_id="DS_1", columns_id=["GEO"], french_query="france", + ) + + result = await search_melodi_modalities( + params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + ) + + assert len(result.results) == 1 + assert result.results[0].matching_modalities == [] + + async def test_missing_label_defaults_to_empty(self): + es_response = { + "hits": {"hits": [{ + "_source": {"code": "COL", "text": "Column"}, + "inner_hits": { + "modalities": {"hits": {"hits": [{ + "_score": 1.0, + "_source": {"code": "X", "label": None}, + }]}}, + }, + }]}, + } + params = SearchMelodiModalitiesInput( + dataset_id="DS_1", columns_id=["COL"], french_query="test", + ) + + result = await search_melodi_modalities( + params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + ) + + mod = result.results[0].matching_modalities[0] + assert mod.label_en == "" + assert mod.label_fr == "" diff --git a/tests/test_rmes_helpers.py b/tests/test_rmes_service.py similarity index 100% rename from tests/test_rmes_helpers.py rename to tests/test_rmes_service.py From 12b582c91f68b5b6b9f5c9c389776a3ee65e3159 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Fri, 28 Aug 2026 10:36:00 +0200 Subject: [PATCH 05/55] fix: code review via 'Fixme' comments --- .gitignore | 3 +- src/mcpdiffusion/config/settings.py | 10 +++++-- src/mcpdiffusion/config/tool_metadata.py | 28 +++++++++++++------ src/mcpdiffusion/core/errors.py | 6 +++- src/mcpdiffusion/core/logging.py | 17 ++++++++++- src/mcpdiffusion/core/middleware.py | 12 ++++++++ src/mcpdiffusion/data/indicators.py | 3 ++ src/mcpdiffusion/data/themes.py | 3 ++ src/mcpdiffusion/infra/__init__.py | 2 ++ src/mcpdiffusion/infra/elasticsearch.py | 1 + src/mcpdiffusion/infra/lifespan.py | 17 ++++++++++- src/mcpdiffusion/models/feedback.py | 2 ++ src/mcpdiffusion/models/insee.py | 10 +++++-- src/mcpdiffusion/models/rmes.py | 4 +++ src/mcpdiffusion/server.py | 9 +++--- src/mcpdiffusion/services/feedback.py | 16 +++++++++++ src/mcpdiffusion/services/insee_document.py | 16 ++++++++++- src/mcpdiffusion/services/insee_search.py | 12 +++++++- src/mcpdiffusion/services/melodi.py | 11 ++++++++ src/mcpdiffusion/services/rmes.py | 18 +++++++++++- .../tools/extras_send_feedback.py | 2 ++ src/mcpdiffusion/tools/insee_get_document.py | 1 + src/mcpdiffusion/tools/insee_get_homepage.py | 4 ++- .../tools/insee_search_chiffrecle.py | 6 +++- .../tools/insee_search_conjoncture.py | 7 ++++- .../tools/insee_search_documents.py | 5 +++- .../tools/melodi_get_observations.py | 1 + src/mcpdiffusion/tools/rmes_run_sparql.py | 3 ++ 28 files changed, 201 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index a4d9852..ed6b96f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .venv/ *.env __pycache__/ -mcp_* \ No newline at end of file +mcp_* +.idea \ No newline at end of file diff --git a/src/mcpdiffusion/config/settings.py b/src/mcpdiffusion/config/settings.py index b1b9bce..d3e20cd 100644 --- a/src/mcpdiffusion/config/settings.py +++ b/src/mcpdiffusion/config/settings.py @@ -1,8 +1,7 @@ """Centralized application settings validated at import time via Pydantic.""" -from __future__ import annotations -import os from functools import lru_cache +# Fixme: prefer more recent syntax - ex: str | None instead of Optional from typing import Optional from pydantic import Field @@ -11,6 +10,7 @@ class Settings(BaseSettings): # Elasticsearch + # Fixme: if the environment variable name matches the variable name, there is no need for an alias es_host: Optional[str] = Field(default=None, alias="ES_HOST") es_index_produits: str = Field(default="produit", alias="ES_INDEX_PRODUITS") es_index_melodi_datasets: str = Field( @@ -27,6 +27,7 @@ class Settings(BaseSettings): mcp_host: str = Field(default="0.0.0.0", alias="MCP_HOST") mcp_port: int = Field(default=8000, alias="MCP_PORT") allowed_hosts: str = Field(default="*", alias="ALLOWED_HOSTS") + # Fixme: some variables are missing from the '.env.example' file forwarded_allow_ips: str = Field(default="*", alias="FORWARDED_ALLOW_IPS") # Rate limiting @@ -37,6 +38,7 @@ class Settings(BaseSettings): log_level: str = Field(default="INFO", alias="LOG_LEVEL") # Tool selection + # Fixme: prefer snake case syntax - ex: tool_list toollist: Optional[str] = Field(default=None, alias="TOOLLIST") # RMES / SPARQL @@ -54,6 +56,10 @@ class Settings(BaseSettings): default="https://www.insee.fr", alias="INSEE_BASE_URL" ) + # Fixme: this is just a preference for reading but multiline objects reads better + # I also think 'populate_by_name' can be ignored if we get rid of aliases + # Eventually 'SettingsConfigDict' is better for config than a plain dict since it catches typo'd key + # Beware .env file resolves relative to the current working directory model_config = {"env_file": ".env", "extra": "ignore", "populate_by_name": True} diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py index 4a5fa57..65ffc20 100644 --- a/src/mcpdiffusion/config/tool_metadata.py +++ b/src/mcpdiffusion/config/tool_metadata.py @@ -3,6 +3,8 @@ Design notes: - Tool *names* are English snake_case; French is kept only where it is actual data (enum literals that hit the ES index, user-supplied queries). + Fixme: the current is computed only once at import time, + if the server is running for 3 weeks, the old date will still be in effect... - `CURRENT_DATE` is computed lazily so long-running servers always report today's date, not the day the process started. - Tool descriptions describe the *final* schemas; rewrite in lockstep @@ -11,13 +13,22 @@ from datetime import date +# Fixme: I'd put such a generic function in a separate module +# just a preference, not mandatory def current_date_iso() -> str: """Return today's date as ISO-8601.""" return date.today().isoformat() +# Fixme: I feel like having the tools metadata separate / not-colocated with the tools might be a mistake +# Fixme: I believe those metadata can live with the corresponding set of tool functions +# by leveraging FastMCP capabilities - descriptions could live in the functions' docstring +# Fixme: I would also create one file per tool as a preference, but I get the argument +# of having a clear overview of tools at the same place # --- MELODI tools ----------------------------------------------------------- +# Fixme: if keeping those metadata separate, prefer multiline strings that read better +# and avoid missing spaces issues GET_DATASET = { "tool_name": "get_melodi_observations", "tool_description": ( @@ -149,6 +160,7 @@ def current_date_iso() -> str: "tool_metadata": {"version": "5.0", "author": "mirlon"}, } +# Fixme: Those docstrings are computed once at import time, therefore, the current date is never re-computed SEARCH_DOCUMENTS = { "tool_name": "search_insee_documents", "tool_description": ( @@ -206,14 +218,14 @@ def current_date_iso() -> str: SEARCH_CHIFFRECLEF = { "tool_name": "search_insee_chiffrecle", "tool_description": "Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, \n" - "comparaisons regionales/departementales et statistiques factuelles simples.\n" - "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" - "prix par categorie, comparaisons geographiques (region, departement, commune).\n" - "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" - "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" - "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" - "ou search_insee_documents selon le contexte).\n" - "Retourne directement les tableaux synthetiques prets a l'emploi.\n", + "comparaisons regionales/departementales et statistiques factuelles simples.\n" + "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" + "prix par categorie, comparaisons geographiques (region, departement, commune).\n" + "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" + "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" + "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" + "ou search_insee_documents selon le contexte).\n" + "Retourne directement les tableaux synthetiques prets a l'emploi.\n", "tool_metadata": {"version": "5.0", "author": "mirlon"}, } diff --git a/src/mcpdiffusion/core/errors.py b/src/mcpdiffusion/core/errors.py index 33c3bbd..eaa87e8 100644 --- a/src/mcpdiffusion/core/errors.py +++ b/src/mcpdiffusion/core/errors.py @@ -17,7 +17,7 @@ "UNKNOWN", ] - +# Fixme: fail returns None while it always raises which can fool type checkers def fail( code: ErrorCode, message: str, @@ -28,7 +28,11 @@ def fail( `message` should be actionable: name the offending parameter, suggest the next step, include the shortest useful excerpt of the upstream error. """ + # Fixme: the prefix can be built once using an f string instead of being re-assigned prefix = f"[{code}] " if retryable: + # Fixme: it might be better to leverage a separate error flag for information like retryable + # so the information is easier to identify vs baked into a string + # this can be achieved by subclassing ToolError, I guess prefix = f"[{code}, retryable] " raise ToolError(prefix + message) diff --git a/src/mcpdiffusion/core/logging.py b/src/mcpdiffusion/core/logging.py index 7ceca27..c3da87b 100644 --- a/src/mcpdiffusion/core/logging.py +++ b/src/mcpdiffusion/core/logging.py @@ -1,5 +1,4 @@ """Structured logging config + per-tool decorator.""" -from __future__ import annotations import functools import inspect @@ -11,14 +10,22 @@ _settings = get_settings() +# Fixme: this is a trade-off to make log fall under the same logger name, I would not recommend it +# a convention is to use the module name for identification MAIN_LOGGER_NAME = "mcp.main" logging.basicConfig( level=_settings.log_level, + # Fixme: the following format string is duplicated format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", force=True, ) +# Fixme: 'UVICORN_LOGGING_CONFIG' is leveraged in the server.py main block +# but this block is not always ran, especially when the app is launched using uvicorn +# this prevents the log level set from being applied to uvicorn logs +# I'd suggest unifying config in a single place to invoke it systematically +# Also, this config competes with the one above UVICORN_LOGGING_CONFIG = { "version": 1, "disable_existing_loggers": False, @@ -49,6 +56,7 @@ _KWARGS_PREVIEW_LIMIT = 800 +# Fixme: only single level items are scrubbed def _scrub(kwargs: dict) -> str: safe = {} for k, v in kwargs.items(): @@ -86,6 +94,8 @@ def log_tool(func: _F) -> _F: def _log_exit(duration_ms: float, result: Any) -> None: count = _result_count(result) if count is None: + # Fixme: prefer using the extra key to provide additional elements to log instead of information + # concatenated in a textual prose logger.info("Tool exit: %s | %.1fms", name, duration_ms) else: logger.info( @@ -93,12 +103,15 @@ def _log_exit(duration_ms: float, result: Any) -> None: ) def _log_error(duration_ms: float, exc: BaseException) -> None: + # Fixme: the exception encapsulated within 'exc' is not leveraged fully, + # the stack is missing which is critical information to log code = getattr(exc, "args", ("",))[0] if exc.args else type(exc).__name__ logger.error( "Tool error: %s | %.1fms | %s: %s", name, duration_ms, type(exc).__name__, str(code)[:200], ) + # Fixme: this code can be simplified, especially when only a few lines differ per outcome if is_async: @functools.wraps(func) async def async_wrapper(*args, **kwargs): @@ -106,6 +119,8 @@ async def async_wrapper(*args, **kwargs): start = time.perf_counter() try: result = await func(*args, **kwargs) + # Fixme: this is very broad exception handling + # Indeed, this also catches KeyboardInterrupt, SystemExit, and asyncio.CancelledError except BaseException as exc: _log_error((time.perf_counter() - start) * 1000, exc) raise diff --git a/src/mcpdiffusion/core/middleware.py b/src/mcpdiffusion/core/middleware.py index 70a7178..7044213 100644 --- a/src/mcpdiffusion/core/middleware.py +++ b/src/mcpdiffusion/core/middleware.py @@ -20,16 +20,26 @@ class RateLimitMiddleware(BaseHTTPMiddleware): (used by tests). Falls back to ``get_settings()`` when not provided. """ + # Fixme: passing the whole settings object is inappropriate + # Only relevant properties should be passed + # It allows for better interface segregation + # This would also avoid having this class creating the settings as a fallback def __init__(self, app, settings: Settings | None = None): super().__init__(app) self._settings = settings or get_settings() self._tz = ZoneInfo(self._settings.tz) + # Fixme: the storage behavior does not scale with multiple replicas + # Consider implementing redis for handling state between multiple instances self._storage = storage.MemoryStorage() self._limiter = strategies.MovingWindowRateLimiter(self._storage) self._rate = parse(f"{self._settings.global_request_min}/minute") + # Fixme: there is no request path filtering on that dispatcher, meaning it runs also for non relevant path like / + # or even health checks async def dispatch(self, request: Request, call_next): client_ip = request.client.host if request.client else "unknown" + # Fixme: if the key is the client ip, double check if anyone can alter it and bypass rate limiting... + # Fixme: also the usage of an fstring is useless here rate_key = f"{client_ip}" if not self._limiter.hit(self._rate, rate_key): @@ -52,9 +62,11 @@ async def dispatch(self, request: Request, call_next): response = await call_next(request) + # Fixme: the result of 'self._limiter.get_window_stats(self._rate, rate_key)' could have been cached remaining = self._limiter.get_window_stats(self._rate, rate_key)[1] response.headers["X-RateLimit-Limit"] = str(self._settings.global_request_min) response.headers["X-RateLimit-Remaining"] = str(remaining) + # Fixme: 60 is a magic number and should be set using a constant config response.headers["X-RateLimit-Window"] = f"{60}s" return response diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/indicators.py index 038cbbc..4411d50 100644 --- a/src/mcpdiffusion/data/indicators.py +++ b/src/mcpdiffusion/data/indicators.py @@ -1,5 +1,8 @@ """Curated INSEE key indicators (homepage data).""" +# Fixme: I am wondering whether this really belongs in the source code or in a separate file or database +# Fixme: The 1st entry seems like a header (contains no real data), is that normal? +# Fixme: This seems like hardcoded, stale statistics... I don't know if this is normal DICT_KV = [ {"cle": "clé", "alias": "alias", "valeur": "valeur"}, {"cle": "estimation de population France", "alias": "", "valeur": "Au 1er janvier 2026, la population résidant en France est estimée à 69,1 millions d'habitants."}, diff --git a/src/mcpdiffusion/data/themes.py b/src/mcpdiffusion/data/themes.py index 125e104..1b30b18 100644 --- a/src/mcpdiffusion/data/themes.py +++ b/src/mcpdiffusion/data/themes.py @@ -1,5 +1,8 @@ """INSEE theme mappings and conjoncture sub-themes.""" +# Fixme: Is that the right place for this kind of data? In the source code? +# Maybe a JSON file or a database is a better place +# Fixme: This seems like mapping themes to IDs manually, this is fragile if so... KEYS_THEME_NIV1 = { "Demographie": 0, "Conditions de vie - Societe": 6, diff --git a/src/mcpdiffusion/infra/__init__.py b/src/mcpdiffusion/infra/__init__.py index f0a00f5..6333438 100644 --- a/src/mcpdiffusion/infra/__init__.py +++ b/src/mcpdiffusion/infra/__init__.py @@ -1 +1,3 @@ """Infrastructure: external clients (ES, HTTP, SPARQL).""" + +# Fixme: each of the dependency functions in that folder does not provide proper typing which is a pity \ No newline at end of file diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py index 974a829..b5e3336 100644 --- a/src/mcpdiffusion/infra/elasticsearch.py +++ b/src/mcpdiffusion/infra/elasticsearch.py @@ -1,4 +1,5 @@ """Elasticsearch client accessor from FastMCP lifespan context.""" +# Fixme: this annotation seems unnecessary from __future__ import annotations from fastmcp import Context diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/infra/lifespan.py index b4cfc14..dfd38ef 100644 --- a/src/mcpdiffusion/infra/lifespan.py +++ b/src/mcpdiffusion/infra/lifespan.py @@ -9,6 +9,7 @@ from ..config.settings import get_settings +# Fixme: was there not a reference for that logger name in 'config/logging.py'? logger = logging.getLogger("mcp.main") _HTTP_USER_AGENT = ( @@ -18,14 +19,25 @@ _HTTP_TIMEOUT = httpx.Timeout(30.0, connect=10.0) _SPARQL_USER_AGENT = "MCP-RMeS/2.0" - +# Fixme: One issue I see with that pattern is that if some client instantiation fail, +# the error is ignored and this can be tricky to identify +# Fixme: Also, i see no client is properly tested upon creation (simple ping). +# It could help identify issues at startup: but this is not mandatory +# Fixme: this piece of code contains too many magic values that belong in settings +# Fixme: server is unused, if required by FastMCP, prefer prefixing it with an underscore @lifespan async def app_lifespan(server): + # Fixme: I am questioning whether this should be the responsibility of that function to instantiate settings s = get_settings() # Elasticsearch + # Fixme: in 'config/settings.py', 'es_host' is deemed optional, so it must be made mandatory if required here if not s.es_host: raise RuntimeError("ES_HOST is not set. See .env.example.") + + # Fixme: if this synchronous client is used within async coroutines, + # it will block the event loop for the duration of the query + # This is a major issue es_client = Elasticsearch( s.es_host, verify_certs=s.tls_verify, @@ -44,6 +56,7 @@ async def app_lifespan(server): logger.info("HTTP client initialized") # SPARQL client + # Fixme: TLS is ignored in some clients which seems inconsistent sparql_client = httpx.AsyncClient( headers={"User-Agent": _SPARQL_USER_AGENT}, ) @@ -55,6 +68,8 @@ async def app_lifespan(server): "sparql_client": sparql_client, } + # Fixme: the instantiated elastic client is synchronous, so cannot be prepended by the 'await' keyword + # this would app to raise and crash on shutdown await es_client.close() await http_client.aclose() await sparql_client.aclose() diff --git a/src/mcpdiffusion/models/feedback.py b/src/mcpdiffusion/models/feedback.py index e013cb5..1af8115 100644 --- a/src/mcpdiffusion/models/feedback.py +++ b/src/mcpdiffusion/models/feedback.py @@ -25,6 +25,8 @@ class SendFeedbackInput(BaseModel): class SendFeedbackOutput(BaseModel): + # Fixme: prefer a Literal status: str = "success" message: str + # Fixme: why not use a datetime object? timestamp: str diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 926213a..2f99c01 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -7,6 +7,10 @@ from pydantic import BaseModel, Field +# Fixme: Pydantic BaseModel inheriting can be leverage to avoid duplication through model composition +# (FastAPI provides great examples on that) +# Fixme: a lot of static data from this file seems derived from the one in the 'data' package +# this could be merged / refactored / better exploited class INSEETheme(StrEnum): ALL = "ALL" METHODES = "Methodes" @@ -77,7 +81,7 @@ class DocumentHit(BaseModel): # --- search_insee_documents --- - +# Fixme: use model composition to avoid duplication class SearchInseeDocumentsInput(BaseModel): query: str = Field( description="Natural-language search query describing the statistics to retrieve.", @@ -105,6 +109,8 @@ class SearchInseeDocumentsInput(BaseModel): "'Bouches-du-Rhone'). Leave null to skip geographic filtering." ), ) + # Fixme: this field annotation is used multiple times and can be put in a variable to avoid duplication + # Fixme: magic values should be avoided number_of_results: int = Field( default=10, description="Maximum number of results to return.", @@ -112,7 +118,7 @@ class SearchInseeDocumentsInput(BaseModel): le=20, ) - +# Fixme: the same model shape is used 3 times class SearchInseeDocumentsOutput(BaseModel): results: list[DocumentHit] count: int diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index e56e6d0..36627b3 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -9,6 +9,7 @@ # --- Shared RMES constants exposed to tools --- +# Fixme: a lot of values in here belongs in settings DEFAULT_TIMEOUT = 20.0 MAX_TIMEOUT = 60.0 DEFAULT_ROW_LIMIT = 200 @@ -38,6 +39,8 @@ class GraphCategoryChoice(StrEnum): # --- Error types --- +# Fixme: I am afraid these error models might compete with what is defined in 'core/errors.py' +# We should provide uniform errors accros the application to simply parsing for clients class SparqlErrorType(StrEnum): INVALID_QUERY_FORM = "INVALID_QUERY_FORM" TIMEOUT = "TIMEOUT" @@ -108,6 +111,7 @@ class DescribeResourceInput(BaseModel): description="URI complete de la ressource RDF a decrire.", examples=["http://id.insee.fr/codes/naf2025/section/A"], ) + # Fixme: either use Optional or the modern pipe syntax, but avoid mixing graph: str | None = Field( default=None, description=( diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 2dc96b1..fb64605 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -3,10 +3,8 @@ Boots Uvicorn, registers every tool via `tools.register_tools(mcp)`, and exposes the HTTP transport on MCP_HOST:MCP_PORT. """ -from __future__ import annotations import logging -import sys from dotenv import load_dotenv from fastmcp import FastMCP @@ -18,8 +16,6 @@ from .tools import register_tools from .infra.lifespan import app_lifespan -load_dotenv() - settings = get_settings() logger = logging.getLogger(MAIN_LOGGER_NAME) @@ -30,6 +26,8 @@ app = mcp.http_app() # TrustedHostMiddleware +# Fixme: the 'ALLOWED_HOSTS' env variable is not present in the .env.example file, defaulting to allowed hosts to "*" +# Fixme: this code that sets '_allowed_hosts' belongs in the settings, not here _allowed_hosts_raw = settings.allowed_hosts.strip() _allowed_hosts = ( ["*"] if _allowed_hosts_raw == "*" @@ -40,8 +38,9 @@ "TrustedHostMiddleware configured with allowed_hosts=['*']. " "Set ALLOWED_HOSTS before exposing the server publicly." ) -app.add_middleware(TrustedHostMiddleware, allowed_hosts=_allowed_hosts) + app.add_middleware(RateLimitMiddleware, settings=settings) +app.add_middleware(TrustedHostMiddleware, allowed_hosts=_allowed_hosts) if __name__ == "__main__": import uvicorn diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py index db0ce0f..cada1bc 100644 --- a/src/mcpdiffusion/services/feedback.py +++ b/src/mcpdiffusion/services/feedback.py @@ -6,10 +6,14 @@ from ..models.feedback import SendFeedbackInput, SendFeedbackOutput +# Fixme: this is extremely hacky, and feedback gets tied to the running instance _FEEDBACK_DIR = Path(__file__).resolve().parent.parent / "feedback" _FEEDBACK_FILE = _FEEDBACK_DIR / "feedback.md" +# Fixme: this function writes files within the running container, this is a major side effect +# Fixme: also check where this code is invoked, because if in the event loop, it is blocking +# Fixme: those kind of checks belong at the app startup, for example in a lifespan function def _ensure_feedback_file() -> Path: _FEEDBACK_DIR.mkdir(parents=True, exist_ok=True) if not _FEEDBACK_FILE.exists(): @@ -23,16 +27,28 @@ def _ensure_feedback_file() -> Path: return _FEEDBACK_FILE +# Fixme: I am wondering where the username come from, because if sent by the client, this can be messed up +# Fixme: Also wondering if there is a cap on the username size of the feedback content, +# it can make the container write uncontrolled amount of data async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: feedback_path = _ensure_feedback_file() + # Fixme: there is no timezone here, while at some in the code we consider timezones + # it seems a bit inconsistent timestamp = datetime.now().isoformat(timespec="seconds") + # Fixme: prefer more readable multiline strings entry = ( + # Fixme: again, people can insert anything and forge data into the feedback file + # just hope this is not ultimately fed to an LLM + # Fixme: a big flaw is that feedback.md is versioned, so user feedback might be fed into git + # Fixme: beware the data is lost on each restart f"## {timestamp} — {params.username}\n\n" f"{params.feedback}\n\n" "---\n\n" ) + # Fixme: this call is blocking the event loop + # consider using aiofiles instead with feedback_path.open("a", encoding="utf-8") as f: f.write(entry) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 99cb087..98c4d35 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -23,6 +23,7 @@ output_format="markdown", links=True, formatting=True, + # Fixme: this URL might belong in the settings source="insee.fr", with_metadata=True, ) @@ -34,7 +35,8 @@ def _as_relative(url: str) -> str: p = urlparse(url) return f"{p.path}?{p.query}" if p.query else p.path - +# Fixme: some complex composed types are involved multiple times - ex: list[dict[str, str] +# it might be better to leverage Pydantic and create meaningful type aliases - ex TableOfContentParams = list[dict[str, str] def _parse_sommaire(html: str, base_url: str) -> list[dict[str, str]]: soup = BeautifulSoup(html, "lxml") results: list[dict[str, str]] = [] @@ -87,8 +89,11 @@ def _format_sommaire(flat_items: list[dict[str, str]]) -> dict[str, dict[str, st def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: if len(text) <= limit: return text, False + # Fixme: avoid magic numbers popping here and there head_size = (limit * 2) // 3 + # Fixme: what happens if limit is too small? 'tail_size' can go negative tail_size = limit - head_size - 200 + # Fixme: prefer multiline strings which are more readable and easier to deal with marker = ( "\n\n\n\n" @@ -96,12 +101,17 @@ def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: return text[:head_size] + marker + text[-tail_size:], True +# Fixme: injecting the whole settings is bad separation of concerns async def _fetch_html(url: str, settings: Settings, http_client: httpx.AsyncClient) -> str: full_url = settings.insee_base_url + url if not url.startswith(("http://", "https://")) else url try: response = await http_client.get(full_url, follow_redirects=True) response.raise_for_status() return response.text + # Fixme: the error handling is not correctly designed, at a global scale + # Fixme: for example, here, the 'fail' invocation raises a ToolError nesting any information within a string + # so an error is logged twice and the client ultimately receives an error string + # he can hardly react on automatically except httpx.TimeoutException as exc: fail( "BACKEND_UNAVAILABLE", @@ -132,12 +142,14 @@ async def _fetch_html(url: str, settings: Settings, http_client: httpx.AsyncClie raise +# Fixme: here params obfuscates the meaning of the input argument async def get_insee_document( params: GetInseeDocumentInput, *, http_client: httpx.AsyncClient, settings: Settings | None = None, ) -> GetInseeDocumentOutput: + # Fixme: not the right place to init settings s = settings or get_settings() if not params.list_of_url: @@ -148,6 +160,8 @@ async def get_insee_document( ) results: list[DocumentResult] = [] + # Fixme: there should be a cap in the number of URLs provided to avoid overloading the server + # Fixme: on top of that, the fetching is done sequentially, impacting the event loop for url in params.list_of_url: try: html = await _fetch_html(str(url), s, http_client) diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index e62d254..575b9ec 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -25,7 +25,8 @@ def _coerce_hit_value(value) -> Optional[str]: # Build query - +# Fixme: the type tuple[list, list, list, list] should be aliased for comprehension +# Fixme: also prefer newer optional syntax def build_text_clauses( query: Optional[str], year_of_reference: Optional[int], @@ -35,6 +36,7 @@ def build_text_clauses( must: list = [] filters: list = [] should: list = [] + # Fixme: must_not is not used and returned as is, this is inappropriate must_not: list = [] if query: @@ -93,6 +95,7 @@ def apply_collection_filters( """Apply INSEE-specific filters. Returns updated (filters, should).""" should: list = [] + # Fixme: The passed in 'filters' is mutated if must_only_rapides: filters.append(Q("term", collection_libelle="Informations rapides")) elif must_not_rapides: @@ -100,7 +103,9 @@ def apply_collection_filters( Q("bool", must_not=[Q("term", collection_libelle="Informations rapides")]) ) + # Fixme: the 1st check seems useless if theme and theme != "ALL": + # Fixme: so if the caller sends an unregistered theme, we drop his filter anyway? id_theme = KEYS_THEME_NIV1.get(theme) if id_theme is not None: filters.append(Q("term", idthemeparent=id_theme)) @@ -111,6 +116,7 @@ def apply_collection_filters( if geo_niveau: key_geo = DICT_GEO.get(geo_niveau) if key_geo: + # Fixme: same as above filters.append(Q("term", geo_niveau=key_geo)) if geo_keyword and geo_keyword.lower() != "all": @@ -130,6 +136,7 @@ def apply_collection_filters( # Execute search with built query +# Fixme: inject only relevant settings parameters def execute_search( *, must: list, @@ -141,6 +148,7 @@ def execute_search( settings: Settings | None = None, ) -> list[DocumentHit]: """Run the assembled bool query and return whitelisted DocumentHit records.""" + # Fixme: such function should not init settings s = settings or get_settings() search = Search(using=es, index=s.es_index_produits).query( Q( @@ -151,6 +159,8 @@ def execute_search( filter=filters, should=should, must_not=must_not, + # Fixme: this seem counter counter intuitive, to require a minimum should match, + # so it is not a should eventually? minimum_should_match=1 if should else 0, ), boost_mode="sum", diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index e90104e..6adb86b 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -27,6 +27,7 @@ async def get_melodi_observations( http_client: httpx.AsyncClient, settings: Settings | None = None, ) -> GetMelodiObservationsOutput: + # Fixme: same comment for settings, inject relevant properties only s = settings or get_settings() url = f"{s.melodi_data_base_url}/{params.dataset_id}" try: @@ -35,6 +36,7 @@ async def get_melodi_observations( params=params.dict_of_columns_and_values or None, ) response.raise_for_status() + # Fixme: the following problematic error handling pattern has already been adressed except httpx.TimeoutException as exc: fail( "BACKEND_UNAVAILABLE", @@ -89,13 +91,18 @@ async def get_melodi_observations( "PARSE_ERROR", "Melodi API response did not contain an 'observations' list.", ) + # Fixme: this not not an appropriate fix + # this piece of code is unreachable since 'fail' raises already before raise # pragma: no cover if params.list_of_year: years_str = {str(y) for y in params.list_of_year} + # Fixme: it seems we retrieve all the observations data and filter next + # I wonder whether the API supports filtering observations = [ obs for obs in observations + # Fixme: 'TIME_PERIOD' could be sanitized if (obs.get("dimensions", {}) .get("TIME_PERIOD", "") .split("-")[0]) in years_str @@ -108,6 +115,8 @@ async def get_melodi_observations( count=len(sliced), ) +# Fixme: I believe this is not the correct place (inside the service) to place a raw complex query +# the code might benefit having a repository layer to encapsulate data access async def search_melodi_datasets( params: SearchMelodiDatasetsInput, @@ -209,6 +218,7 @@ async def search_melodi_datasets( description = source.get("metadata", {}).get("description") if isinstance(description, list) and description: description = description[0] + # Fixme: this branch does nothing elif isinstance(description, dict): description = description else: @@ -236,6 +246,7 @@ async def search_melodi_modalities( filters.append({"terms": {"code": params.columns_id}}) try: + # Fixme: the 1st es.search call was formatted differently, pick a single convention ds_column = es.search( index=s.es_index_melodi_columns, size=20, diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index a04a41e..5c0956d 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -31,6 +31,7 @@ ListGraphsInput, ) +# Fixme: follow a clear convention for logger names logger = logging.getLogger("mcp.rmes") # Cache for raw graph rows (expensive COUNT query) @@ -60,9 +61,10 @@ # Graph taxonomy # --------------------------------------------------------------------------- +# Fixme: this is too broad of a type CategoryMatcher = Any # Callable[[str], bool] - +# Fixme: you can use an immutable (frozen) dataclass instead - ex: annotate the class with '@dataclass(frozen=True)' class _CategoryRule: __slots__ = ("key", "label", "description", "match") @@ -198,6 +200,7 @@ def _prefix(prefix: str) -> CategoryMatcher: match=lambda path: True, ) +# Fixme: '_RULES_BY_KEY' uses '_ALL_RULES', but ultimately, '_RULES_BY_KEY' is never used _ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] _RULES_BY_KEY = {r.key: r for r in _ALL_RULES} @@ -234,6 +237,8 @@ def _detect_query_form(query: str) -> str: def _ensure_limit(query: str, query_form: str, max_rows: int) -> tuple[str, bool]: if query_form not in ("SELECT", "CONSTRUCT"): return query, False + # Fixme: this is a particular case, but if there is inner queries with the word limit, + # nothing prevents outer queries from not being bound if _LIMIT_RE.search(query): return query, False return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True @@ -324,6 +329,8 @@ async def _execute_sparql( if accept == "text/turtle": return {"format": "turtle", "limit_added": limit_added, "data": response.text} + # Fixme: if the parsing of the response fails, it will lead to an unhandled exception + # as this line of code is not wrapped within the try except block result = response.json() if limit_added: result.setdefault("_meta", {})["limit_added"] = max_rows @@ -346,7 +353,12 @@ async def _get_raw_graph_rows( "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } " "GROUP BY ?g ORDER BY DESC(?nbTriples)" ) + # Fixme: note that while this request runs (async nature), + # other concurrent requests can still enter the current block + # consider an asyncio.Lock + a second freshness check inside it, + # otherwise each waiter just re-runs the same expensive query result = await _execute_sparql( + # Fixme: those magic values belong in the settings query, timeout=45.0, max_rows=1000, sparql_client=sparql_client, settings=settings, ) @@ -425,6 +437,7 @@ async def list_graphs( bucket_rows = [ GraphRow(graph=g, triples=t) for g, t in rows_by_graph.items() + # Fixme: I though '_build_category_summary' already categorized every row if _categorize(g).key == bucket.category ] bucket_rows.sort(key=lambda r: r.triples, reverse=True) @@ -457,6 +470,8 @@ async def describe_resource( ) -> DescribeResourceOutput: graph_clause = f"<{params.graph}>" if params.graph else "?g" graph_values = f"VALUES ?g {{ <{params.graph}> }}" if params.graph else "" + # Fixme: the query is built using string interpolation + # just check whether injection can cause problems here query = f""" SELECT ?g ?direction ?p ?o WHERE {{ {graph_values} @@ -469,6 +484,7 @@ async def describe_resource( }} }} LIMIT {MAX_ROW_LIMIT} """ + # Fixme: put the import at the top from ..models.rmes import DEFAULT_TIMEOUT result = await _execute_sparql( query, timeout=DEFAULT_TIMEOUT, max_rows=MAX_ROW_LIMIT, diff --git a/src/mcpdiffusion/tools/extras_send_feedback.py b/src/mcpdiffusion/tools/extras_send_feedback.py index cc5d18c..21569de 100644 --- a/src/mcpdiffusion/tools/extras_send_feedback.py +++ b/src/mcpdiffusion/tools/extras_send_feedback.py @@ -9,6 +9,8 @@ from ..services.feedback import send_feedback +# Fixme: I advocated for co-location schema + tools using docstrings if possible +# Fixme: I already stated clients can send anything as username and feedback def register_extras_send_feedback(mcp: FastMCP) -> None: @mcp.tool( name=SEND_FEEDBACK["tool_name"], diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 9673eff..78825be 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -21,4 +21,5 @@ async def get_insee_documents( params: GetInseeDocumentInput, ctx: Context, ) -> GetInseeDocumentOutput: + # Fixme: use singular or plural and stick to it return await get_insee_document(params, http_client=get_http_client(ctx)) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index 7989333..f3c0b5b 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -8,13 +8,15 @@ from ..data.indicators import DICT_KV from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator - +# Fixme: the tool contains no service which is kind of breaking the convention I saw earlier +# this correlates unit testing to the tool mechanics def register_get_insee_homepage(mcp: FastMCP) -> None: @mcp.tool( name=GET_HOMEPAGE["tool_name"], description=GET_HOMEPAGE["tool_description"], meta=GET_HOMEPAGE["tool_metadata"], ) + # Fixme: this is an async function with nothing to await @log_tool async def get_insee_homepage() -> KeyIndicatorsOutput: indicators = [ diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index 402d24e..010d05e 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -19,7 +19,8 @@ execute_search, ) - +# Fixme: the orchestration present in that function belongs in a service +# Indeed, the approach from one tool to another is inconsistent def register_search_insee_chiffreclef(mcp: FastMCP) -> None: @mcp.tool( name=SEARCH_CHIFFRECLEF["tool_name"], @@ -35,6 +36,8 @@ async def search_insee_chiffrecle( query=params.query, year_of_reference=params.year_of_reference, ) + + # Fixme: should is overridden here filters, should = apply_collection_filters( filters, must_not_rapides=True, @@ -60,5 +63,6 @@ async def search_insee_chiffrecle( "Verify ES_HOST and try again.", retryable=True, ) + # Fixme: we saw that the fail function did already raise raise return SearchInseeChiffrecleOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index 011e6fe..7232280 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -21,7 +21,7 @@ execute_search, ) - +# Fixme: again, a lot of code in that tool that should belong in the service def register_search_insee_conjoncture(mcp: FastMCP) -> None: @mcp.tool( name=SEARCH_CONJONCTURE["tool_name"], @@ -37,13 +37,17 @@ async def search_insee_conjoncture( query=params.query, year_of_reference=params.year_of_reference, ) + # Fixme: should is overridden filters, should = apply_collection_filters( filters, + # Fixme: the following allows for creating confusing combinaison must_not_rapides=False, must_only_rapides=True, ) if params.theme_conjoncture: subthemes = DICT_THEME_CONJ.get(params.theme_conjoncture) + # Fixme: I don't know if this is the wanted behavior, but a subtheme miss will discard filtering, + # so return everything? if subthemes: filters.append(Q("terms", conjoncture_libelle=subthemes)) @@ -63,5 +67,6 @@ async def search_insee_conjoncture( "Verify ES_HOST and try again.", retryable=True, ) + # Fixme: as stated, this raise is dead raise return SearchInseeConjonctureOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index 211934d..cc4639d 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -19,7 +19,8 @@ execute_search, ) - +# Fixme: state clear conventions between what goes to a tool and what do not +# most of the code might belong in the service def register_search_insee_documents(mcp: FastMCP) -> None: @mcp.tool( name=SEARCH_DOCUMENTS["tool_name"], @@ -35,6 +36,7 @@ async def search_insee_documents( query=params.query, year_of_reference=params.year_of_reference, ) + # Fixme: should is overridden filters, should = apply_collection_filters( filters, must_not_rapides=True, @@ -60,5 +62,6 @@ async def search_insee_documents( "Verify ES_HOST and try again.", retryable=True, ) + # Fixme: dead raise raise return SearchInseeDocumentsOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index a792205..ea456d0 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -12,6 +12,7 @@ def register_get_melodi_observations(mcp: FastMCP) -> None: @mcp.tool( + # Fixme: 'GET_DATASET' as variable name is too broad, thus misleading name=GET_DATASET["tool_name"], description=GET_DATASET["tool_description"], meta=GET_DATASET["tool_metadata"], diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index 8e9c534..c2aeab3 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -13,6 +13,9 @@ def register_rmes_run_sparql(mcp: FastMCP) -> None: @mcp.tool( name=RMES_RUN_SPARQL["tool_name"], + # Fixme: this description combinaison involves too many sources of data, the metadata file, + # the service plus a hardoded description + # Fixme: this is also not the right place for a query description=RMES_RUN_SPARQL["tool_description"] + "\n" + KNOWN_VOCABULARIES_NOTE + "\n\n" "Exemple -- recherche de codes NAF contenant \"extraction\" :\n" "PREFIX skos: \n" From d00e9a7374c8f77ec03f1538b2eeddca704603cd Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Mon, 31 Aug 2026 10:31:11 +0200 Subject: [PATCH 06/55] change toolist --- src/mcpdiffusion/config/settings.py | 5 ++++- src/mcpdiffusion/server.py | 2 +- src/mcpdiffusion/tools/__init__.py | 25 ++++++------------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/mcpdiffusion/config/settings.py b/src/mcpdiffusion/config/settings.py index b1b9bce..01bc94c 100644 --- a/src/mcpdiffusion/config/settings.py +++ b/src/mcpdiffusion/config/settings.py @@ -37,7 +37,10 @@ class Settings(BaseSettings): log_level: str = Field(default="INFO", alias="LOG_LEVEL") # Tool selection - toollist: Optional[str] = Field(default=None, alias="TOOLLIST") + enable_melodi : bool = Field(default=True, alias = "ENABLE_MELODI") + enable_inseefr : bool = Field(default=True, alias = "ENABLE_INSEEFR") + enable_rmes : bool = Field(default=True, alias = "ENABLE_RMES") + # RMES / SPARQL rmes_endpoint: str = Field( diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 2dc96b1..3027713 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -25,7 +25,7 @@ mcp = FastMCP("INSEE-mcp-diffusion", lifespan=app_lifespan) -register_tools(mcp, toollist=settings.toollist) +register_tools(mcp, settings) app = mcp.http_app() diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 147ae07..6f91412 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -21,10 +21,11 @@ from .rmes_run_sparql import register_rmes_run_sparql from .extras_send_feedback import register_extras_send_feedback -def register_tools(mcp: FastMCP, toollist: str | None = None) -> None: + +def register_tools(mcp: FastMCP, settings ) -> None: """Register all MCP tools with the given FastMCP instance.""" # INSEE.fr - if toollist == ("insee"): + if settings.enable_inseefr : register_search_insee_documents(mcp) register_get_insee_homepage(mcp) register_get_insee_document(mcp) @@ -32,27 +33,13 @@ def register_tools(mcp: FastMCP, toollist: str | None = None) -> None: register_search_insee_chiffreclef(mcp) # Melodi - if toollist == ("melodi"): + if settings.enable_melodi : register_search_melodi_datasets(mcp) register_search_melodi_modalities(mcp) register_get_melodi_observations(mcp) # RMES (SPARQL) - if toollist == ("rmes"): - register_rmes_list_graphs(mcp) - register_rmes_describe_resource(mcp) - register_rmes_run_sparql(mcp) - - else: - register_search_insee_documents(mcp) - register_get_insee_homepage(mcp) - register_get_insee_document(mcp) - register_search_insee_conjoncture(mcp) - register_search_insee_chiffreclef(mcp) - register_search_melodi_datasets(mcp) - register_search_melodi_modalities(mcp) - register_get_melodi_observations(mcp) + if settings.enable_rmes: register_rmes_list_graphs(mcp) register_rmes_describe_resource(mcp) - register_rmes_run_sparql(mcp) - register_extras_send_feedback(mcp) + register_rmes_run_sparql(mcp) \ No newline at end of file From ffd134376c83d8e777c3816f77f9f8953984db54 Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Tue, 1 Sep 2026 09:30:42 +0200 Subject: [PATCH 07/55] [feat} multistage dockerfile --- Dockerfile | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b44eea..5dffef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.12-slim +# ---- Build stage ---- +FROM python:3.12-slim AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 @@ -14,14 +15,26 @@ COPY pyproject.toml uv.lock /app/ # Install dependencies (no dev, no editable install) RUN uv sync --no-dev --no-install-project --frozen +COPY src/ /app/src/ + +# Install the project itself +RUN uv sync --no-dev --frozen + +# ---- Runtime stage ---- +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + # Utilisateur non privilegie (UID/GID fixes pour la coherence des volumes) RUN groupadd --gid 1000 app \ && useradd --uid 1000 --gid 1000 --create-home --shell /usr/sbin/nologin app -COPY --chown=app:app src/ /app/src/ - -# Install the project itself -RUN uv sync --no-dev --frozen +# Copy only the virtual environment and source from the build stage +COPY --from=builder --chown=app:app /app/.venv /app/.venv +COPY --from=builder --chown=app:app /app/src /app/src USER app @@ -29,6 +42,5 @@ EXPOSE 8000 # Default ES_HOST points at the Docker-compose service name; override when # running the image standalone. -ENV ES_HOST="http://elasticsearch:9200" -CMD ["uv", "run", "--no-dev", "python", "-m", "mcpdiffusion.server"] +CMD ["/app/.venv/bin/python", "-m", "mcpdiffusion.server"] From ab42b58b66043be29edd2a6615eea0cc138066ae Mon Sep 17 00:00:00 2001 From: ESDH3T Date: Tue, 1 Sep 2026 11:27:59 +0200 Subject: [PATCH 08/55] [fix] add k8s es service --- k8s/1_es_deploy.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/k8s/1_es_deploy.yaml b/k8s/1_es_deploy.yaml index 953dc4b..2d0f2a9 100644 --- a/k8s/1_es_deploy.yaml +++ b/k8s/1_es_deploy.yaml @@ -43,4 +43,20 @@ spec: path: / port: 9200 initialDelaySeconds: 30 - periodSeconds: 15 \ No newline at end of file + periodSeconds: 15 +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-service + labels: + app: elasticsearch +spec: + type: ClusterIP + selector: + app: elasticsearch + ports: + - name: http + port: 9200 + targetPort: 9200 + protocol: TCP \ No newline at end of file From 03d5b2c3f5f7504dded98f711457c1fcf30c5887 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Thu, 3 Sep 2026 11:17:53 +0200 Subject: [PATCH 09/55] build: upgrade fastmcp to 4.0.0 Drops the unused tasks extra. No source changes were required: the lifespan decorator, tool registration and http_app all survive the major bump. Verified with the full suite plus a live tool call through an in-memory client. --- pyproject.toml | 2 +- uv.lock | 215 ++++++++++++++++--------------------------------- 2 files changed, 71 insertions(+), 146 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 54673d1..0607459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "MCP server for INSEE data diffusion" requires-python = ">=3.12" dependencies = [ - "fastmcp[tasks]==3.4.2", + "fastmcp>=4.0.0", "elasticsearch==9.5.0", "uvicorn==0.52.4", "requests==2.34.2", diff --git a/uv.lock b/uv.lock index c40b373..2e5b4d1 100644 --- a/uv.lock +++ b/uv.lock @@ -20,15 +20,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" }, ] -[[package]] -name = "annotated-doc" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, -] - [[package]] name = "annotated-types" version = "0.8.0" @@ -104,22 +95,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, ] -[[package]] -name = "burner-redis" -version = "0.1.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/89/54706febafc135095b2a9d797cfbd4eed2ab1ad7819808b99b587020471b/burner_redis-0.1.7.tar.gz", hash = "sha256:7474ff092669fd11ef765411572cdafcc3d89b8054aef4ca0617be6d6be4c680", size = 638644, upload-time = "2026-05-08T15:01:42.961Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6c/5d/198bd1d22e504b3034353430703afbdb3efe6e25cb90bf52d896e1d266a7/burner_redis-0.1.7-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f80c866996e0455d584eb3c0f3b067e411c632fb0519eab454e0968edf01e62c", size = 1288888, upload-time = "2026-05-08T15:01:26.103Z" }, - { url = "https://files.pythonhosted.org/packages/2f/4e/ce5c91b884ac37fcd380756402536f8810964014097950900517ce8bd30c/burner_redis-0.1.7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a3d9569a376b690fb5876d454e4904443332dc3ad5c0057e149fc2ad220bf599", size = 1234282, upload-time = "2026-05-08T15:01:28.286Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/31c25cc88143eac2dddcc394151a0db627923d44c94376a83768552c9f13/burner_redis-0.1.7-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20eba1917e3bca9eea5957d5700ff8defcb5a209e57a7841d005549aa0151f44", size = 1337341, upload-time = "2026-05-08T15:01:30.397Z" }, - { url = "https://files.pythonhosted.org/packages/e1/32/95cfa1833316ca2b6b2e58150a4900bc1ad256043cdd36198f1887618ccc/burner_redis-0.1.7-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39111467059b8a28f15ea061d2414ec25c3e57c65759983f90f4d358e7d6a72d", size = 1366800, upload-time = "2026-05-08T15:01:32.891Z" }, - { url = "https://files.pythonhosted.org/packages/34/ad/93c3916f053f89b7b5760da5bf855cd78b7885d480f9cfcc64f3732c1dc2/burner_redis-0.1.7-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9b5adfe99aeb8407f468078f3769b2a63e9168fea12f7709df5d2a3b152706e4", size = 1538160, upload-time = "2026-05-08T15:01:34.667Z" }, - { url = "https://files.pythonhosted.org/packages/5c/b9/19bae42cb124932d71168bc8e5bcb1da33aa62b908e5e632b3d298d7cb15/burner_redis-0.1.7-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:591a9d20685f9d6d22bf0c863b50b12dfcf328b06111b3f62c33cd3185d48ce0", size = 1591491, upload-time = "2026-05-08T15:01:36.708Z" }, - { url = "https://files.pythonhosted.org/packages/f5/30/207f47f406619a5b564355d2946c3171f84231a28b800709b5645b06a5ae/burner_redis-0.1.7-cp310-abi3-win_amd64.whl", hash = "sha256:f6cf4ac666766b32fd63940aad0c120847905fd3102c17e5b6b305f91a21d079", size = 1117564, upload-time = "2026-05-08T15:01:39.221Z" }, - { url = "https://files.pythonhosted.org/packages/76/6f/e9beaf46c5e9fd10dfcdb889ebf7d3aa85142c650c0ab17ab284194f58e1/burner_redis-0.1.7-cp310-abi3-win_arm64.whl", hash = "sha256:458f88feeddfb40a586cc3fcbd8e9384bbdfd2a4512a695af4900e06052570d4", size = 1040407, upload-time = "2026-05-08T15:01:41.235Z" }, -] - [[package]] name = "cachetools" version = "7.1.7" @@ -396,15 +371,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] -[[package]] -name = "cloudpickle" -version = "3.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -428,14 +394,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/38/ce65091ff20a16e06d17418c4353af5f56d3190821b1a06983c79ae79274/courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e", size = 34193, upload-time = "2026-06-01T17:30:14.984Z" }, ] -[[package]] -name = "cronsim" -version = "2.7" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/1a/02f105147f7f2e06ed4f734ff5a6439590bb275a53dd91fc73df6312298a/cronsim-2.7-py3-none-any.whl", hash = "sha256:1e1431fa08c51dc7f72e67e571c7c7a09af26420169b607badd4ca9677ffad1e", size = 14213, upload-time = "2025-10-21T16:38:20.431Z" }, -] - [[package]] name = "cryptography" version = "50.0.0" @@ -603,26 +561,22 @@ wheels = [ [[package]] name = "fastmcp" -version = "3.4.2" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastmcp-slim", extra = ["client", "server"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/29/18/46beaec18c9f86a599ae3f9cdf6677dd6b50240cfd844d18233710b47f13/fastmcp-3.4.2.tar.gz", hash = "sha256:b468722946fc467c3796a6572f7a14d93d48c014cf8fea12910245220cbbe4e1", size = 28756849, upload-time = "2026-06-06T01:30:35.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/02/4f53258f4fb2b88675246a022d1ed9f653d9129c0ddcc40f88479abde455/fastmcp-4.0.0.tar.gz", hash = "sha256:613d925f687609973575039afc6bd8874e60ab373e4f5b8c60f7860897063598", size = 42300863, upload-time = "2026-08-31T18:20:33.564Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/4d/8b1ba42251160e11ca34686344572121432c23a082d56ef6bbdec5888fc1/fastmcp-3.4.2-py3-none-any.whl", hash = "sha256:c87a62b029f0c5400ada85f683629345d2466c39169f0cb853e487b2f7308c08", size = 8018, upload-time = "2026-06-06T01:30:38.118Z" }, -] - -[package.optional-dependencies] -tasks = [ - { name = "fastmcp-slim", extra = ["tasks"] }, + { url = "https://files.pythonhosted.org/packages/24/6a/03160d06bcf2957caf02555d296b343136895e07a1160a6f4b85d0f672af/fastmcp-4.0.0-py3-none-any.whl", hash = "sha256:b041d669971f2325ab41797961bb4e729d1195d0da38d213bfbbcc6ddd65ca75", size = 8077, upload-time = "2026-08-31T18:20:31.342Z" }, ] [[package]] name = "fastmcp-slim" -version = "3.4.2" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "mcp-types" }, { name = "platformdirs" }, { name = "pydantic", extra = ["email"] }, { name = "pydantic-settings" }, @@ -630,16 +584,16 @@ dependencies = [ { name = "rich" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/2e/d627b28b7403ecc526991ef732921b08bde010006e6148635f053fd29f4c/fastmcp_slim-3.4.2.tar.gz", hash = "sha256:290646e0955a516235a317151034559aa48336cb843d3f006131aedad8759bb4", size = 576291, upload-time = "2026-06-06T01:30:12.553Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/b1/8abb7c56159cf817718c1fc6b4547fd0f35cb91f05659ffc1d4f5cee1198/fastmcp_slim-4.0.0.tar.gz", hash = "sha256:b6f78c26e369b4c29b485d7d7b662838d9631e765dc496b4560274762f144e6a", size = 683960, upload-time = "2026-08-31T18:20:09.778Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/58/22afebf18df7260b09148199cbeb90cdcc4b3a4e1b5d7460e3591c3a7add/fastmcp_slim-3.4.2-py3-none-any.whl", hash = "sha256:bdc72492212681ca502755fa8acc0457f559295da1fc3dfc0599adc1c04b82f3", size = 749195, upload-time = "2026-06-06T01:30:11.22Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4b/7bc65d74cc93684ec8b10d49a51fa14c5e4aa3a08120c7001a85bd2a159a/fastmcp_slim-4.0.0-py3-none-any.whl", hash = "sha256:75259ad8033af011f926f4b99cda9ce080b7853f9831d38f2056a392908e11c2", size = 857981, upload-time = "2026-08-31T18:20:07.951Z" }, ] [package.optional-dependencies] client = [ { name = "authlib" }, { name = "exceptiongroup" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "mcp" }, { name = "opentelemetry-api" }, { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, @@ -650,7 +604,7 @@ server = [ { name = "cyclopts" }, { name = "exceptiongroup" }, { name = "griffelib" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "joserfc" }, { name = "jsonref" }, { name = "jsonschema-path" }, @@ -668,9 +622,6 @@ server = [ { name = "watchfiles" }, { name = "websockets" }, ] -tasks = [ - { name = "pydocket" }, -] [[package]] name = "griffelib" @@ -719,6 +670,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -735,12 +699,29 @@ wheels = [ ] [[package]] -name = "httpx-sse" -version = "0.4.3" +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] [[package]] @@ -1050,15 +1031,15 @@ wheels = [ [[package]] name = "mcp" -version = "1.29.0" +version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, + { name = "httpx2" }, { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, { name = "pydantic" }, - { name = "pydantic-settings" }, { name = "pyjwt", extra = ["crypto"] }, { name = "python-multipart" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, @@ -1068,9 +1049,22 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/30/d3/f9acc21dfc886e4f78e2add1a47db46ce16884346afde53f8a064c02c891/mcp-1.29.0.tar.gz", hash = "sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36", size = 643148, upload-time = "2026-07-28T13:41:41.939Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/6e/21fb8e5d579dbe21d96ea4d5034200d46d8bdf2261053b5bd041f3c2f612/mcp-2.1.1.tar.gz", hash = "sha256:50b7ba1ebbe117008ea7bdd288234043e69c20b403d6851d19661e6d431a75ef", size = 3984589, upload-time = "2026-08-25T16:14:02.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/af/8644cc5fa26a59afd2df2e98eeb19e72926887fa4b7441aba4ff661140db/mcp-2.1.1-py3-none-any.whl", hash = "sha256:1c6c31c5d6471c58db76af3af8af67f46d11d01f0a59077d0a308cbdb3d3e915", size = 357912, upload-time = "2026-08-25T16:13:59.024Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/dd/1c4417dc0b722c23a1669032d5f044e41170fe5d4773b488a50fcce98c32/mcp_types-2.1.1.tar.gz", hash = "sha256:77dcbe48fba73cca71a673f2646a5f037a017b7a0a07ac89cec1113028890eda", size = 66674, upload-time = "2026-08-25T16:14:03.861Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/01/c8/248b201f6d753d69fd5d6506011abbb35a946d9142b2ae311a948fd0be3d/mcp-1.29.0-py3-none-any.whl", hash = "sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7", size = 223436, upload-time = "2026-07-28T13:41:40.337Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/242e63c510f4a17381f55b1549a3f94f5687a0595984febd2b6f87a687a0/mcp_types-2.1.1-py3-none-any.whl", hash = "sha256:26f9f7f03f2a5730717a5b98e2ab7eb640ac352d05a00cdc725c311864778295", size = 69656, upload-time = "2026-08-25T16:14:00.667Z" }, ] [[package]] @@ -1080,7 +1074,7 @@ source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, { name = "elasticsearch" }, - { name = "fastmcp", extra = ["tasks"] }, + { name = "fastmcp" }, { name = "httpx" }, { name = "limits" }, { name = "lxml" }, @@ -1102,7 +1096,7 @@ dev = [ requires-dist = [ { name = "beautifulsoup4", specifier = "==4.15.0" }, { name = "elasticsearch", specifier = "==9.5.0" }, - { name = "fastmcp", extras = ["tasks"], specifier = "==3.4.2" }, + { name = "fastmcp", specifier = "==4.0.0" }, { name = "httpx", specifier = "==0.28.1" }, { name = "limits", specifier = ">=5.8.0" }, { name = "lxml", specifier = "==6.1.2" }, @@ -1198,15 +1192,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "prometheus-client" -version = "0.26.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, -] - [[package]] name = "py-key-value-aio" version = "0.4.5" @@ -1231,9 +1216,6 @@ keyring = [ memory = [ { name = "cachetools" }, ] -redis = [ - { name = "redis" }, -] [[package]] name = "pycparser" @@ -1353,30 +1335,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] -[[package]] -name = "pydocket" -version = "0.24.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "burner-redis" }, - { name = "cloudpickle" }, - { name = "cronsim" }, - { name = "opentelemetry-api" }, - { name = "prometheus-client" }, - { name = "py-key-value-aio", extra = ["memory", "redis"] }, - { name = "python-json-logger" }, - { name = "redis" }, - { name = "rich" }, - { name = "typer" }, - { name = "typing-extensions" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, - { name = "uncalled-for" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/6b/a87c6e3fd197807f630af4270aa2ce8f4c1fca4e43bba783f273d298d646/pydocket-0.24.1.tar.gz", hash = "sha256:477d77be1fcfd10ee0c2d0b8aa8c6e97851b9c7f39bb6f2b4e6d42e9b4d6e95a", size = 430759, upload-time = "2026-08-10T19:50:03.368Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/05/4e3b902bc0ca407188aa5fe38af49be634487aec242a77287103242e11b2/pydocket-0.24.1-py3-none-any.whl", hash = "sha256:1faa6c3d566f1f0431e35dfa12db4ccee515d945b913b516ee3e6279afb9e789", size = 130249, upload-time = "2026-08-10T19:50:01.627Z" }, -] - [[package]] name = "pygments" version = "2.21.0" @@ -1459,15 +1417,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] -[[package]] -name = "python-json-logger" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/25/5473e46b179f8e8b4ad3aeeb36773d1701b7770eaf5e5bc2025c7303b598/python_json_logger-4.2.0.tar.gz", hash = "sha256:e371ebe22ec01e289850102091a2b1f6fc9e655c7f1f5f29073936756c290afa", size = 18211, upload-time = "2026-08-15T11:36:38.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/55/6467fde553886cb293e41538f3a8b4e4fd4688c6df242cf982162d8367fb/python_json_logger-4.2.0-py3-none-any.whl", hash = "sha256:158a52126fcd6869e09574d2b66272666f3dc8f468c62637ef9a1fa883719cb9", size = 14988, upload-time = "2026-08-15T11:36:36.821Z" }, -] - [[package]] name = "python-multipart" version = "0.0.32" @@ -1560,15 +1509,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] -[[package]] -name = "redis" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, -] - [[package]] name = "referencing" version = "0.37.0" @@ -1813,23 +1753,14 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "sys_platform != 'win32'" }, - { name = "jeepney", marker = "sys_platform != 'win32'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, ] -[[package]] -name = "shellingham" -version = "1.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, -] - [[package]] name = "six" version = "1.17.0" @@ -1911,18 +1842,12 @@ wheels = [ ] [[package]] -name = "typer" -version = "0.27.1" +name = "truststore" +version = "0.10.4" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ae/40/4a3db7990d1f62a53182aa96eaef57aeb2886a27f90a195bc66713565d31/typer-0.27.1.tar.gz", hash = "sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df", size = 203994, upload-time = "2026-08-03T14:41:03.438Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] [[package]] From 8ba64667335bc429f8c03abb5bed3b8c1b8a72b3 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Thu, 3 Sep 2026 11:18:07 +0200 Subject: [PATCH 10/55] chore: add the claude code harness Project instructions in CLAUDE.md, topic rules under .claude/rules/ (python, git, error handling, logging), shared permissions in .claude/settings.json, and the FastMCP documentation MCP server in .mcp.json so contributors get it without local setup. Personal and reference files stay out of git: settings.local.json, TODO.md and CLAUDE-example.md are ignored. --- .claude/rules/error.md | 42 ++++++ .claude/rules/git.md | 49 +++++++ .claude/rules/logging.md | 58 ++++++++ .claude/rules/python.md | 45 ++++++ .../{settings.local.json => settings.json} | 5 +- .gitignore | 4 +- .mcp.json | 8 + CLAUDE.md | 137 ++++++++++++++++++ 8 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 .claude/rules/error.md create mode 100644 .claude/rules/git.md create mode 100644 .claude/rules/logging.md create mode 100644 .claude/rules/python.md rename .claude/{settings.local.json => settings.json} (54%) create mode 100644 .mcp.json create mode 100644 CLAUDE.md diff --git a/.claude/rules/error.md b/.claude/rules/error.md new file mode 100644 index 0000000..341baf8 --- /dev/null +++ b/.claude/rules/error.md @@ -0,0 +1,42 @@ +# Error handling + +Errors are part of the tool contract: the caller is an LLM, so an error must tell it what to do next. +A failed call returns the same result shape as a successful one, with `is_error` set. `is_error` is a flag, +not a code, and every documented client path reads the reason from `content[0].text`. The message is the +error contract — consistency means a consistent message, produced in one place. + +## Where errors live + +- `core/errors.py` owns every error type. Nothing else defines an error enum, model or vocabulary. +- Error types subclass `ToolError` — its message always reaches the client. Anything else is an internal + fault and must not leak. +- Code and retryability are attributes on the exception, never formatted into the message. + +## Raising + +- Services raise. Tools do not build error messages. +- Raise the narrowest type that fits. +- Input validation belongs in the Pydantic model, not a runtime check in the tool. +- An empty result is not an error. Return an empty list and a count. + +## Message content + +Name the backend and operation that failed, the shortest useful excerpt of the upstream error, and the next +step — the offending parameter, or the tool that produces a valid value. Never a stack trace or a full body. + +## Catching + +- Catch the narrowest upstream exception you can name. Never `except Exception` or `except BaseException` + (it swallows `CancelledError`). +- Translate once, at the boundary owning the dependency. Never re-wrap an already typed error. +- Never swallow: no empty `except`, no default on failure, no log-and-continue. +- Chain with `raise ... from exc`. + +## Use what FastMCP provides + +- Set `mask_error_details=True` on the `FastMCP` instance. It defaults to `False`, which sends every raw + exception message to the client. With it on, the call still fails visibly but unexpected exceptions carry a + generic message; `ToolError` subclasses keep theirs. +- `ErrorHandlingMiddleware` catches, logs and converts every exception. Register it first, so it sees the + rest of the chain. Failures are logged there, not by the code that raises — see `logging.md`. +- `RetryMiddleware` handles transient failures with backoff. Do not write a retry loop. diff --git a/.claude/rules/git.md b/.claude/rules/git.md new file mode 100644 index 0000000..cfbd06b --- /dev/null +++ b/.claude/rules/git.md @@ -0,0 +1,49 @@ +# Git + +Commits exist to produce a readable changelog. The existing history does not follow these rules — +do not imitate it. + +## Committing + +- Never commit or push unless asked. +- One concern per commit. If the subject needs "and", it is two commits. + +## Message format + +[Conventional Commits](https://www.conventionalcommits.org): `type(scope): subject`, with the scope +taxonomy below. + +- Changelog types: `feat` (new capability), `fix` (something broken for a user now works). +- Silent types: `refactor`, `perf`, `test`, `docs`, `build`, `ci`, `chore`. +- Pick the type by what the line would say in release notes. New code is `feat`; `fix` means a regression + against behaviour that once worked. +- Subject: imperative, lowercase, no trailing period. Say what the change gives a user, not which files moved. + +### Scope + +- User-facing changes take the data source: `insee`, `melodi`, `rmes`. +- Internal changes take the module area: `server`, `tools`, `services`, `config`, `core`, `docker`, `ci`. +- One scope per commit — where the capability lives, not every directory touched. +- Two data sources gaining independent capability is two commits. +- A cross-cutting change with no primary home takes no scope. Never comma-separate scopes. + +## Branches and merging + +- Branch off `main`. Never commit to `main` directly. +- Branches squash-merge. Branch commits may be WIP; the squash subject becomes the changelog line and + describes the whole branch, not its final commit. + +## Breaking changes + +- Mark with `!` plus a `BREAKING CHANGE:` footer stating the migration: + `feat(melodi)!: rename the dataset filter argument`. +- Breaking here means the **tool contract** changed: a renamed tool, a changed schema, a reordered workflow. + Connected clients keep calling the old shape and fail silently. A description rewrite is not breaking. + +## Versions + +- Never edit `version` in `pyproject.toml` in a feature or fix commit. +- Versions and `CHANGELOG.md` are derived from commit history at release time, in a separate + `chore(release):` commit. Do not hand-write either. +- CI builds and pushes an image on every push to `main` and on `v*` tags. A tag is a release action — + never push one casually. diff --git a/.claude/rules/logging.md b/.claude/rules/logging.md new file mode 100644 index 0000000..4348c18 --- /dev/null +++ b/.claude/rules/logging.md @@ -0,0 +1,58 @@ +# Logging + +Two channels, two audiences. Confusing them is the common mistake. + +- **Client logging** — `ctx.debug/info/warning/error()`. Travels to the MCP client over the protocol. + Audience: the calling LLM and the person watching it. +- **Server logging** — Python `logging`. Goes to stdout and the aggregator. Audience: whoever is on call. + +Never send the same message to both. + +## Client logging + +- Use it to narrate a call so the model can react: which index was searched, why a result set came back + empty, which filter was ignored. +- Never use it to report failure. `ctx.error()` does not fail the call — raise a `ToolError` instead. +- Never send credentials, connection strings or upstream response bodies. This leaves the process. +- It is async: await it. Each call is a protocol notification, so never put one inside a loop over results. +- Structured data goes in `extra=`, not formatted into the message. +- It needs a `Context`, so it exists only during a request. + +## Use the middleware + +- Never reimplement what the middleware provides: `LoggingMiddleware` (human-readable), + `StructuredLoggingMiddleware` (JSON for aggregation), `TimingMiddleware` and `DetailedTimingMiddleware` + (durations), `ErrorHandlingMiddleware` (exceptions). A per-tool decorator that logs entry, duration and + errors is one of these. +- `LoggingMiddleware(include_payloads=...)` truncates, it does not redact. Leave it `False` unless a custom + `logger` with a redacting filter is in place. +- Register logging middleware last, so it records execution after the rest of the chain has run. + +## Who logs what + +- **Middleware logs failures, not services.** `ErrorHandlingMiddleware` already catches, logs and converts + every exception. Code that logs before raising records the same failure twice. +- A service logs only what the exception cannot carry, and never at `error` level. +- Never log credentials, tokens or request bodies. + +## Configuration + +- Configure logging once at startup, never at import time. `logging.basicConfig` in a module body fires as + a side effect of importing that module and cannot be overridden by the process hosting the app. +- The configuration must apply whether the server runs via `__main__` or under an external ASGI server. +- Logger names follow the module: `logging.getLogger(__name__)`. Never a hand-picked shared name. + +## Where the channels meet + +Everything sent with `ctx.log()` is also written to the server log at `DEBUG` on the +`fastmcp.server.context.to_client` logger. Enable it to audit what clients were told — do not log the same +message twice yourself. + +## Protocol notes + +- Log messages are one-way notifications, so they always reach the client. (Two-way features like sampling + were removed from the protocol; logging was not.) +- Ignore the SDK's `MCPDeprecationWarning` about the logging capability. It is about the handshake, not the + messages. They still arrive. +- The client decides which levels it keeps. `logging/setLevel` no longer works, so never rely on the server + filtering levels for a client. diff --git a/.claude/rules/python.md b/.claude/rules/python.md new file mode 100644 index 0000000..d332c82 --- /dev/null +++ b/.claude/rules/python.md @@ -0,0 +1,45 @@ +# Python conventions + +## Functions and side effects + +Default to pure: same arguments in, same value out. + +- Push I/O to the edges. Parsing, filtering, query building and result shaping stay pure. +- Never read a global inside a function — no `get_settings()`, no module-level client or cache. +- Never mutate an argument. Return a new value. +- No I/O and no clock reads at import time. +- Take what you need as a parameter, keyword-only unless it is the subject of the call. Pass specific + values, never a whole configuration object. + +## Typing and syntax + +- Target Python 3.12. +- `str | None`, never `Optional[str]`. Never mix both styles. +- Annotate every return, including `-> None`. A function that always raises returns `NoReturn`. +- Alias a composed type you repeat: `TableOfContents = list[dict[str, str]]`. + +## Async + +- Async for anything doing I/O. +- Never call a blocking API inside a coroutine. If it cannot be avoided, tell me before writing it. + +## Naming + +Spell names out. The reader should not have to look up what something holds. + +- `settings`, not `s`. `elasticsearch_client`, not `es`. +- Name what an argument is: `search_input`, not `params`. +- A name broader than the behaviour is misleading. +- Functions start with a verb that describes the actual work: `fail()` always raises, so `raise_tool_error()`. +- `get_` is for cheap in-memory lookups. Anything doing I/O is `fetch_`, `load_` or `search_`. + +## Layout + +- Dicts, lists and other objects: multiline, one entry per line, including as call arguments. +- Signatures and calls with two or more arguments: multiline, one per line. +- Calls with more than two arguments name each one. Positional only where keywords are forbidden + (`getattr`, `dict`, `join`). +- Imports at the top of the module, never inside a function. +- Text spanning more than one line is a triple-quoted string, `textwrap.dedent`-ed when indented. +- Never glue adjacent literals or chain `+`: implicit concatenation drops the space at line breaks. +- Long text is data. Keep it out of `if` branches and out of multi-source assembly. diff --git a/.claude/settings.local.json b/.claude/settings.json similarity index 54% rename from .claude/settings.local.json rename to .claude/settings.json index 1a8165b..7bbed8f 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.json @@ -3,5 +3,8 @@ "allow": [ "Bash(uv run:*)" ] - } + }, + "enabledMcpjsonServers": [ + "fastmcp-docs" + ] } diff --git a/.gitignore b/.gitignore index ed6b96f..1f1d4ea 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ *.env __pycache__/ mcp_* -.idea \ No newline at end of file +.idea +.claude/settings.local.json +TODO.md diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..3c78f74 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "fastmcp-docs": { + "type": "http", + "url": "https://gofastmcp.com/mcp" + } + } +} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ff89097 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,137 @@ +# Project instructions + +This file may be updated during the refactoring process. + +## What this repo is + +McpDiffusion is an **MCP server that exposes INSEE public data to LLM clients**. +It puts three INSEE sources behind one HTTP MCP endpoint: + +| Source | Access | Content | +|----------|------------------------------------------|------------------------------------------------------------------------| +| insee.fr | Elasticsearch index + live HTML scraping | publications, *Informations rapides*, key figures, homepage indicators | +| MELODI | Elasticsearch index + REST API | dataset catalogue and observations | +| RMES | SPARQL (`rdf.insee.fr`) | definitions, classifications, metadata (no figures) | + +Single Python package under `src/mcpdiffusion/`, managed with `uv`, built on **FastMCP 4**. +`docs/project.md` is the long-form overview. + +Elasticsearch is required for the insee.fr and MELODI tools; only RMES works without it. This repo contains +no indexing code — it only reads. The index ships as a prebuilt snapshot from outside the repo. + +`SKILL.md` is a usage guide written **for the LLM client**, not for maintainers. It describes tool names and +workflows, so any change to those makes it wrong — tell me when that happens. It gets regenerated from the +code once the refactor settles, so do not patch it as you go. + +## Current mission + +Make this project production-ready. + +- **Adopt the FastMCP 4 APIs, not just the version.** + - The version bump has been done. It does not mean the code is idiomatic of the v4 patterns + - Apply more recent and adapted patterns when possible and notify me previously + - **Review the MCP layer against the official docs** — tool declaration, descriptions, lifespan, context, + dependency injection, middleware, error handling. Some of it may not follow current FastMCP recommendations. +- **When you see overengineering** — something hand-rolled that FastMCP already provides — notify, plan, + and propose an accurate correction. Do not silently rewrite it. This applies not only for fastmcp but at the whole + project source scale +- **Fix the identified issues.** Bugs and review comments are marked `# Fixme:` in the source. Also fix any + other bug you find, and say what you found. + +Verified against `fastmcp-docs` — these are confirmed, not guesses: + +- **Tool descriptions belong in docstrings.** FastMCP parses the docstring for both the tool description and + every parameter description (Google/NumPy/Sphinx). `Annotated[x, "..."]` and `Field(description=...)` take + precedence, so adoption can be incremental. `config/tool_metadata.py` is largely redundant. + → `/servers/tools#docstring-descriptions` +- **The rate limiter is built in.** `RateLimitingMiddleware` (token bucket) and + `SlidingWindowRateLimitingMiddleware` (precise window, no burst) both accept `get_client_id` for per-client + keying. `core/middleware.py` reimplements this, and the `limits` dependency goes with it. + → `/servers/middleware#rate-limiting` +- **Host protection is built in.** `mcp.http_app(host_origin_protection=True, allowed_hosts=[...], + allowed_origins=[...])` replaces the hand-wired `TrustedHostMiddleware` in `server.py`. + → `/deployment/http.mdx` +- **`mcp.http_app()` is current.** It also takes `middleware=` for ASGI middleware. No change needed. +- **`ctx.lifespan_context` is current** — the documented way to reach shared clients, exactly as `infra/` does + it. `fastmcp.dependencies.Depends` is a *different* tool (hiding parameters from the LLM schema), not a + replacement. The `# Fixme:` about those accessors being untyped still stands; "four files go away" does not. + → `/servers/lifespan#accessing-lifespan-context` + +## Main commands + +All commands run from the repo root. + +```bash +uv sync # install (dev deps included) +uv run python -m mcpdiffusion.server # run the server locally, needs ES_HOST +uv run pytest -q # test suite — do not trust it, see Hard rules + +docker build -t mcp-insee . +docker compose -f docker-compose-dev.yaml up # server + MCP Inspector (needs the `elastic` network) +``` + +## Hard rules + +- **The `fastmcp-docs` MCP server is connected and available right now. Use it.** Never answer a FastMCP + question, and never write or change FastMCP code, from memory. v4 is recent and moved things, so a + remembered API is more likely wrong than right. Look it up first, every time — including when you are + confident. If a lookup contradicts what you were about to write, the docs win. +- **The markdown documentation is out of date. Never base a change on it.** `README.md` and `SKILL.md` + describe the pre-refactor code — wrong layout, wrong tool count, wrong tool names. `docs/project.md` is + the most accurate but still not authoritative. **The code is the only source of truth.** Read a `.md` to + learn intent, never to learn behaviour. They get regenerated once the refactor settles; until then report + a mismatch, never silently follow it. +- **All configuration is typed in `config/settings.py`.** No magic numbers, URLs, timeouts or limits in the + code, and nothing read from the environment anywhere else. +- **A setting is not done until the documented configuration surface changes in the same commit.** The example + env file is the only description of what this server can be configured with — how the values actually reach + the process (shell, compose `env_file`, k8s `env:`) does not change that. +- **Keep every place that names a setting in sync**: the example env file, the env file + `docker-compose-dev.yaml` expects, and the `env:` block in `k8s/`. A variable set in a manifest that + `Settings` no longer reads is a bug, not leftovers. Touching `k8s/` for this is expected — it is the + exception to the rule below. +- Ask before adding a dependency, a new tool, or a new data source. +- Never commit to `src/mcpdiffusion/feedback/feedback.md`. It is user-submitted content. +- Do not touch `k8s/` or `.github/workflows/` unless the task is about deployment. +- Do not fix a `# Fixme:` by deleting the comment without changing the code. If the comment turns out to be + wrong or irrelevant, say so and ask before removing it. +- **Do not rely on the existing tests.** They were auto-generated and never reviewed. Verify your own work + (see below). +- There is no linter, formatter or type checker configured. Do not assume a command exists; propose one first. + +## Application layer + +**The current code is not a reference. Do not copy a pattern just because you found it in the repo** — several +files predate any convention. + +Targeted conventions, to be confirmed or infirmed as we go. Plan and propose better ones freely: + +- `tools/` — declares the MCP tools. Wires, does not compute. +- `services/` — orchestration and business logic. +- `repositories/` — data access: Elasticsearch queries, HTTP calls, SPARQL. *(proposed; does not exist yet — + these currently live in `services/`)* +- `clients/` — builds and provides the shared Elasticsearch / HTTP / SPARQL clients. *(currently `infra/`; + the accessors stay — `ctx.lifespan_context` is the documented API — but they need real typing)* +- `config/` — settings only. Clients are live objects with a lifecycle; settings are static values. Keep them apart. +- `core/` — cross-cutting concerns that belong to no single source: error types, logging setup, middleware. + +This section gets adjusted as we settle on FastMCP patterns and conventions. + +## Verifying a change + +No test imports `server.py`, so the app, middlewares, lifespan and tool registration are never exercised by +the suite. A green suite does not mean the server boots: + +```bash +ES_HOST=http://localhost:9200 uv run python -c " +import asyncio +from fastmcp import Client +from mcpdiffusion import server +async def main(): + async with Client(server.mcp) as c: + print([t.name for t in await c.list_tools()]) +asyncio.run(main())" +``` + +A pass prints the tool list **and exits 0**. `FastMCP.get_tools()` no longer exists in v4 — list tools +through an in-memory `Client` as above. From c1339e6af7114f9a5df5f2e2d1ab4563ea3d3ca8 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Thu, 3 Sep 2026 22:44:52 +0200 Subject: [PATCH 11/55] docs(claude): add the business rule marker convention `# Fixme:` is ours to fix; `# Business rule:` marks a question only the owner of the search and data semantics can answer. Preserve current behaviour and flag it rather than deciding. --- CLAUDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index ff89097..25fd978 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,6 +95,9 @@ docker compose -f docker-compose-dev.yaml up # server + MCP Inspector (needs th - Do not touch `k8s/` or `.github/workflows/` unless the task is about deployment. - Do not fix a `# Fixme:` by deleting the comment without changing the code. If the comment turns out to be wrong or irrelevant, say so and ask before removing it. +- **Two markers, two meanings.** `# Fixme:` is ours to fix. `# Business rule:` marks a question only whoever + owns the search and data semantics can answer — preserve the current behaviour, flag it, and never decide + it yourself. Reclassifying one as the other needs my agreement. - **Do not rely on the existing tests.** They were auto-generated and never reviewed. Verify your own work (see below). - There is no linter, formatter or type checker configured. Do not assume a command exists; propose one first. From d7c0e95bc4bf64849d742d3e97c26f25075f06ef Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Thu, 3 Sep 2026 22:45:20 +0200 Subject: [PATCH 12/55] refactor(server): build settings once at startup and inject narrow values Settings are constructed in server.py and passed down. `get_settings()` is gone: no service or client reaches for configuration, each takes the specific value it needs. Elasticsearch queries no longer block the event loop. Replaces hand-rolled infrastructure with what FastMCP already ships: - core/middleware.py -> SlidingWindowRateLimitingMiddleware, keyed per client - @log_tool -> LoggingMiddleware + TimingMiddleware + ErrorHandlingMiddleware - TrustedHostMiddleware -> http_app(host_origin_protection=...) Fixes found on the way: - the title relevance boost was discarded by every INSEE search tool, which overwrote its `should` clauses with the collection filters - _truncate could compute a negative tail, and a zero-length tail appended the whole document instead of nothing - the server crashed on shutdown, awaiting a synchronous close() - `must_not` was threaded through three tools while always empty Configuration is not backward compatible. Renamed: TLS_VERIFY -> ES_TLS_VERIFY, GLOBAL_REQUEST_MIN -> RATE_LIMIT_MAX_REQUESTS, FORWARDED_ALLOW_IPS -> TRUSTED_PROXY_HOSTS, ENABLE_* -> ENABLE_*_TOOLS. ALLOWED_HOSTS and TRUSTED_PROXY_HOSTS now take JSON lists. TZ and ES_HOST_LOCAL are removed, nothing read them. TRUSTED_PROXY_HOSTS defaults to 127.0.0.1 rather than "*", so a deployment behind a proxy must name it or every client shares one rate-limit bucket. --- k8s/3_mcp_deploy.yaml | 2 - pyproject.toml | 3 +- src/mcpdiffusion/.env.example | 55 +- src/mcpdiffusion/config/settings.py | 108 ++-- src/mcpdiffusion/core/logging.py | 173 +----- src/mcpdiffusion/core/middleware.py | 72 --- src/mcpdiffusion/core/rate_limiting.py | 26 + src/mcpdiffusion/infra/elasticsearch.py | 9 +- src/mcpdiffusion/infra/http.py | 12 +- src/mcpdiffusion/infra/lifespan.py | 134 ++-- src/mcpdiffusion/infra/sparql.py | 8 +- src/mcpdiffusion/models/rmes.py | 8 +- src/mcpdiffusion/server.py | 87 ++- src/mcpdiffusion/services/insee_document.py | 43 +- src/mcpdiffusion/services/insee_search.py | 65 +- src/mcpdiffusion/services/melodi.py | 36 +- src/mcpdiffusion/services/rmes.py | 47 +- src/mcpdiffusion/tools/__init__.py | 33 +- .../tools/extras_send_feedback.py | 2 - src/mcpdiffusion/tools/insee_get_document.py | 6 +- src/mcpdiffusion/tools/insee_get_homepage.py | 2 - .../tools/insee_search_chiffrecle.py | 19 +- .../tools/insee_search_conjoncture.py | 24 +- .../tools/insee_search_documents.py | 20 +- .../tools/melodi_get_observations.py | 6 +- .../tools/melodi_search_datasets.py | 12 +- .../tools/melodi_search_modalities.py | 12 +- .../tools/rmes_describe_resource.py | 12 +- src/mcpdiffusion/tools/rmes_list_graphs.py | 12 +- src/mcpdiffusion/tools/rmes_run_sparql.py | 12 +- uv.lock | 581 +++++++++++++++--- 31 files changed, 956 insertions(+), 685 deletions(-) delete mode 100644 src/mcpdiffusion/core/middleware.py create mode 100644 src/mcpdiffusion/core/rate_limiting.py diff --git a/k8s/3_mcp_deploy.yaml b/k8s/3_mcp_deploy.yaml index ed7105c..6f52813 100644 --- a/k8s/3_mcp_deploy.yaml +++ b/k8s/3_mcp_deploy.yaml @@ -45,8 +45,6 @@ spec: env: - name: ES_HOST value: "http://elasticsearch-service:9200" - - name: ES_HOST_LOCAL - value: "http://elasticsearch-service:9200" # Optional resources block – adjust as needed resources: limits: diff --git a/pyproject.toml b/pyproject.toml index 0607459..5b1dd14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "MCP server for INSEE data diffusion" requires-python = ">=3.12" dependencies = [ "fastmcp>=4.0.0", - "elasticsearch==9.5.0", + "elasticsearch[async]==9.5.0", "uvicorn==0.52.4", "requests==2.34.2", "beautifulsoup4==4.15.0", @@ -15,7 +15,6 @@ dependencies = [ "python-dotenv==1.2.3", "pydantic-settings>=2.0.0", "trafilatura==2.2.0", - "limits>=5.8.0", ] [build-system] diff --git a/src/mcpdiffusion/.env.example b/src/mcpdiffusion/.env.example index 6092ff6..9e10203 100644 --- a/src/mcpdiffusion/.env.example +++ b/src/mcpdiffusion/.env.example @@ -1,18 +1,47 @@ -# Server -MCP_HOST="0.0.0.0" -MCP_PORT="8000" +# Every variable this server reads. Only ES_HOST is required; the values shown are the defaults. +# +# This file is resolved from the current working directory, not from this package. Docker and +# Kubernetes inject real environment variables instead and never read it. -# Elasticsearch -- single endpoint variable. +# HTTP server ---------------------------------------------------------------------------------------------------------- +MCP_HOST=0.0.0.0 +MCP_PORT=8000 +# JSON list of hosts this server answers to. +# "*" accepts any host and is unsafe once the server is publicly reachable. +ALLOWED_HOSTS=["*"] +# JSON list of peers whose X-Forwarded-For header is believed. Set this to your reverse proxy. +# Accepts addresses, CIDR networks and literals. Widening it lets any caller forge their own +# address, which defeats per-client rate limiting. +TRUSTED_PROXY_HOSTS=["127.0.0.1"] + +# Tool selection ------------------------------------------------------------------------------------------------------- +ENABLE_INSEEFR_TOOLS=true +ENABLE_MELODI_TOOLS=true +ENABLE_RMES_TOOLS=true + +# Elasticsearch -------------------------------------------------------------------------------------------------------- # Inside Docker use the service name; on the host use localhost. -ES_HOST="http://localhost:9200" +ES_HOST=http://localhost:9200 +ES_INDEX_PRODUITS=produit +ES_INDEX_MELODI_DATASETS=melodi_datasets +ES_INDEX_MELODI_COLUMNS=melodi_columns +# Only Elasticsearch is configurable here. +ES_TLS_VERIFY=true +ES_REQUEST_TIMEOUT_SECONDS=30 -# TLS verification for outbound HTTPS calls (insee.fr scraping + ES over TLS). -# Default is true. Set to "false" when hitting a self-signed / internal endpoint. -TLS_VERIFY="true" +# INSEE services ------------------------------------------------------------------------------------------------------- +INSEE_BASE_URL=https://www.insee.fr +INSEE_REQUEST_TIMEOUT_SECONDS=30 +INSEE_CONNECT_TIMEOUT_SECONDS=10 +MELODI_DATA_BASE_URL=https://api.insee.fr/melodi/data +MELODI_REQUEST_TIMEOUT_SECONDS=30 +MELODI_CONNECT_TIMEOUT_SECONDS=10 +# RMES takes its timeout per query, from the tool's own input. +RMES_ENDPOINT=https://rdf.insee.fr/sparql -# Application log level: DEBUG, INFO, WARNING, ERROR, CRITICAL. -LOG_LEVEL="INFO" +# Rate limiting -------------------------------------------------------------------------------------------------------- +RATE_LIMIT_MAX_REQUESTS=100 +RATE_LIMIT_WINDOW_MINUTES=1 -TOOLLIST="ALL" -# Rate limiter params -GLOBAL_REQUEST_MIN=100 \ No newline at end of file +# Logging -------------------------------------------------------------------------------------------------------------- +LOG_LEVEL=INFO diff --git a/src/mcpdiffusion/config/settings.py b/src/mcpdiffusion/config/settings.py index 3afcf92..708be4e 100644 --- a/src/mcpdiffusion/config/settings.py +++ b/src/mcpdiffusion/config/settings.py @@ -1,70 +1,54 @@ -"""Centralized application settings validated at import time via Pydantic.""" +"""Every value this server can be configured with.""" -from functools import lru_cache -# Fixme: prefer more recent syntax - ex: str | None instead of Optional -from typing import Optional - -from pydantic import Field -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): - # Elasticsearch - # Fixme: if the environment variable name matches the variable name, there is no need for an alias - es_host: Optional[str] = Field(default=None, alias="ES_HOST") - es_index_produits: str = Field(default="produit", alias="ES_INDEX_PRODUITS") - es_index_melodi_datasets: str = Field( - default="melodi_datasets", alias="ES_INDEX_MELODI_DATASETS" - ) - es_index_melodi_columns: str = Field( - default="melodi_columns", alias="ES_INDEX_MELODI_COLUMNS" + # HTTP server ------------------------------------------------------------------------------------------------------ + mcp_host: str = "0.0.0.0" + mcp_port: int = 8000 + # JSON list. "*" accepts any host and is unsafe once the server is publicly reachable. + allowed_hosts: list[str] = ["*"] + # Peers whose X-Forwarded-For header is believed; anything else keeps its real socket address. + # Accepts addresses, CIDR networks and literals. Widening this lets callers forge their own address. + trusted_proxy_hosts: list[str] = ["127.0.0.1"] + + # Tool selection --------------------------------------------------------------------------------------------------- + enable_inseefr_tools: bool = True + enable_melodi_tools: bool = True + enable_rmes_tools: bool = True + + # Elasticsearch ---------------------------------------------------------------------------------------------------- + es_host: str + es_index_produits: str = "produit" + es_index_melodi_datasets: str = "melodi_datasets" + es_index_melodi_columns: str = "melodi_columns" + # Elasticsearch is often internal with a self-signed certificate. + es_tls_verify: bool = True + es_request_timeout_seconds: int = 30 + + # INSEE services --------------------------------------------------------------------------------------------------- + insee_base_url: str = "https://www.insee.fr" + insee_request_timeout_seconds: int = 30 + insee_connect_timeout_seconds: int = 10 + melodi_data_base_url: str = "https://api.insee.fr/melodi/data" + melodi_request_timeout_seconds: int = 30 + melodi_connect_timeout_seconds: int = 10 + # RMES takes its timeout per query, from the tool's own input. + rmes_endpoint: str = "https://rdf.insee.fr/sparql" + + # Rate limiting ---------------------------------------------------------------------------------------------------- + rate_limit_max_requests: int = 100 + rate_limit_window_minutes: int = 1 + + # Logging ---------------------------------------------------------------------------------------------------------- + log_level: str = "INFO" + + model_config = SettingsConfigDict( + env_file=".env", + extra="ignore", ) - # TLS - tls_verify: bool = Field(default=True, alias="TLS_VERIFY") - - # Server - mcp_host: str = Field(default="0.0.0.0", alias="MCP_HOST") - mcp_port: int = Field(default=8000, alias="MCP_PORT") - allowed_hosts: str = Field(default="*", alias="ALLOWED_HOSTS") - # Fixme: some variables are missing from the '.env.example' file - forwarded_allow_ips: str = Field(default="*", alias="FORWARDED_ALLOW_IPS") - - # Rate limiting - global_request_min: int = Field(default=100, alias="GLOBAL_REQUEST_MIN") - tz: str = Field(default="Europe/Paris", alias="TZ") - - # Logging - log_level: str = Field(default="INFO", alias="LOG_LEVEL") - - # Tool selection - enable_melodi : bool = Field(default=True, alias = "ENABLE_MELODI") - enable_inseefr : bool = Field(default=True, alias = "ENABLE_INSEEFR") - enable_rmes : bool = Field(default=True, alias = "ENABLE_RMES") - - - # RMES / SPARQL - rmes_endpoint: str = Field( - default="https://rdf.insee.fr/sparql", alias="RMES_ENDPOINT" - ) - - # Melodi - melodi_data_base_url: str = Field( - default="https://api.insee.fr/melodi/data", alias="MELODI_DATA_BASE_URL" - ) - - # INSEE.fr - insee_base_url: str = Field( - default="https://www.insee.fr", alias="INSEE_BASE_URL" - ) - - # Fixme: this is just a preference for reading but multiline objects reads better - # I also think 'populate_by_name' can be ignored if we get rid of aliases - # Eventually 'SettingsConfigDict' is better for config than a plain dict since it catches typo'd key - # Beware .env file resolves relative to the current working directory - model_config = {"env_file": ".env", "extra": "ignore", "populate_by_name": True} - -@lru_cache -def get_settings() -> Settings: +def load_settings() -> Settings: return Settings() diff --git a/src/mcpdiffusion/core/logging.py b/src/mcpdiffusion/core/logging.py index c3da87b..bf8bde1 100644 --- a/src/mcpdiffusion/core/logging.py +++ b/src/mcpdiffusion/core/logging.py @@ -1,145 +1,32 @@ -"""Structured logging config + per-tool decorator.""" +"""Logging configuration, applied once at startup.""" -import functools -import inspect import logging -import time -from typing import Any, Callable, TypeVar - -from ..config.settings import get_settings - -_settings = get_settings() - -# Fixme: this is a trade-off to make log fall under the same logger name, I would not recommend it -# a convention is to use the module name for identification -MAIN_LOGGER_NAME = "mcp.main" - -logging.basicConfig( - level=_settings.log_level, - # Fixme: the following format string is duplicated - format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", - force=True, -) - -# Fixme: 'UVICORN_LOGGING_CONFIG' is leveraged in the server.py main block -# but this block is not always ran, especially when the app is launched using uvicorn -# this prevents the log level set from being applied to uvicorn logs -# I'd suggest unifying config in a single place to invoke it systematically -# Also, this config competes with the one above -UVICORN_LOGGING_CONFIG = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "default": { - "format": "%(asctime)s | %(levelname)s | %(name)s | %(message)s" - } - }, - "handlers": { - "default": { - "class": "logging.StreamHandler", - "formatter": "default", - } - }, - "root": { - "level": _settings.log_level, - "handlers": ["default"], - }, -} - -TOOLS_LOGGER_NAME = "mcp.tools" -logger = logging.getLogger(TOOLS_LOGGER_NAME) - - -_F = TypeVar("_F", bound=Callable[..., Any]) - -_SCRUB_FIELDS = {"password", "mdp", "token", "secret", "auth", "api_key"} -_KWARGS_PREVIEW_LIMIT = 800 - - -# Fixme: only single level items are scrubbed -def _scrub(kwargs: dict) -> str: - safe = {} - for k, v in kwargs.items(): - if any(s in k.lower() for s in _SCRUB_FIELDS): - safe[k] = "***" - else: - safe[k] = v - text = repr(safe) - if len(text) > _KWARGS_PREVIEW_LIMIT: - return text[:_KWARGS_PREVIEW_LIMIT] + "..." - return text - - -def _result_count(result: Any) -> int | None: - if result is None: - return 0 - if isinstance(result, (list, tuple)): - return len(result) - if isinstance(result, dict): - if "results" in result and isinstance(result["results"], list): - return len(result["results"]) - if "count" in result: - return result["count"] - r = getattr(result, "results", None) - if isinstance(r, list): - return len(r) - return None - - -def log_tool(func: _F) -> _F: - """Decorator that logs entry, exit (duration + count) and errors.""" - is_async = inspect.iscoroutinefunction(func) - name = func.__name__ - - def _log_exit(duration_ms: float, result: Any) -> None: - count = _result_count(result) - if count is None: - # Fixme: prefer using the extra key to provide additional elements to log instead of information - # concatenated in a textual prose - logger.info("Tool exit: %s | %.1fms", name, duration_ms) - else: - logger.info( - "Tool exit: %s | %.1fms | count=%d", name, duration_ms, count - ) - - def _log_error(duration_ms: float, exc: BaseException) -> None: - # Fixme: the exception encapsulated within 'exc' is not leveraged fully, - # the stack is missing which is critical information to log - code = getattr(exc, "args", ("",))[0] if exc.args else type(exc).__name__ - logger.error( - "Tool error: %s | %.1fms | %s: %s", - name, duration_ms, type(exc).__name__, str(code)[:200], - ) - - # Fixme: this code can be simplified, especially when only a few lines differ per outcome - if is_async: - @functools.wraps(func) - async def async_wrapper(*args, **kwargs): - logger.info("Tool call: %s | kwargs=%s", name, _scrub(kwargs)) - start = time.perf_counter() - try: - result = await func(*args, **kwargs) - # Fixme: this is very broad exception handling - # Indeed, this also catches KeyboardInterrupt, SystemExit, and asyncio.CancelledError - except BaseException as exc: - _log_error((time.perf_counter() - start) * 1000, exc) - raise - _log_exit((time.perf_counter() - start) * 1000, result) - return result - wrapper: Callable[..., Any] = async_wrapper - else: - @functools.wraps(func) - def sync_wrapper(*args, **kwargs): - logger.info("Tool call: %s | kwargs=%s", name, _scrub(kwargs)) - start = time.perf_counter() - try: - result = func(*args, **kwargs) - except BaseException as exc: - _log_error((time.perf_counter() - start) * 1000, exc) - raise - _log_exit((time.perf_counter() - start) * 1000, result) - return result - wrapper = sync_wrapper - - wrapper.__signature__ = inspect.signature(func) # type: ignore[attr-defined] - return wrapper # type: ignore[return-value] +import logging.config + +LOG_FORMAT = "%(asctime)s | %(levelname)s | %(name)s | %(message)s" + + +def build_logging_config(level: str) -> dict: + return { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": LOG_FORMAT, + }, + }, + "handlers": { + "default": { + "class": "logging.StreamHandler", + "formatter": "default", + }, + }, + "root": { + "level": level, + "handlers": ["default"], + }, + } + + +def configure_logging(level: str) -> None: + logging.config.dictConfig(build_logging_config(level)) diff --git a/src/mcpdiffusion/core/middleware.py b/src/mcpdiffusion/core/middleware.py deleted file mode 100644 index 7044213..0000000 --- a/src/mcpdiffusion/core/middleware.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Rate-limiting middleware with injected settings.""" -from __future__ import annotations - -from datetime import datetime -from zoneinfo import ZoneInfo - -from limits import parse, storage, strategies -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.status import HTTP_429_TOO_MANY_REQUESTS - -from ..config.settings import Settings, get_settings - - -class RateLimitMiddleware(BaseHTTPMiddleware): - """Middleware that applies per-IP rate limiting. - - Accepts an optional ``settings`` parameter for dependency injection - (used by tests). Falls back to ``get_settings()`` when not provided. - """ - - # Fixme: passing the whole settings object is inappropriate - # Only relevant properties should be passed - # It allows for better interface segregation - # This would also avoid having this class creating the settings as a fallback - def __init__(self, app, settings: Settings | None = None): - super().__init__(app) - self._settings = settings or get_settings() - self._tz = ZoneInfo(self._settings.tz) - # Fixme: the storage behavior does not scale with multiple replicas - # Consider implementing redis for handling state between multiple instances - self._storage = storage.MemoryStorage() - self._limiter = strategies.MovingWindowRateLimiter(self._storage) - self._rate = parse(f"{self._settings.global_request_min}/minute") - - # Fixme: there is no request path filtering on that dispatcher, meaning it runs also for non relevant path like / - # or even health checks - async def dispatch(self, request: Request, call_next): - client_ip = request.client.host if request.client else "unknown" - # Fixme: if the key is the client ip, double check if anyone can alter it and bypass rate limiting... - # Fixme: also the usage of an fstring is useless here - rate_key = f"{client_ip}" - - if not self._limiter.hit(self._rate, rate_key): - retry_after_ts = self._limiter.get_window_stats(self._rate, rate_key)[0] - retry_after_time = datetime.fromtimestamp( - retry_after_ts, tz=self._tz - ).strftime("%H:%M:%S") - return JSONResponse( - status_code=HTTP_429_TOO_MANY_REQUESTS, - content={ - "detail": f"Trop de requetes. Reessayez apres {retry_after_time}.", - "retry_after": retry_after_time, - }, - headers={ - "Retry-After": str(int(retry_after_ts)), - "X-RateLimit-Limit": str(self._settings.global_request_min), - "X-RateLimit-Remaining": "0", - }, - ) - - response = await call_next(request) - - # Fixme: the result of 'self._limiter.get_window_stats(self._rate, rate_key)' could have been cached - remaining = self._limiter.get_window_stats(self._rate, rate_key)[1] - response.headers["X-RateLimit-Limit"] = str(self._settings.global_request_min) - response.headers["X-RateLimit-Remaining"] = str(remaining) - # Fixme: 60 is a magic number and should be set using a constant config - response.headers["X-RateLimit-Window"] = f"{60}s" - - return response diff --git a/src/mcpdiffusion/core/rate_limiting.py b/src/mcpdiffusion/core/rate_limiting.py new file mode 100644 index 0000000..c8710f5 --- /dev/null +++ b/src/mcpdiffusion/core/rate_limiting.py @@ -0,0 +1,26 @@ +"""Client identity for rate limiting.""" + +import logging + +from fastmcp.server.dependencies import get_http_request +from fastmcp.server.middleware.middleware import MiddlewareContext + +logger = logging.getLogger(__name__) + +UNKNOWN_CLIENT = "unknown" + + +def resolve_client_host(_context: MiddlewareContext) -> str: + """Rate-limit key. Only as trustworthy as `trusted_proxy_hosts`: widen that and a caller can forge it. + + All callers without a resolvable host share one bucket, so a transport that never carries an HTTP + request would rate-limit every client together. + """ + try: + client = get_http_request().client + except RuntimeError: + client = None + if client is None: + logger.debug("No client address available; this caller shares the fallback rate-limit bucket.") + return UNKNOWN_CLIENT + return client.host diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py index b5e3336..653bf3d 100644 --- a/src/mcpdiffusion/infra/elasticsearch.py +++ b/src/mcpdiffusion/infra/elasticsearch.py @@ -1,9 +1,8 @@ -"""Elasticsearch client accessor from FastMCP lifespan context.""" -# Fixme: this annotation seems unnecessary -from __future__ import annotations +"""Elasticsearch client accessor from the FastMCP lifespan context.""" +from elasticsearch import AsyncElasticsearch from fastmcp import Context -def get_client_es(ctx: Context): - return ctx.lifespan_context["es_client"] +def get_elasticsearch_client(ctx: Context) -> AsyncElasticsearch: + return ctx.lifespan_context["elasticsearch_client"] diff --git a/src/mcpdiffusion/infra/http.py b/src/mcpdiffusion/infra/http.py index 890ee76..7b723f1 100644 --- a/src/mcpdiffusion/infra/http.py +++ b/src/mcpdiffusion/infra/http.py @@ -1,8 +1,12 @@ -"""HTTP client accessor from FastMCP lifespan context.""" -from __future__ import annotations +"""HTTP client accessors from the FastMCP lifespan context.""" from fastmcp import Context +from httpx import AsyncClient -def get_http_client(ctx: Context): - return ctx.lifespan_context["http_client"] +def get_insee_http_client(ctx: Context) -> AsyncClient: + return ctx.lifespan_context["insee_http_client"] + + +def get_melodi_http_client(ctx: Context) -> AsyncClient: + return ctx.lifespan_context["melodi_http_client"] diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/infra/lifespan.py index dfd38ef..dbbc99d 100644 --- a/src/mcpdiffusion/infra/lifespan.py +++ b/src/mcpdiffusion/infra/lifespan.py @@ -1,75 +1,87 @@ -"""Combined application lifespan: creates and tears down shared clients.""" -from __future__ import annotations +"""Shared clients, created once at startup and torn down on shutdown.""" import logging +from collections.abc import AsyncIterator, Callable +from typing import Any -import httpx -from elasticsearch import Elasticsearch +from elasticsearch import AsyncElasticsearch +from httpx import AsyncClient, Timeout from fastmcp.server.lifespan import lifespan -from ..config.settings import get_settings +logger = logging.getLogger(__name__) -# Fixme: was there not a reference for that logger name in 'config/logging.py'? -logger = logging.getLogger("mcp.main") - -_HTTP_USER_AGENT = ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/124.0 Safari/537.36" +# insee.fr serves different markup to unknown agents, so the scraper has to look like a browser. +INSEE_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" ) -_HTTP_TIMEOUT = httpx.Timeout(30.0, connect=10.0) -_SPARQL_USER_AGENT = "MCP-RMeS/2.0" +# The APIs have no such requirement, so they get an honest identity. +MELODI_USER_AGENT = "McpDiffusion/0.1" +SPARQL_USER_AGENT = "MCP-RMeS/2.0" + +ES_MAX_RETRIES = 2 + -# Fixme: One issue I see with that pattern is that if some client instantiation fail, -# the error is ignored and this can be tricky to identify -# Fixme: Also, i see no client is properly tested upon creation (simple ping). -# It could help identify issues at startup: but this is not mandatory -# Fixme: this piece of code contains too many magic values that belong in settings -# Fixme: server is unused, if required by FastMCP, prefer prefixing it with an underscore -@lifespan -async def app_lifespan(server): - # Fixme: I am questioning whether this should be the responsibility of that function to instantiate settings - s = get_settings() +def build_lifespan( + *, + es_host: str, + es_tls_verify: bool, + es_request_timeout_seconds: int, + insee_base_url: str, + insee_request_timeout_seconds: int, + insee_connect_timeout_seconds: int, + melodi_data_base_url: str, + melodi_request_timeout_seconds: int, + melodi_connect_timeout_seconds: int, +) -> Callable[..., Any]: - # Elasticsearch - # Fixme: in 'config/settings.py', 'es_host' is deemed optional, so it must be made mandatory if required here - if not s.es_host: - raise RuntimeError("ES_HOST is not set. See .env.example.") + @lifespan + async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: + elasticsearch_client = AsyncElasticsearch( + es_host, + verify_certs=es_tls_verify, + request_timeout=es_request_timeout_seconds, + max_retries=ES_MAX_RETRIES, + retry_on_timeout=True, + ) + logger.info("Elasticsearch client initialized for %s", es_host) - # Fixme: if this synchronous client is used within async coroutines, - # it will block the event loop for the duration of the query - # This is a major issue - es_client = Elasticsearch( - s.es_host, - verify_certs=s.tls_verify, - request_timeout=30, - max_retries=2, - retry_on_timeout=True, - ) - logger.info("Elasticsearch client initialized for %s", s.es_host) + insee_http_client = AsyncClient( + base_url=insee_base_url, + headers={"User-Agent": INSEE_USER_AGENT}, + timeout=Timeout( + insee_request_timeout_seconds, + connect=insee_connect_timeout_seconds, + ), + ) + logger.info("insee.fr client initialized for %s", insee_base_url) - # HTTP client (insee.fr, melodi API) - http_client = httpx.AsyncClient( - verify=s.tls_verify, - headers={"User-Agent": _HTTP_USER_AGENT}, - timeout=_HTTP_TIMEOUT, - ) - logger.info("HTTP client initialized") + melodi_http_client = AsyncClient( + base_url=melodi_data_base_url, + headers={"User-Agent": MELODI_USER_AGENT}, + timeout=Timeout( + melodi_request_timeout_seconds, + connect=melodi_connect_timeout_seconds, + ), + ) + logger.info("MELODI client initialized for %s", melodi_data_base_url) - # SPARQL client - # Fixme: TLS is ignored in some clients which seems inconsistent - sparql_client = httpx.AsyncClient( - headers={"User-Agent": _SPARQL_USER_AGENT}, - ) - logger.info("SPARQL client initialized") + # RMES passes its own timeout per query, so this client sets none. + sparql_http_client = AsyncClient( + headers={"User-Agent": SPARQL_USER_AGENT}, + ) + logger.info("SPARQL client initialized") - yield { - "es_client": es_client, - "http_client": http_client, - "sparql_client": sparql_client, - } + try: + yield { + "elasticsearch_client": elasticsearch_client, + "insee_http_client": insee_http_client, + "melodi_http_client": melodi_http_client, + "sparql_http_client": sparql_http_client, + } + finally: + await elasticsearch_client.close() + await insee_http_client.aclose() + await melodi_http_client.aclose() + await sparql_http_client.aclose() - # Fixme: the instantiated elastic client is synchronous, so cannot be prepended by the 'await' keyword - # this would app to raise and crash on shutdown - await es_client.close() - await http_client.aclose() - await sparql_client.aclose() + return app_lifespan diff --git a/src/mcpdiffusion/infra/sparql.py b/src/mcpdiffusion/infra/sparql.py index 0e9ab72..a0ff10d 100644 --- a/src/mcpdiffusion/infra/sparql.py +++ b/src/mcpdiffusion/infra/sparql.py @@ -1,8 +1,8 @@ -"""SPARQL client accessor from FastMCP lifespan context.""" -from __future__ import annotations +"""SPARQL client accessor from the FastMCP lifespan context.""" from fastmcp import Context +from httpx import AsyncClient -def get_sparql_client(ctx: Context): - return ctx.lifespan_context["sparql_client"] +def get_sparql_http_client(ctx: Context) -> AsyncClient: + return ctx.lifespan_context["sparql_http_client"] diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index 36627b3..073fc0e 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -10,8 +10,8 @@ # --- Shared RMES constants exposed to tools --- # Fixme: a lot of values in here belongs in settings -DEFAULT_TIMEOUT = 20.0 -MAX_TIMEOUT = 60.0 +DEFAULT_QUERY_TIMEOUT_SECONDS = 20.0 +MAX_QUERY_TIMEOUT_SECONDS = 60.0 DEFAULT_ROW_LIMIT = 200 MAX_ROW_LIMIT = 2000 @@ -144,8 +144,8 @@ class RunSparqlInput(BaseModel): description="Requete SPARQL complete (SELECT / ASK / CONSTRUCT / DESCRIBE).", ) timeout: float = Field( - default=DEFAULT_TIMEOUT, - description=f"Timeout en secondes (plafonne a {MAX_TIMEOUT}s).", + default=DEFAULT_QUERY_TIMEOUT_SECONDS, + description=f"Timeout en secondes (plafonne a {MAX_QUERY_TIMEOUT_SECONDS}s).", gt=0, ) max_rows: int = Field( diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index db8a976..4a245fe 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -1,46 +1,68 @@ -"""FastMCP entrypoint for the mcp-diffusion server. - -Boots Uvicorn, registers every tool via `tools.register_tools(mcp)`, -and exposes the HTTP transport on MCP_HOST:MCP_PORT. -""" +"""Entrypoint: builds the settings, the clients and the MCP application, then serves it over HTTP.""" import logging -from dotenv import load_dotenv from fastmcp import FastMCP -from starlette.middleware.trustedhost import TrustedHostMiddleware +from fastmcp.server.middleware.error_handling import ErrorHandlingMiddleware +from fastmcp.server.middleware.logging import LoggingMiddleware +from fastmcp.server.middleware.rate_limiting import SlidingWindowRateLimitingMiddleware +from fastmcp.server.middleware.timing import TimingMiddleware -from .config.settings import get_settings -from .core.logging import MAIN_LOGGER_NAME, UVICORN_LOGGING_CONFIG -from .core.middleware import RateLimitMiddleware +from .config.settings import load_settings +from .core.logging import build_logging_config, configure_logging +from .core.rate_limiting import resolve_client_host +from .infra.lifespan import build_lifespan from .tools import register_tools -from .infra.lifespan import app_lifespan -settings = get_settings() -logger = logging.getLogger(MAIN_LOGGER_NAME) +settings = load_settings() +configure_logging(settings.log_level) +logger = logging.getLogger(__name__) -mcp = FastMCP("INSEE-mcp-diffusion", lifespan=app_lifespan) +mcp = FastMCP( + "INSEE-mcp-diffusion", + lifespan=build_lifespan( + es_host=settings.es_host, + es_tls_verify=settings.es_tls_verify, + es_request_timeout_seconds=settings.es_request_timeout_seconds, + insee_base_url=settings.insee_base_url, + insee_request_timeout_seconds=settings.insee_request_timeout_seconds, + insee_connect_timeout_seconds=settings.insee_connect_timeout_seconds, + melodi_data_base_url=settings.melodi_data_base_url, + melodi_request_timeout_seconds=settings.melodi_request_timeout_seconds, + melodi_connect_timeout_seconds=settings.melodi_connect_timeout_seconds, + ), +) register_tools(mcp, settings) -app = mcp.http_app() - -# TrustedHostMiddleware -# Fixme: the 'ALLOWED_HOSTS' env variable is not present in the .env.example file, defaulting to allowed hosts to "*" -# Fixme: this code that sets '_allowed_hosts' belongs in the settings, not here -_allowed_hosts_raw = settings.allowed_hosts.strip() -_allowed_hosts = ( - ["*"] if _allowed_hosts_raw == "*" - else [h.strip() for h in _allowed_hosts_raw.split(",") if h.strip()] +# Order matters: error handling first so it sees the whole chain, logging last so it records what ran. +# Each middleware logs under its own `fastmcp.*` logger; set levels there to tune the output. +# None of them logs how many results a tool returned. If empty results become hard to diagnose, add an +# `on_call_tool` middleware that inspects the ToolResult, or have the tool report it with `ctx.info`. +mcp.add_middleware( + ErrorHandlingMiddleware(), +) +mcp.add_middleware( + SlidingWindowRateLimitingMiddleware( + max_requests=settings.rate_limit_max_requests, + window_minutes=settings.rate_limit_window_minutes, + get_client_id=resolve_client_host, + ), +) +mcp.add_middleware( + TimingMiddleware(), +) +mcp.add_middleware( + LoggingMiddleware(), ) -if _allowed_hosts == ["*"]: - logger.warning( - "TrustedHostMiddleware configured with allowed_hosts=['*']. " - "Set ALLOWED_HOSTS before exposing the server publicly." - ) -app.add_middleware(RateLimitMiddleware, settings=settings) -app.add_middleware(TrustedHostMiddleware, allowed_hosts=_allowed_hosts) +if settings.allowed_hosts == ["*"]: + logger.warning("allowed_hosts is ['*']. Set ALLOWED_HOSTS before exposing the server publicly.") + +app = mcp.http_app( + host_origin_protection="auto", + allowed_hosts=settings.allowed_hosts, +) if __name__ == "__main__": import uvicorn @@ -50,7 +72,6 @@ host=settings.mcp_host, port=settings.mcp_port, proxy_headers=True, - forwarded_allow_ips=settings.forwarded_allow_ips, - log_level="info", - log_config=UVICORN_LOGGING_CONFIG, + forwarded_allow_ips=settings.trusted_proxy_hosts, + log_config=build_logging_config(settings.log_level), ) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 98c4d35..2d7f415 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -2,7 +2,6 @@ from __future__ import annotations from collections import defaultdict -from typing import Optional from urllib.parse import urljoin, urlparse from bs4 import BeautifulSoup @@ -11,7 +10,6 @@ import httpx -from ..config.settings import Settings, get_settings from ..core.errors import fail from ..models.insee import ( DocumentResult, @@ -86,26 +84,28 @@ def _format_sommaire(flat_items: list[dict[str, str]]) -> dict[str, dict[str, st return dict(grouped) +_TRUNCATION_MARKER = """ + + + +""" + + def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: if len(text) <= limit: return text, False - # Fixme: avoid magic numbers popping here and there - head_size = (limit * 2) // 3 - # Fixme: what happens if limit is too small? 'tail_size' can go negative - tail_size = limit - head_size - 200 - # Fixme: prefer multiline strings which are more readable and easier to deal with - marker = ( - "\n\n\n\n" - ) - return text[:head_size] + marker + text[-tail_size:], True + budget = max(0, limit - len(_TRUNCATION_MARKER)) + head_size = (budget * 2) // 3 + tail_size = budget - head_size + # text[-0:] returns the whole string, so an empty tail has to be spelled out. + tail = text[-tail_size:] if tail_size else "" + return text[:head_size] + _TRUNCATION_MARKER + tail, True -# Fixme: injecting the whole settings is bad separation of concerns -async def _fetch_html(url: str, settings: Settings, http_client: httpx.AsyncClient) -> str: - full_url = settings.insee_base_url + url if not url.startswith(("http://", "https://")) else url +async def _fetch_html(url: str, http_client: httpx.AsyncClient) -> str: + # A relative path resolves against the client's base_url; an absolute one overrides it. try: - response = await http_client.get(full_url, follow_redirects=True) + response = await http_client.get(url, follow_redirects=True) response.raise_for_status() return response.text # Fixme: the error handling is not correctly designed, at a global scale @@ -142,16 +142,11 @@ async def _fetch_html(url: str, settings: Settings, http_client: httpx.AsyncClie raise -# Fixme: here params obfuscates the meaning of the input argument async def get_insee_document( params: GetInseeDocumentInput, *, http_client: httpx.AsyncClient, - settings: Settings | None = None, ) -> GetInseeDocumentOutput: - # Fixme: not the right place to init settings - s = settings or get_settings() - if not params.list_of_url: fail( "INVALID_INPUT", @@ -164,16 +159,16 @@ async def get_insee_document( # Fixme: on top of that, the fetching is done sequentially, impacting the event loop for url in params.list_of_url: try: - html = await _fetch_html(str(url), s, http_client) + html = await _fetch_html(str(url), http_client) markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" if params.truncate_content: markdown, truncated = _truncate(markdown) else: truncated = False - sommaire: Optional[dict[str, dict[str, str]]] = None + sommaire: dict[str, dict[str, str]] | None = None if params.include_sommaire: - flat = _parse_sommaire(html, s.insee_base_url) + flat = _parse_sommaire(html, str(http_client.base_url)) sommaire = _format_sommaire(flat) if flat else None results.append( diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index 575b9ec..f835213 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -5,18 +5,19 @@ """ from __future__ import annotations -from typing import Iterable, Optional +from collections.abc import Iterable -from elasticsearch import Elasticsearch -from elasticsearch.dsl import Q, Search +from elasticsearch import AsyncElasticsearch +from elasticsearch.dsl import AsyncSearch, Q -from ..config.settings import Settings, get_settings from ..data.geography import DICT_GEO from ..data.themes import KEYS_THEME_NIV1 from ..models.insee import DocumentHit +QueryClauses = tuple[list, list, list] -def _coerce_hit_value(value) -> Optional[str]: + +def _coerce_hit_value(value) -> str | None: if value is None: return None if isinstance(value, list): @@ -25,19 +26,15 @@ def _coerce_hit_value(value) -> Optional[str]: # Build query -# Fixme: the type tuple[list, list, list, list] should be aliased for comprehension -# Fixme: also prefer newer optional syntax def build_text_clauses( - query: Optional[str], - year_of_reference: Optional[int], + query: str | None, + year_of_reference: int | None, keywords: Iterable[str] = (), -) -> tuple[list, list, list, list]: - """Return (must, filter, should, must_not) clause lists.""" +) -> QueryClauses: + """Return the (must, filter, should) clause lists for a text query.""" must: list = [] filters: list = [] should: list = [] - # Fixme: must_not is not used and returned as is, this is inappropriate - must_not: list = [] if query: must.append( @@ -79,7 +76,7 @@ def build_text_clauses( ) ) - return must, filters, should, must_not + return must, filters, should def apply_collection_filters( @@ -88,14 +85,14 @@ def apply_collection_filters( must_not_rapides: bool, must_only_rapides: bool, chiffre_clef: bool = False, - theme: Optional[str] = None, - geo_niveau: Optional[str] = None, - geo_keyword: Optional[str] = None, + theme: str | None = None, + geo_niveau: str | None = None, + geo_keyword: str | None = None, ) -> tuple[list, list]: - """Apply INSEE-specific filters. Returns updated (filters, should).""" + """Return new (filters, should) lists. The caller's `filters` is left untouched.""" + filters = list(filters) should: list = [] - # Fixme: The passed in 'filters' is mutated if must_only_rapides: filters.append(Q("term", collection_libelle="Informations rapides")) elif must_not_rapides: @@ -105,7 +102,8 @@ def apply_collection_filters( # Fixme: the 1st check seems useless if theme and theme != "ALL": - # Fixme: so if the caller sends an unregistered theme, we drop his filter anyway? + # Business rule: an unrecognised theme drops the filter silently, so the search returns more + # than the caller asked for. Reject the value, or accept it and say so in the response? id_theme = KEYS_THEME_NIV1.get(theme) if id_theme is not None: filters.append(Q("term", idthemeparent=id_theme)) @@ -116,7 +114,8 @@ def apply_collection_filters( if geo_niveau: key_geo = DICT_GEO.get(geo_niveau) if key_geo: - # Fixme: same as above + # Business rule: same as the theme filter above — an unrecognised geo_niveau is dropped + # silently and broadens the search. filters.append(Q("term", geo_niveau=key_geo)) if geo_keyword and geo_keyword.lower() != "all": @@ -136,21 +135,18 @@ def apply_collection_filters( # Execute search with built query -# Fixme: inject only relevant settings parameters -def execute_search( +async def execute_search( *, must: list, filters: list, should: list, - must_not: list, + minimum_should_match: int, number_of_results: int, - es: Elasticsearch, - settings: Settings | None = None, + es: AsyncElasticsearch, + index: str, ) -> list[DocumentHit]: """Run the assembled bool query and return whitelisted DocumentHit records.""" - # Fixme: such function should not init settings - s = settings or get_settings() - search = Search(using=es, index=s.es_index_produits).query( + search = AsyncSearch(using=es, index=index).query( Q( "function_score", query=Q( @@ -158,16 +154,17 @@ def execute_search( must=must, filter=filters, should=should, - must_not=must_not, - # Fixme: this seem counter counter intuitive, to require a minimum should match, - # so it is not a should eventually? - minimum_should_match=1 if should else 0, + # `should` mixes pure score boosts with the geo clauses, which the caller wants + # required when present. Only the caller knows which it passed, so it decides. + # Business rule: a supplied `geo_keyword` is currently *required* to match, not just + # boosted, so it silently narrows results. Confirm this is intended. + minimum_should_match=minimum_should_match, ), boost_mode="sum", ) ) search = search[: max(1, number_of_results)] - res = search.execute() + res = await search.execute() hits: list[DocumentHit] = [] for hit in res: diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 6adb86b..194b090 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -4,9 +4,10 @@ from typing import Any import httpx +from elasticsearch import AsyncElasticsearch from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError, Elasticsearch -from ..config.settings import Settings, get_settings +from elasticsearch import TransportError + from ..core.errors import fail from ..models.melodi import ( ColumnResult, @@ -25,11 +26,9 @@ async def get_melodi_observations( params: GetMelodiObservationsInput, *, http_client: httpx.AsyncClient, - settings: Settings | None = None, ) -> GetMelodiObservationsOutput: - # Fixme: same comment for settings, inject relevant properties only - s = settings or get_settings() - url = f"{s.melodi_data_base_url}/{params.dataset_id}" + # Resolved against the client's base_url. + url = f"/{params.dataset_id}" try: response = await http_client.get( url, @@ -121,10 +120,9 @@ async def get_melodi_observations( async def search_melodi_datasets( params: SearchMelodiDatasetsInput, *, - settings: Settings | None = None, - es: Elasticsearch, + es: AsyncElasticsearch, + index: str, ) -> SearchMelodiDatasetsOutput: - s = settings or get_settings() filters: list[dict[str, Any]] = [] if params.start_year: filters.append({ @@ -202,7 +200,10 @@ async def search_melodi_datasets( } try: - ds_res = es.search(index=s.es_index_melodi_datasets, body=body) + ds_res = await es.search( + index=index, + body=body, + ) except (ESConnectionError, TransportError) as exc: fail( "BACKEND_UNAVAILABLE", @@ -218,10 +219,7 @@ async def search_melodi_datasets( description = source.get("metadata", {}).get("description") if isinstance(description, list) and description: description = description[0] - # Fixme: this branch does nothing - elif isinstance(description, dict): - description = description - else: + elif not isinstance(description, dict): description = {"content": "", "lang": "fr"} results.append( DatasetSearchResult( @@ -237,18 +235,16 @@ async def search_melodi_datasets( async def search_melodi_modalities( params: SearchMelodiModalitiesInput, *, - settings: Settings | None = None, - es: Elasticsearch, + es: AsyncElasticsearch, + index: str, ) -> SearchMelodiModalitiesOutput: - s = settings or get_settings() filters: list[dict[str, Any]] = [{"term": {"dataset_id": params.dataset_id}}] if params.columns_id: filters.append({"terms": {"code": params.columns_id}}) try: - # Fixme: the 1st es.search call was formatted differently, pick a single convention - ds_column = es.search( - index=s.es_index_melodi_columns, + ds_column = await es.search( + index=index, size=20, query={ "bool": { diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 5c0956d..3f1460d 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -12,11 +12,11 @@ import httpx -from ..config.settings import Settings, get_settings from ..models.rmes import ( + DEFAULT_QUERY_TIMEOUT_SECONDS, GRAPH_BASE, MAX_ROW_LIMIT, - MAX_TIMEOUT, + MAX_QUERY_TIMEOUT_SECONDS, CategoryBucket, DescribeResourceOutput, GraphCategoryChoice, @@ -32,7 +32,11 @@ ) # Fixme: follow a clear convention for logger names -logger = logging.getLogger("mcp.rmes") +logger = logging.getLogger(__name__) + +# Listing every graph is far heavier than a normal user query, so it gets its own budget. +GRAPH_LISTING_TIMEOUT_SECONDS = 45.0 +GRAPH_LISTING_MAX_ROWS = 1000 # Cache for raw graph rows (expensive COUNT query) _GRAPH_CACHE: dict[str, Any] = {"data": None, "ts": 0.0} @@ -200,9 +204,7 @@ def _prefix(prefix: str) -> CategoryMatcher: match=lambda path: True, ) -# Fixme: '_RULES_BY_KEY' uses '_ALL_RULES', but ultimately, '_RULES_BY_KEY' is never used _ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] -_RULES_BY_KEY = {r.key: r for r in _ALL_RULES} def _relative_path(graph_uri: str) -> str: @@ -266,9 +268,8 @@ async def _execute_sparql( max_rows: int, *, sparql_client: httpx.AsyncClient, - settings: Settings | None = None, + endpoint: str, ) -> dict[str, Any]: - s = settings or get_settings() query_form = _detect_query_form(query) if query_form == "UNKNOWN": @@ -285,10 +286,10 @@ async def _execute_sparql( try: client = sparql_client response = await client.post( - s.rmes_endpoint, + endpoint, data={"query": effective_query}, headers={"Accept": accept}, - timeout=min(timeout, MAX_TIMEOUT), + timeout=min(timeout, MAX_QUERY_TIMEOUT_SECONDS), ) response.raise_for_status() @@ -319,7 +320,7 @@ async def _execute_sparql( ) except httpx.RequestError as exc: - logger.warning("Erreur reseau vers %s: %s", s.rmes_endpoint, exc) + logger.warning("Erreur reseau vers %s: %s", endpoint, exc) return _error_payload( SparqlErrorType.NETWORK_ERROR, f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", @@ -345,7 +346,7 @@ async def _execute_sparql( async def _get_raw_graph_rows( *, sparql_client: httpx.AsyncClient, - settings: Settings | None = None, + endpoint: str, ) -> dict[str, Any]: now = time.time() if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: @@ -358,9 +359,11 @@ async def _get_raw_graph_rows( # consider an asyncio.Lock + a second freshness check inside it, # otherwise each waiter just re-runs the same expensive query result = await _execute_sparql( - # Fixme: those magic values belong in the settings - query, timeout=45.0, max_rows=1000, - sparql_client=sparql_client, settings=settings, + query, + timeout=GRAPH_LISTING_TIMEOUT_SECONDS, + max_rows=GRAPH_LISTING_MAX_ROWS, + sparql_client=sparql_client, + endpoint=endpoint, ) if "error" in result: return result @@ -406,10 +409,10 @@ async def list_graphs( params: ListGraphsInput, *, sparql_client: httpx.AsyncClient, - settings: Settings | None = None, + endpoint: str, ) -> ListGraphsOutput: raw = await _get_raw_graph_rows( - sparql_client=sparql_client, settings=settings, + sparql_client=sparql_client, endpoint=endpoint, ) if "error" in raw: return ListGraphsOutput( @@ -466,7 +469,7 @@ async def describe_resource( params: DescribeResourceInput, *, sparql_client: httpx.AsyncClient, - settings: Settings | None = None, + endpoint: str, ) -> DescribeResourceOutput: graph_clause = f"<{params.graph}>" if params.graph else "?g" graph_values = f"VALUES ?g {{ <{params.graph}> }}" if params.graph else "" @@ -484,11 +487,9 @@ async def describe_resource( }} }} LIMIT {MAX_ROW_LIMIT} """ - # Fixme: put the import at the top - from ..models.rmes import DEFAULT_TIMEOUT result = await _execute_sparql( - query, timeout=DEFAULT_TIMEOUT, max_rows=MAX_ROW_LIMIT, - sparql_client=sparql_client, settings=settings, + query, timeout=DEFAULT_QUERY_TIMEOUT_SECONDS, max_rows=MAX_ROW_LIMIT, + sparql_client=sparql_client, endpoint=endpoint, ) if "error" in result: @@ -504,7 +505,7 @@ async def run_sparql( params: RunSparqlInput, *, sparql_client: httpx.AsyncClient, - settings: Settings | None = None, + endpoint: str, ) -> RunSparqlOutput: if not params.full_sparql_query or not params.full_sparql_query.strip(): return RunSparqlOutput( @@ -518,7 +519,7 @@ async def run_sparql( max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) result = await _execute_sparql( params.full_sparql_query, timeout=params.timeout, max_rows=max_rows, - sparql_client=sparql_client, settings=settings, + sparql_client=sparql_client, endpoint=endpoint, ) if "error" in result: diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 6f91412..881613d 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -8,6 +8,8 @@ from fastmcp import FastMCP +from ..config.settings import Settings + from .melodi_get_observations import register_get_melodi_observations from .melodi_search_datasets import register_search_melodi_datasets from .melodi_search_modalities import register_search_melodi_modalities @@ -21,25 +23,22 @@ from .rmes_run_sparql import register_rmes_run_sparql from .extras_send_feedback import register_extras_send_feedback - -def register_tools(mcp: FastMCP, settings ) -> None: - """Register all MCP tools with the given FastMCP instance.""" - # INSEE.fr - if settings.enable_inseefr : - register_search_insee_documents(mcp) +# Fixme: there might be better pattern instead of iterating with if statements on tool groups +def register_tools(mcp: FastMCP, settings: Settings) -> None: + """Register the enabled tools, handing each the settings it needs.""" + if settings.enable_inseefr_tools: + register_search_insee_documents(mcp, index=settings.es_index_produits) register_get_insee_homepage(mcp) register_get_insee_document(mcp) - register_search_insee_conjoncture(mcp) - register_search_insee_chiffreclef(mcp) + register_search_insee_conjoncture(mcp, index=settings.es_index_produits) + register_search_insee_chiffreclef(mcp, index=settings.es_index_produits) - # Melodi - if settings.enable_melodi : - register_search_melodi_datasets(mcp) - register_search_melodi_modalities(mcp) + if settings.enable_melodi_tools: + register_search_melodi_datasets(mcp, index=settings.es_index_melodi_datasets) + register_search_melodi_modalities(mcp, index=settings.es_index_melodi_columns) register_get_melodi_observations(mcp) - # RMES (SPARQL) - if settings.enable_rmes: - register_rmes_list_graphs(mcp) - register_rmes_describe_resource(mcp) - register_rmes_run_sparql(mcp) \ No newline at end of file + if settings.enable_rmes_tools: + register_rmes_list_graphs(mcp, endpoint=settings.rmes_endpoint) + register_rmes_describe_resource(mcp, endpoint=settings.rmes_endpoint) + register_rmes_run_sparql(mcp, endpoint=settings.rmes_endpoint) diff --git a/src/mcpdiffusion/tools/extras_send_feedback.py b/src/mcpdiffusion/tools/extras_send_feedback.py index 21569de..f8d3117 100644 --- a/src/mcpdiffusion/tools/extras_send_feedback.py +++ b/src/mcpdiffusion/tools/extras_send_feedback.py @@ -4,7 +4,6 @@ from fastmcp import FastMCP from ..config.tool_metadata import SEND_FEEDBACK -from ..core.logging import log_tool from ..models.feedback import SendFeedbackInput, SendFeedbackOutput from ..services.feedback import send_feedback @@ -17,6 +16,5 @@ def register_extras_send_feedback(mcp: FastMCP) -> None: description=SEND_FEEDBACK["tool_description"], meta=SEND_FEEDBACK["tool_metadata"], ) - @log_tool async def send_feedback_tool(params: SendFeedbackInput) -> SendFeedbackOutput: return await send_feedback(params) diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 78825be..61889df 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -4,8 +4,7 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import GET_DOCUMENT -from ..core.logging import log_tool -from ..infra.http import get_http_client +from ..infra.http import get_insee_http_client from ..models.insee import GetInseeDocumentInput, GetInseeDocumentOutput from ..services.insee_document import get_insee_document @@ -16,10 +15,9 @@ def register_get_insee_document(mcp: FastMCP) -> None: description=GET_DOCUMENT["tool_description"], meta=GET_DOCUMENT["tool_metadata"], ) - @log_tool async def get_insee_documents( params: GetInseeDocumentInput, ctx: Context, ) -> GetInseeDocumentOutput: # Fixme: use singular or plural and stick to it - return await get_insee_document(params, http_client=get_http_client(ctx)) + return await get_insee_document(params, http_client=get_insee_http_client(ctx)) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index f3c0b5b..389d01d 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -4,7 +4,6 @@ from fastmcp import FastMCP from ..config.tool_metadata import GET_HOMEPAGE -from ..core.logging import log_tool from ..data.indicators import DICT_KV from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator @@ -17,7 +16,6 @@ def register_get_insee_homepage(mcp: FastMCP) -> None: meta=GET_HOMEPAGE["tool_metadata"], ) # Fixme: this is an async function with nothing to await - @log_tool async def get_insee_homepage() -> KeyIndicatorsOutput: indicators = [ KeyValueIndicator( diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index 010d05e..d130bf1 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -7,8 +7,7 @@ from ..config.tool_metadata import SEARCH_CHIFFRECLEF from ..core.errors import fail -from ..core.logging import log_tool -from ..infra.elasticsearch import get_client_es +from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( SearchInseeChiffrecleInput, SearchInseeChiffrecleOutput, @@ -21,24 +20,23 @@ # Fixme: the orchestration present in that function belongs in a service # Indeed, the approach from one tool to another is inconsistent -def register_search_insee_chiffreclef(mcp: FastMCP) -> None: +def register_search_insee_chiffreclef(mcp: FastMCP, *, index: str) -> None: @mcp.tool( name=SEARCH_CHIFFRECLEF["tool_name"], description=SEARCH_CHIFFRECLEF["tool_description"], meta=SEARCH_CHIFFRECLEF["tool_metadata"], ) - @log_tool async def search_insee_chiffrecle( params: SearchInseeChiffrecleInput, ctx: Context, ) -> SearchInseeChiffrecleOutput: - must, filters, should, must_not = build_text_clauses( + must, filters, should = build_text_clauses( query=params.query, year_of_reference=params.year_of_reference, ) # Fixme: should is overridden here - filters, should = apply_collection_filters( + filters, collection_should = apply_collection_filters( filters, must_not_rapides=True, must_only_rapides=False, @@ -48,13 +46,14 @@ async def search_insee_chiffrecle( geo_keyword=params.geo_keyword, ) try: - hits = execute_search( + hits = await execute_search( must=must, filters=filters, - should=should, - must_not=must_not, + should=should + collection_should, + minimum_should_match=1 if collection_should else 0, number_of_results=params.number_of_results, - es=get_client_es(ctx), + es=get_elasticsearch_client(ctx), + index=index, ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index 7232280..a26328a 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -8,9 +8,8 @@ from ..config.tool_metadata import SEARCH_CONJONCTURE from ..core.errors import fail -from ..core.logging import log_tool from ..data.themes import DICT_THEME_CONJ -from ..infra.elasticsearch import get_client_es +from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( SearchInseeConjonctureInput, SearchInseeConjonctureOutput, @@ -22,23 +21,21 @@ ) # Fixme: again, a lot of code in that tool that should belong in the service -def register_search_insee_conjoncture(mcp: FastMCP) -> None: +def register_search_insee_conjoncture(mcp: FastMCP, *, index: str) -> None: @mcp.tool( name=SEARCH_CONJONCTURE["tool_name"], description=SEARCH_CONJONCTURE["tool_description"], meta=SEARCH_CONJONCTURE["tool_metadata"], ) - @log_tool async def search_insee_conjoncture( params: SearchInseeConjonctureInput, ctx: Context, ) -> SearchInseeConjonctureOutput: - must, filters, should, must_not = build_text_clauses( + must, filters, should = build_text_clauses( query=params.query, year_of_reference=params.year_of_reference, ) - # Fixme: should is overridden - filters, should = apply_collection_filters( + filters, collection_should = apply_collection_filters( filters, # Fixme: the following allows for creating confusing combinaison must_not_rapides=False, @@ -46,19 +43,20 @@ async def search_insee_conjoncture( ) if params.theme_conjoncture: subthemes = DICT_THEME_CONJ.get(params.theme_conjoncture) - # Fixme: I don't know if this is the wanted behavior, but a subtheme miss will discard filtering, - # so return everything? + # Business rule: an unrecognised subtheme drops the filter silently and returns everything, + # the same shape as the theme and geo_niveau filters. if subthemes: filters.append(Q("terms", conjoncture_libelle=subthemes)) try: - hits = execute_search( + hits = await execute_search( must=must, filters=filters, - should=should, - must_not=must_not, + should=should + collection_should, + minimum_should_match=1 if collection_should else 0, number_of_results=params.number_of_results, - es=get_client_es(ctx), + es=get_elasticsearch_client(ctx), + index=index, ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index cc4639d..1277d92 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -7,8 +7,7 @@ from ..config.tool_metadata import SEARCH_DOCUMENTS from ..core.errors import fail -from ..core.logging import log_tool -from ..infra.elasticsearch import get_client_es +from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( SearchInseeDocumentsInput, SearchInseeDocumentsOutput, @@ -21,23 +20,21 @@ # Fixme: state clear conventions between what goes to a tool and what do not # most of the code might belong in the service -def register_search_insee_documents(mcp: FastMCP) -> None: +def register_search_insee_documents(mcp: FastMCP, *, index: str) -> None: @mcp.tool( name=SEARCH_DOCUMENTS["tool_name"], description=SEARCH_DOCUMENTS["tool_description"], meta=SEARCH_DOCUMENTS["tool_metadata"], ) - @log_tool async def search_insee_documents( params: SearchInseeDocumentsInput, ctx: Context, ) -> SearchInseeDocumentsOutput: - must, filters, should, must_not = build_text_clauses( + must, filters, should = build_text_clauses( query=params.query, year_of_reference=params.year_of_reference, ) - # Fixme: should is overridden - filters, should = apply_collection_filters( + filters, collection_should = apply_collection_filters( filters, must_not_rapides=True, must_only_rapides=False, @@ -47,13 +44,14 @@ async def search_insee_documents( geo_keyword=params.geo_keyword, ) try: - hits = execute_search( + hits = await execute_search( must=must, filters=filters, - should=should, - must_not=must_not, + should=should + collection_should, + minimum_should_match=1 if collection_should else 0, number_of_results=params.number_of_results, - es=get_client_es(ctx), + es=get_elasticsearch_client(ctx), + index=index, ) except (ESConnectionError, TransportError) as exc: fail( diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index ea456d0..76975fb 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -4,8 +4,7 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import GET_DATASET -from ..core.logging import log_tool -from ..infra.http import get_http_client +from ..infra.http import get_melodi_http_client from ..models.melodi import GetMelodiObservationsInput, GetMelodiObservationsOutput from ..services.melodi import get_melodi_observations @@ -17,9 +16,8 @@ def register_get_melodi_observations(mcp: FastMCP) -> None: description=GET_DATASET["tool_description"], meta=GET_DATASET["tool_metadata"], ) - @log_tool async def get_melodi_observations_tool( params: GetMelodiObservationsInput, ctx: Context, ) -> GetMelodiObservationsOutput: - return await get_melodi_observations(params, http_client=get_http_client(ctx)) + return await get_melodi_observations(params, http_client=get_melodi_http_client(ctx)) diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py index 7a0c590..c3ec01a 100644 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ b/src/mcpdiffusion/tools/melodi_search_datasets.py @@ -4,21 +4,23 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_DATASET -from ..core.logging import log_tool -from ..infra.elasticsearch import get_client_es +from ..infra.elasticsearch import get_elasticsearch_client from ..models.melodi import SearchMelodiDatasetsInput, SearchMelodiDatasetsOutput from ..services.melodi import search_melodi_datasets -def register_search_melodi_datasets(mcp: FastMCP) -> None: +def register_search_melodi_datasets(mcp: FastMCP, *, index: str) -> None: @mcp.tool( name=SEARCH_DATASET["tool_name"], description=SEARCH_DATASET["tool_description"], meta=SEARCH_DATASET["tool_metadata"], ) - @log_tool async def search_melodi_datasets_tool( params: SearchMelodiDatasetsInput, ctx: Context, ) -> SearchMelodiDatasetsOutput: - return await search_melodi_datasets(params, es=get_client_es(ctx)) + return await search_melodi_datasets( + params, + es=get_elasticsearch_client(ctx), + index=index, + ) diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py index c01a48b..6074f92 100644 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ b/src/mcpdiffusion/tools/melodi_search_modalities.py @@ -4,21 +4,23 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_MODALITIES -from ..core.logging import log_tool -from ..infra.elasticsearch import get_client_es +from ..infra.elasticsearch import get_elasticsearch_client from ..models.melodi import SearchMelodiModalitiesInput, SearchMelodiModalitiesOutput from ..services.melodi import search_melodi_modalities -def register_search_melodi_modalities(mcp: FastMCP) -> None: +def register_search_melodi_modalities(mcp: FastMCP, *, index: str) -> None: @mcp.tool( name=SEARCH_MODALITIES["tool_name"], description=SEARCH_MODALITIES["tool_description"], meta=SEARCH_MODALITIES["tool_metadata"], ) - @log_tool async def search_melodi_modalities_tool( params: SearchMelodiModalitiesInput, ctx: Context, ) -> SearchMelodiModalitiesOutput: - return await search_melodi_modalities(params, es=get_client_es(ctx)) + return await search_melodi_modalities( + params, + es=get_elasticsearch_client(ctx), + index=index, + ) diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index 72c2710..b62562e 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -4,18 +4,20 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_DESCRIBE_RESOURCE -from ..core.logging import log_tool -from ..infra.sparql import get_sparql_client +from ..infra.sparql import get_sparql_http_client from ..models.rmes import DescribeResourceInput, DescribeResourceOutput from ..services.rmes import describe_resource -def register_rmes_describe_resource(mcp: FastMCP) -> None: +def register_rmes_describe_resource(mcp: FastMCP, *, endpoint: str) -> None: @mcp.tool( name=RMES_DESCRIBE_RESOURCE["tool_name"], description=RMES_DESCRIBE_RESOURCE["tool_description"], meta=RMES_DESCRIBE_RESOURCE["tool_metadata"], ) - @log_tool async def describe_resource_tool(params: DescribeResourceInput, ctx: Context) -> DescribeResourceOutput: - return await describe_resource(params, sparql_client=get_sparql_client(ctx)) + return await describe_resource( + params, + sparql_client=get_sparql_http_client(ctx), + endpoint=endpoint, + ) diff --git a/src/mcpdiffusion/tools/rmes_list_graphs.py b/src/mcpdiffusion/tools/rmes_list_graphs.py index ec4d6b6..a2d31a4 100644 --- a/src/mcpdiffusion/tools/rmes_list_graphs.py +++ b/src/mcpdiffusion/tools/rmes_list_graphs.py @@ -4,18 +4,20 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_LIST_GRAPHS -from ..core.logging import log_tool -from ..infra.sparql import get_sparql_client +from ..infra.sparql import get_sparql_http_client from ..models.rmes import ListGraphsInput, ListGraphsOutput from ..services.rmes import list_graphs -def register_rmes_list_graphs(mcp: FastMCP) -> None: +def register_rmes_list_graphs(mcp: FastMCP, *, endpoint: str) -> None: @mcp.tool( name=RMES_LIST_GRAPHS["tool_name"], description=RMES_LIST_GRAPHS["tool_description"], meta=RMES_LIST_GRAPHS["tool_metadata"], ) - @log_tool async def list_graphs_tool(params: ListGraphsInput, ctx: Context) -> ListGraphsOutput: - return await list_graphs(params, sparql_client=get_sparql_client(ctx)) + return await list_graphs( + params, + sparql_client=get_sparql_http_client(ctx), + endpoint=endpoint, + ) diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index c2aeab3..494a0b9 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -4,13 +4,12 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import RMES_RUN_SPARQL -from ..core.logging import log_tool -from ..infra.sparql import get_sparql_client +from ..infra.sparql import get_sparql_http_client from ..models.rmes import RunSparqlInput, RunSparqlOutput from ..services.rmes import KNOWN_VOCABULARIES_NOTE, run_sparql -def register_rmes_run_sparql(mcp: FastMCP) -> None: +def register_rmes_run_sparql(mcp: FastMCP, *, endpoint: str) -> None: @mcp.tool( name=RMES_RUN_SPARQL["tool_name"], # Fixme: this description combinaison involves too many sources of data, the metadata file, @@ -31,6 +30,9 @@ def register_rmes_run_sparql(mcp: FastMCP) -> None: "plutot que des lignes (`format=\"json\"`, champs `variables`/`bindings`).", meta=RMES_RUN_SPARQL["tool_metadata"], ) - @log_tool async def run_sparql_tool(params: RunSparqlInput, ctx: Context) -> RunSparqlOutput: - return await run_sparql(params, sparql_client=get_sparql_client(ctx)) + return await run_sparql( + params, + sparql_client=get_sparql_http_client(ctx), + endpoint=endpoint, + ) diff --git a/uv.lock b/uv.lock index 2e5b4d1..41da271 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,128 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" }, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -474,18 +596,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/1b/349e07ad184d64e81109e85a3557d7e05631fa3d05344169114ba743c4d3/dateparser-1.4.2-py3-none-any.whl", hash = "sha256:752f3d49d477cf7f60a7a9c8bcb19c882496ede0e377d5a3d80014cdfeca7050", size = 316546, upload-time = "2026-08-04T12:11:01.396Z" }, ] -[[package]] -name = "deprecated" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, -] - [[package]] name = "dnspython" version = "2.8.0" @@ -534,6 +644,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/8c/d84fe1f2a6f60ce1fbc682a2e98776575d7b73e60680ac5aa90f53052466/elasticsearch-9.5.0-py3-none-any.whl", hash = "sha256:010e04f44fd161428f0ab7f94b93a533d3dcc3285d02c9df509b4e0127916420", size = 1011512, upload-time = "2026-08-04T17:55:54.792Z" }, ] +[package.optional-dependencies] +async = [ + { name = "aiohttp" }, +] + [[package]] name = "email-validator" version = "2.3.0" @@ -623,6 +738,95 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + [[package]] name = "griffelib" version = "2.2.0" @@ -876,20 +1080,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, ] -[[package]] -name = "limits" -version = "5.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "packaging" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/69/826a5d1f45426c68d8f6539f8d275c0e4fcaa57f0c017ec3100986558a41/limits-5.8.0.tar.gz", hash = "sha256:c9e0d74aed837e8f6f50d1fcebcf5fd8130957287206bc3799adaee5092655da", size = 226104, upload-time = "2026-02-05T07:17:35.859Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/98/cb5ca20618d205a09d5bec7591fbc4130369c7e6308d9a676a28ff3ab22c/limits-5.8.0-py3-none-any.whl", hash = "sha256:ae1b008a43eb43073c3c579398bd4eb4c795de60952532dc24720ab45e1ac6b8", size = 60954, upload-time = "2026-02-05T07:17:34.425Z" }, -] - [[package]] name = "lxml" version = "6.1.2" @@ -1073,10 +1263,9 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, - { name = "elasticsearch" }, + { name = "elasticsearch", extra = ["async"] }, { name = "fastmcp" }, { name = "httpx" }, - { name = "limits" }, { name = "lxml" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, @@ -1095,10 +1284,9 @@ dev = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = "==4.15.0" }, - { name = "elasticsearch", specifier = "==9.5.0" }, - { name = "fastmcp", specifier = "==4.0.0" }, + { name = "elasticsearch", extras = ["async"], specifier = "==9.5.0" }, + { name = "fastmcp", specifier = ">=4.0.0" }, { name = "httpx", specifier = "==0.28.1" }, - { name = "limits", specifier = ">=5.8.0" }, { name = "lxml", specifier = "==6.1.2" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "python-dotenv", specifier = "==1.2.3" }, @@ -1132,6 +1320,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192", size = 72226, upload-time = "2026-05-22T14:14:28.824Z" }, ] +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + [[package]] name = "openapi-pydantic" version = "0.5.1" @@ -1192,6 +1479,100 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + [[package]] name = "py-key-value-aio" version = "0.4.5" @@ -2099,65 +2480,83 @@ wheels = [ ] [[package]] -name = "wrapt" -version = "2.3.0" +name = "yarl" +version = "1.24.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/b0/c1f5a970721f06b85c0cd5142e0ff8fe067708abd779b0c4f4be7d61d09f/wrapt-2.3.0.tar.gz", hash = "sha256:681a2d0eefd721998f90642762b8e75c2159ec531b20ad5e437245ea7b06a107", size = 131509, upload-time = "2026-07-28T06:06:14.895Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5b/4a/d17a0fad1bf1c5f2c887ff71fef75654141b0880bff71d157d955b5bec3a/wrapt-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0a45ffae742ce91a16e11cb6c7cd71e7f9994f3cbd283b962ab093f5c6dcf525", size = 82139, upload-time = "2026-07-28T06:04:35.082Z" }, - { url = "https://files.pythonhosted.org/packages/6e/55/51b92daaf6defb57f4dc56bdcce985400f75c6984a03ca5e78ccac717028/wrapt-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:69e477046f2237ef0bc6547544ee73008dc764ca26eff44f09e976d221b34d5d", size = 82723, upload-time = "2026-07-28T06:04:36.502Z" }, - { url = "https://files.pythonhosted.org/packages/28/7f/cfd9bc4b1f5e424eeea83d0493e43f3b1b02707ce8e50c47945873982bd5/wrapt-2.3.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d221a6e6ddd302b8397433184e96b59f259f50024b854db1c411a881586b6b8", size = 172381, upload-time = "2026-07-28T06:04:37.674Z" }, - { url = "https://files.pythonhosted.org/packages/cb/89/ff7814f6eb6856b479946117d1138a2fbb46cdb6b1f379db359056c69743/wrapt-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:392158c9a7f2ab1b8699418bfc0fe6f83548788c418b27d7bf2019ad3405cebb", size = 174120, upload-time = "2026-07-28T06:04:38.987Z" }, - { url = "https://files.pythonhosted.org/packages/12/1e/8eded8615d39e3ce81f626937a3a87b280a2a86239a2bf14a4b4bb345034/wrapt-2.3.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e5301c35cf75655eb33498f2bd6ae8703ca19940e3167dc9cdf740c712a39c60", size = 163035, upload-time = "2026-07-28T06:04:40.361Z" }, - { url = "https://files.pythonhosted.org/packages/35/ea/a0af2d9da62897af2a055484920de05dade30d2ba2c0d65cbdea875d3d8b/wrapt-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:418f54bb09d1762db02c7009b4051149893af3153a87f92d70356703c11eea02", size = 171887, upload-time = "2026-07-28T06:04:41.614Z" }, - { url = "https://files.pythonhosted.org/packages/7e/dd/63cd4c864c65ef4906df64bd2d378f4a62b54f28063f282dfb3bf93caead/wrapt-2.3.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1598becd30f8f2777d18564064eb4f4dbe1ab0e05a8f09786d0ef505ac782bf3", size = 161113, upload-time = "2026-07-28T06:04:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ee/82f1fc9e431b5c2c5a6d201aa865dbeae3984c311c6d11a185f0c8367cf6/wrapt-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3da470536bf9645143323dd41b32db55c6f4304ad382094c1a1da8a92061e10d", size = 170530, upload-time = "2026-07-28T06:04:44.212Z" }, - { url = "https://files.pythonhosted.org/packages/37/a5/5dc590e863a419930d988f8b7ca3e75a6befcfb10b6003b3a152f3d5f732/wrapt-2.3.0-cp312-cp312-win32.whl", hash = "sha256:fb8e2e6704a1e0b1b989546c69e2688371ef4a07fa5f61bde3eb6211186f5ac1", size = 78323, upload-time = "2026-07-28T06:04:45.484Z" }, - { url = "https://files.pythonhosted.org/packages/51/f9/4a6925a07951df56394f7e6ebe14f69f1c5ef9d87aa63e0839acf15aa63a/wrapt-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:cdc021cb0b62471d6aac7f2bd92f3b4658073775f9ee7fcd325c511129e7bcc8", size = 81180, upload-time = "2026-07-28T06:04:47.021Z" }, - { url = "https://files.pythonhosted.org/packages/a8/4f/8b5de0395b2a72216751d41c9861df6facaeb611b619d8810ed2b3b23eb2/wrapt-2.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:67bfe2485f50368c3fcd2275fc1fd100e350d601e0058921a7c82678a465aeab", size = 80155, upload-time = "2026-07-28T06:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/8e/6e/0f88a072483e76b881e3fdcd6b6ffb4a5791002514fe541e72b1b73c859a/wrapt-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d3fb71e65b001adfc42684522eeccd9c21d8ba679945abc993439567b66e59f", size = 81960, upload-time = "2026-07-28T06:04:49.622Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ff/b7e2776e7c294075eb712cc9ef573d1b818f393006d09787262b8fc871c4/wrapt-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51a7a4181c1295774812271fbcd7c909df372bc25579d4ed9eb875caaf0ae86f", size = 82435, upload-time = "2026-07-28T06:04:50.9Z" }, - { url = "https://files.pythonhosted.org/packages/d8/90/343bb5d0f1f9669bc252a6073f085b4abf862511bd5c9c9eaec754341f1d/wrapt-2.3.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9045917809c63fdf7abe3a2ceaed3d670b8ee4500ddd9291192d30aeb34467c5", size = 170350, upload-time = "2026-07-28T06:04:52.187Z" }, - { url = "https://files.pythonhosted.org/packages/59/f8/13b79a392930bd0dd6b86cbfbfe1c40944110456e1dc6d809e5c46ece904/wrapt-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54ca1d5573f69b5fe1d74f1f65799c68015e82f685efec9fd8cfa40a094c44d0", size = 170022, upload-time = "2026-07-28T06:04:53.599Z" }, - { url = "https://files.pythonhosted.org/packages/b2/fc/4f1b6918f5290db959d6e0c07f77385d87cede29c39c9cf8f145e9c82954/wrapt-2.3.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:242b60c21e30866e6a2fa606c612b47c553fa60c0eaeeeb7797fb842ac0ce609", size = 161043, upload-time = "2026-07-28T06:04:54.936Z" }, - { url = "https://files.pythonhosted.org/packages/01/e1/45d3cf74414780bdff6d0380467e003f6eb0f028b6c9403db868dbc7209c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3f3d7ec0a51fbfe00d3aef047641ff2c58b25565b4717fc1f90e050be01cba8", size = 168576, upload-time = "2026-07-28T06:04:56.261Z" }, - { url = "https://files.pythonhosted.org/packages/f3/73/2fa58dd97f191c997755e2c6d569a68f0c433db4e4b36099bdd7227b6cac/wrapt-2.3.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:261f53870cd4fb2bf38f9f972c56c728fd224cb7c65721307de59d9e7e6741ae", size = 159140, upload-time = "2026-07-28T06:04:57.754Z" }, - { url = "https://files.pythonhosted.org/packages/29/a8/08a56e2000a8816d449dcbad8c8b081697acbbd490821ceca0f9d8e8d20c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8159ec0b0cb7608175eb150de94c19e34f4d47ac655f5ca9baf45df6b688ffd3", size = 169263, upload-time = "2026-07-28T06:04:59.161Z" }, - { url = "https://files.pythonhosted.org/packages/9e/d4/354e1725e35a73b2af4fa70a3e024c7a5d1bf1802dfb862dcb668aae0253/wrapt-2.3.0-cp313-cp313-win32.whl", hash = "sha256:10461884b3014fbfc8eb7d09a93c5f246363e6711d9d881f95eb8c27fdef049f", size = 78241, upload-time = "2026-07-28T06:05:00.507Z" }, - { url = "https://files.pythonhosted.org/packages/6c/7e/34c87fa2174848dfee820322aaa318bab08913998ccecc8d2f57b4ad4639/wrapt-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:ac870cc97b73bb00ac353329e9559a4bebc47c4c86792ed9b23b58c15b6ad838", size = 81113, upload-time = "2026-07-28T06:05:01.839Z" }, - { url = "https://files.pythonhosted.org/packages/11/86/fcc9a530579e008c9478bb565a6cdfbfd33536660f069c8b91a6607c5050/wrapt-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:a65e8db2b4e90c2e7ade931086351c98ef420bf7a94ee08c95ac8a3cbbc43579", size = 80182, upload-time = "2026-07-28T06:05:03.152Z" }, - { url = "https://files.pythonhosted.org/packages/96/50/3864848b95b28ef73e17551fc8dccbff2628a834f52cf26a57f9c419fb83/wrapt-2.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:fd1f2f557dd3491fe75905e578f4db967393d40d1a8f468edc4d40ac7f2d5944", size = 83921, upload-time = "2026-07-28T06:05:04.476Z" }, - { url = "https://files.pythonhosted.org/packages/3b/4c/3d1921a60c3e8c71c540ff136e6a47a1fbccf7f671e818394889f7871d9c/wrapt-2.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9f5d2aec29dfc76c37e23897dee92766a3fd4f3bff3ae7fc9c6b4bf37d8c1360", size = 84412, upload-time = "2026-07-28T06:05:05.921Z" }, - { url = "https://files.pythonhosted.org/packages/fa/1a/4a796ff7adb26ada6d4b758c94d47a38320b085e7099afc088efbbcdb006/wrapt-2.3.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:646d20d413ffcd1b0a2f700076e2d0252d872dcb7754860a73e45a59ea883614", size = 207168, upload-time = "2026-07-28T06:05:07.256Z" }, - { url = "https://files.pythonhosted.org/packages/1d/3e/d7777776806c579b761bac2f91721dda9f04c7a1b380213c5935cc750ae6/wrapt-2.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:379f670f45b7bb8993edd9f6fc36c6cc65edb81cffa0b504be34acb0303fff0a", size = 214351, upload-time = "2026-07-28T06:05:08.945Z" }, - { url = "https://files.pythonhosted.org/packages/63/27/2d64d394df7bf181955b3bb562bf33c4492fb4be113f53071106d43ad8b5/wrapt-2.3.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6208f302f110295d64b22a7ac96500c791bf492dce4366e622e4912b077c9687", size = 199020, upload-time = "2026-07-28T06:05:10.418Z" }, - { url = "https://files.pythonhosted.org/packages/3e/3d/fb31d3db7d9834d265fb1a27a2adf0ddf51557c67458c97b22439ad6ae3d/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ed635a9ca4f3a5a2b900c10c69e823373bc00ebc114b459383596d3487da3570", size = 209969, upload-time = "2026-07-28T06:05:11.983Z" }, - { url = "https://files.pythonhosted.org/packages/1f/d1/8724b5da582e62070dc9bf4d8bf1972f317297eefd7ba1f2b5c6393ccf6c/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:e3b9eaa742ae7a0aaaaad4ca4b69469d757af2d6e6663ef1dadc47adec0aeb41", size = 196324, upload-time = "2026-07-28T06:05:13.557Z" }, - { url = "https://files.pythonhosted.org/packages/0d/5c/3d9ef411149543016ee6bcf3af707f787cebd946527452b94bf122e9b7b4/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d0f7284f88f4833705132d06d3b425a43095c2cbd07c58166aac3ab646ba12a4", size = 202610, upload-time = "2026-07-28T06:05:15.048Z" }, - { url = "https://files.pythonhosted.org/packages/13/9b/4fc042ceb757866dd4a5fc057b3b736f2b360d3703ce9f830d83dc9226e0/wrapt-2.3.0-cp313-cp313t-win32.whl", hash = "sha256:7ebb274aba688b043429eb1500ff8a76ce0cb8ac0812ca3e301f06247b8722b3", size = 79178, upload-time = "2026-07-28T06:05:16.469Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ff/b94878f8eed809ca042685276bcea9f24e8c2ca7c9653bb80bbb920a68a5/wrapt-2.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c4bded758ad6f03b965830944a2f0bc5b2eb3767fe5a7310134315d1a6610e98", size = 82634, upload-time = "2026-07-28T06:05:18.026Z" }, - { url = "https://files.pythonhosted.org/packages/80/fb/663e1de5332a71685a729754312d327d4cada767c36e1c5a2db4c8de49e6/wrapt-2.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:d2cc64539da63e39ffb9c7ede849b6e8ddaaf7b3876b5cfb04efd85a5f3f4eb6", size = 81387, upload-time = "2026-07-28T06:05:19.417Z" }, - { url = "https://files.pythonhosted.org/packages/58/10/b073beaea89bc0d3670a75ff51139430a54b6af7ba7796507730634536dd/wrapt-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea52a0d0f08c584943d5764be0e84efa912c8da23c23e1e285ff2f5641c18fcc", size = 81978, upload-time = "2026-07-28T06:05:21.133Z" }, - { url = "https://files.pythonhosted.org/packages/b3/31/0916d9cebf848ed3f1a0c1888faee421747df77331e4db2bc527a9a85988/wrapt-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd85b0aa88efdb189d6ae2f35f4526943a8f091c38599c9c31478241c819e6a1", size = 82518, upload-time = "2026-07-28T06:05:22.562Z" }, - { url = "https://files.pythonhosted.org/packages/f5/73/31c1bf0f3384062751c2094dadb314916d70aa9b6bfd26d994b4a7b393fa/wrapt-2.3.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:141ed6211286a9660d8d6702de598b43f0934b4f0eda16393f100a80f501d945", size = 170187, upload-time = "2026-07-28T06:05:23.904Z" }, - { url = "https://files.pythonhosted.org/packages/ed/25/fce087d54b79b8905f3c3c9dd5f454bbd8d8acb80b960c4a6aee5b4659b3/wrapt-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e49885a62ec4ee854d1b9e6371fda6afd219917225752abf729a3f36d4df9a5", size = 169288, upload-time = "2026-07-28T06:05:25.378Z" }, - { url = "https://files.pythonhosted.org/packages/c7/30/0d09e6dddc6b7a7230ac77f50254b5980ab4fcd22976f72f8cc8a0404458/wrapt-2.3.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d6159c9b2fefec02314e1332dbbbfaf960e369dfd26bcf7f8b258b5732065b3", size = 160932, upload-time = "2026-07-28T06:05:27.022Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ca/0913af0d2ec0c43865d32d615f518fea66c13c5c930e489e9b0de248e9a8/wrapt-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24da48596326ef8e448cfa837b454f638713d3531262375f00e5a9681682fc07", size = 169017, upload-time = "2026-07-28T06:05:28.501Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f2/3d1e47ea81b822210f5df1bf942fd90780a75c055243d569b664529dea88/wrapt-2.3.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cd3a2edf0427013736b8127955cec62608c56e53ea47e82812ea32059cda407f", size = 159065, upload-time = "2026-07-28T06:05:30.01Z" }, - { url = "https://files.pythonhosted.org/packages/43/a5/ef2066ced8e5fca204e2b361e9708e36555b40949c583d997ea3b590817d/wrapt-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fa0df3bff4e7ce45759f33fd39335fe2f60477bb9ecf7b8aa41e7d07ee36a23", size = 168821, upload-time = "2026-07-28T06:05:31.649Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e1/016104650d4e572fa91506eb396b3dd8efbccc9284fdc1c9479c3d21db28/wrapt-2.3.0-cp314-cp314-win32.whl", hash = "sha256:2935d5454b3f179a29b12cf390ee47246740ba2c3a7545b1b46ba31a5f2a4a0b", size = 78700, upload-time = "2026-07-28T06:05:33.391Z" }, - { url = "https://files.pythonhosted.org/packages/3d/97/6fdc20a9f2ca304748b3f0819cbf377d55260562777bf0b615431bc3c181/wrapt-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:cc2cea812e5cb179a796b766747e7d3b21088760d8deb95676d482b8c8e6fa7d", size = 81422, upload-time = "2026-07-28T06:05:34.774Z" }, - { url = "https://files.pythonhosted.org/packages/5e/a4/9cbd53bf05746bea2c392af39cb052427a8ec95cbd494d930733d8f44681/wrapt-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:22cc5c0a717bd4da87018ae0bffd4c19c6fb679d3ff357216ba566ab26c76cab", size = 80639, upload-time = "2026-07-28T06:05:36.228Z" }, - { url = "https://files.pythonhosted.org/packages/43/bb/6c5e4a0f66ea0d2b2dd267e8dd05a0014eea56840b3c8595d40b0a5d1f91/wrapt-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a6b5984cd65dd639546f0eb4b8eacf1c31cb2fe9fb5c27bffe240987cdb2cf84", size = 84030, upload-time = "2026-07-28T06:05:37.714Z" }, - { url = "https://files.pythonhosted.org/packages/6a/eb/a1aedf03283bc9cbf8a1783995ddc54e3c5a86878f19002d2c428494f4c5/wrapt-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c88abcf53daef80e01a75c7530e727fa6e2c1888fe83e3dcdba4c96216a1f5c7", size = 84419, upload-time = "2026-07-28T06:05:39.131Z" }, - { url = "https://files.pythonhosted.org/packages/63/61/50d511c0dc5105563849e86daa3e16ac7feef699f79fb05af45ea70107d5/wrapt-2.3.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85de890ff968196e92dd1ae73a9fb8970495e7650a457b1c9ef0ac3dd550bce2", size = 207171, upload-time = "2026-07-28T06:05:40.69Z" }, - { url = "https://files.pythonhosted.org/packages/3f/59/9b538cf7795217e810699d16bc88b96a830d9b5c403eb2ec2db6b5f2ae81/wrapt-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50f416b74d092bb9f41b424e90dd457f365f7ba4b11de62a23679769a21bd85c", size = 214329, upload-time = "2026-07-28T06:05:42.287Z" }, - { url = "https://files.pythonhosted.org/packages/b3/28/9935d62b1499e5c8b3d191e99ba4eb31ca237a0b699142011a837e9dc7ea/wrapt-2.3.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39febbee6d77301d31da6996b152ce52452da7c7ef72aba10c2fa976dff9c295", size = 199079, upload-time = "2026-07-28T06:05:43.958Z" }, - { url = "https://files.pythonhosted.org/packages/2b/01/4446b80fa2ffa47a3449b250d004ba1c1937f07f64a179608fec735df866/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:93513bec052c6cd987f9f580c3df068c8bc4ebae6543736be3ca7ec5959cafcd", size = 209992, upload-time = "2026-07-28T06:05:45.677Z" }, - { url = "https://files.pythonhosted.org/packages/d4/07/56f26c9f9979586a021e8148747004aba4498f49458c90b0502969b904e1/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:729126e667da34d251b8ebf8a45ef0c5ddadc21542b3d6e1abf4259ece6508df", size = 196334, upload-time = "2026-07-28T06:05:47.608Z" }, - { url = "https://files.pythonhosted.org/packages/8b/41/6d7bcc895b0f28b2250e10908f060687b9165429dcd7f22ddb3d4c031b74/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:626b69db2021aa01671ec7bbc9740e558522bd44c18cf2ce69bf3d666a014109", size = 202644, upload-time = "2026-07-28T06:05:49.183Z" }, - { url = "https://files.pythonhosted.org/packages/cd/25/7860927edba06b758b8852a6f02e832be715563c67a6795d94350bc81099/wrapt-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:629d73378082c00a8173031f9fb30a3ac6abbc894a5bfdfae71fabc60642d501", size = 79685, upload-time = "2026-07-28T06:05:50.976Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0f/270bafe92fde3b069a39bc01e39ee79340895b335640df861d43d2a51885/wrapt-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:42869085687f0aefd57c0f636c3f9354f8ffb321a8ba9cb52d19beb796e561c5", size = 83104, upload-time = "2026-07-28T06:05:52.405Z" }, - { url = "https://files.pythonhosted.org/packages/55/b3/af176d79a8515a8a720eccdad9a96f6e31a30abf2865430c8c42adf2fd13/wrapt-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b1e5aa486e269b00ed35e64771c7d0ab8096cfd2643405ca8cd60ebedc099a51", size = 81774, upload-time = "2026-07-28T06:05:53.902Z" }, - { url = "https://files.pythonhosted.org/packages/00/39/3daf9f47be208606586de4568ba6713db53ebc8fd7a575aea1fe57983b69/wrapt-2.3.0-py3-none-any.whl", hash = "sha256:d8c7ed08477429752b8c44991f40ad7838b18332a160698740a6bfbc10d998a2", size = 61866, upload-time = "2026-07-28T06:06:12.9Z" }, +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, + { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, + { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, + { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, + { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, + { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, + { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, + { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/e1/63/64ef361967cc983573149dc1515d531db5da8a4c92d22bb833d59e01b313/yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2", size = 135075, upload-time = "2026-07-20T02:05:59.671Z" }, + { url = "https://files.pythonhosted.org/packages/bb/89/55920fd853ce43e608adbc3962456f0d649d6bb15250dc2988321da0fe1c/yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb", size = 97225, upload-time = "2026-07-20T02:06:01.769Z" }, + { url = "https://files.pythonhosted.org/packages/15/f0/7688d3f2cfff7590df2af38ec46d969f4281a4dddb08a9ad2eafbcdddf98/yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075", size = 96751, upload-time = "2026-07-20T02:06:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/a851a0f94aaaf379dd4f901bfc80f634280bec51eb260b47363e2a4cd62e/yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff", size = 107960, upload-time = "2026-07-20T02:06:05.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a8/faea066c12f9c77ca0de90641f1655f9dd7b412477bf28c76d692f3aecff/yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448", size = 103500, upload-time = "2026-07-20T02:06:07.556Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9c/1e67084c2a6e2f2db0e3be798328cb3be42c0119b621d25461479a224d21/yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f", size = 115780, upload-time = "2026-07-20T02:06:09.599Z" }, + { url = "https://files.pythonhosted.org/packages/58/86/1f94664e147474337e3359f52012cf3d02f825f694317b178bfba1078c62/yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd", size = 115308, upload-time = "2026-07-20T02:06:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/0a/43/8e55ae7538ba5f28ccb3c845c6dd4549cf7016d5992e5326512519107cdd/yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16", size = 110574, upload-time = "2026-07-20T02:06:13.129Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ba/a889ec8765cedcf2ac44dcb02d6a21e4861399b243b263c5f2dde27ee740/yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213", size = 109914, upload-time = "2026-07-20T02:06:15.243Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c3/e45f821af67b791c2dbbe4a9f4137a1d33f8d386654a05a0c3f47bdfa25d/yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24", size = 107712, upload-time = "2026-07-20T02:06:17.443Z" }, + { url = "https://files.pythonhosted.org/packages/02/00/2ab0f42c9857fcb490bfaa6647b14540b53d241ab209f23220b958cc5832/yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385", size = 104251, upload-time = "2026-07-20T02:06:19.259Z" }, + { url = "https://files.pythonhosted.org/packages/7a/70/709d9a286e98af2c7fd8e4e6cada658b5c0e30d87dd7e2a63c2fb5767217/yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c", size = 115319, upload-time = "2026-07-20T02:06:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6c/3eaa515142991fe84cfc483ff986492211f1978f90161ccefdbec919d09b/yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4", size = 109163, upload-time = "2026-07-20T02:06:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/711dafce66c323a3144d470547a71c5384c57623308ac8bb5e4b903ac148/yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144", size = 115435, upload-time = "2026-07-20T02:06:24.923Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f3/9b9d0e6d84bea851eb1ba99e4bdc755b86fd813e49ec86dfe42f26befdef/yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4", size = 110691, upload-time = "2026-07-20T02:06:26.973Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/62a06b7e87c4246ac76b7c2da136f972eb4a3a1fc94abb07e7022d6fdb0a/yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740", size = 97454, upload-time = "2026-07-20T02:06:29.163Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/5fc8025b318ab10db413b61056bd0d95c557a70e8df4210c7511f866329c/yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1", size = 92813, upload-time = "2026-07-20T02:06:31.113Z" }, + { url = "https://files.pythonhosted.org/packages/a9/08/5f3085fef9564217074db9dd8573de1795bc82cde61a7ad10b6a7234a569/yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76", size = 135680, upload-time = "2026-07-20T02:06:33.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/35/ba9436e579bd48a8801f2021d842d9ab4994c26e4c7dd3a4c1f1bcb57a9e/yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d", size = 97395, upload-time = "2026-07-20T02:06:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/18/a9/a07f76f3c44e02b25cc743af5ef93eef27f7013eadca770451b6a6ccb5db/yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75", size = 97223, upload-time = "2026-07-20T02:06:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/77/f7/a9a1d6fa7dd9e388f95b30f6ad3ec4e285f6c8f61f44ce16070c3fcfe414/yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9", size = 108777, upload-time = "2026-07-20T02:06:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/2f/44/e0b86c302471fabd6f02808ecf2ac52b8412b624787849d4bf2cdb466f6f/yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede", size = 103119, upload-time = "2026-07-20T02:06:41.456Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/9c16d180bf8faaf223225eb50e1245870ff1ae0e302a27153988e65c51fd/yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca", size = 116471, upload-time = "2026-07-20T02:06:43.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/b219b9df28a02ce95cfbdd41d2f7caa5669d0ff979c1c9975697145e33c5/yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027", size = 115974, upload-time = "2026-07-20T02:06:45.874Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e8/f20557aca240d88e69850ad1ee91756821d094bb1310565c04d25c6682a2/yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9", size = 110830, upload-time = "2026-07-20T02:06:47.852Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/199b85109a53eeca64ee19c9cca228287e8e4ab0cc1a09b28f530e65cce0/yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41", size = 110054, upload-time = "2026-07-20T02:06:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/ed28147f8cd7f48c49367c90713b30a555284b6105a6a56f3a05568da795/yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373", size = 108312, upload-time = "2026-07-20T02:06:51.835Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/55e16ae0a5c227cea8df1c6871ba57d614a34243146c05729caf2a1bd9c5/yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36", size = 103662, upload-time = "2026-07-20T02:06:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ea/dbd7c2caec459c9a426f18b02688ecbfb58620d0f6a3422d24769fbaf8ab/yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0", size = 116090, upload-time = "2026-07-20T02:06:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/06/84/39ce4ce3059e07fece5fbdbee8c4053406af9aca911ce9fa5f8548aab6af/yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5", size = 109523, upload-time = "2026-07-20T02:06:57.926Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8b/71ff44137b405c64a7788075669c24010019f57a7464b78c3a6cbee539d9/yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5", size = 116084, upload-time = "2026-07-20T02:06:59.868Z" }, + { url = "https://files.pythonhosted.org/packages/62/c0/423078fdd4042e1862c11f0ffd977a0ffa393783c12bee94685923bc189e/yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4", size = 111006, upload-time = "2026-07-20T02:07:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/6daa2ee9d95e5c98b8128f8df91eb692eb423ab274b8cf08db52152fad26/yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad", size = 99215, upload-time = "2026-07-20T02:07:03.852Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0e/464a847d7359e0da75dd9fc5c1d1aa35d0159ea31e5f8e66a3c1c29ff3d0/yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f", size = 94566, upload-time = "2026-07-20T02:07:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/e03acc4446772660bc335e86e41ef31e4d0d838fd641531a11a5ee33b493/yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88", size = 142533, upload-time = "2026-07-20T02:07:08.284Z" }, + { url = "https://files.pythonhosted.org/packages/ae/71/4acd3a1fc7cf14345cdb302665ecd2097f62c365b4f14ca17d4f37775cf9/yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba", size = 100776, upload-time = "2026-07-20T02:07:10.197Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/cfb76b7fe99686db264bff829779a539d923e7564ffd7ef18da6c54c3774/yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928", size = 100913, upload-time = "2026-07-20T02:07:12.357Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3f/7116e782992abbd4fb6948488aec72078895e929a23078290739e8396fce/yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f", size = 106507, upload-time = "2026-07-20T02:07:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/d4d2d73ee78229cc889872eb8e085d8f5c6f51abdb178409fd9b23cf74fd/yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95", size = 99219, upload-time = "2026-07-20T02:07:16.019Z" }, + { url = "https://files.pythonhosted.org/packages/3e/fa/a6df1a9bccd644eec00abee0dff4277416222cec435330fd1f2858523ec1/yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc", size = 111804, upload-time = "2026-07-20T02:07:18.141Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/7b2a1f4bcc20e9447156dd2b1c4d01f70d9df0759025ee7d09a84ffae134/yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da", size = 110943, upload-time = "2026-07-20T02:07:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/22c92affb0f9b623ca753d27d968b5625b868f12c6378d049d55ae247643/yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a", size = 108251, upload-time = "2026-07-20T02:07:22.217Z" }, + { url = "https://files.pythonhosted.org/packages/45/44/5769b96298c1e195fb412997b6090af2a84105cf59c17613558a2d011d1f/yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0", size = 106025, upload-time = "2026-07-20T02:07:24.083Z" }, + { url = "https://files.pythonhosted.org/packages/4c/40/009e8e791fd9762c0e1567e69248acb4f49064597e1680874c16dd8bb798/yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498", size = 106573, upload-time = "2026-07-20T02:07:26.248Z" }, + { url = "https://files.pythonhosted.org/packages/20/c6/b7480578f8a0a80946f36ad6df547ecec704f9ba69d2de60f8aa6f1c1cbf/yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104", size = 100751, upload-time = "2026-07-20T02:07:28.098Z" }, + { url = "https://files.pythonhosted.org/packages/d4/27/4476f3360b91a48c5cf125e91f59a3bd35299d84a431a258d57f5977bb11/yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331", size = 111643, upload-time = "2026-07-20T02:07:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/5cdd3e5ee944e8af31e52f6cd3d3af5fd7b937e036ccbbba2c9ffebede95/yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550", size = 106312, upload-time = "2026-07-20T02:07:33.06Z" }, + { url = "https://files.pythonhosted.org/packages/18/86/f406b0c2a6f99575de2da671ef47aa06f89a5be83a27a46971c3b86cecdb/yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6", size = 110379, upload-time = "2026-07-20T02:07:35.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/9f3adfbd3b30b4fa0f7ccb3a83eba2c1152d3fff554d535e640ba0f7ba2b/yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047", size = 108497, upload-time = "2026-07-20T02:07:37.35Z" }, + { url = "https://files.pythonhosted.org/packages/dd/37/91eb2e5ca883a529c1b390348a74cd9fc0512171727f547ce70bfe02be5c/yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104", size = 102450, upload-time = "2026-07-20T02:07:39.578Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, ] From 974cc5a0b21642235cb4b1085a1a3deefbd80c7c Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Fri, 4 Sep 2026 11:28:17 +0200 Subject: [PATCH 13/55] fix!: make every tool failure a visible, actionable error A failed call now always reaches the caller as is_error with a message it can act on. Three mechanisms coexisted before: raising, returning an error inside a successful payload, and per-item statuses. Only the last one was right, and it stays where it belongs. RMES failures were the worst case. A timeout, a syntax error or an unreachable endpoint returned is_error=False with the reason tucked into an `error` field, so a failed query looked like a success to the caller. - AppToolError replaces fail(). Being a class, `raise` is visible at the call site, so the eight dead `raise` statements that followed fail() are gone and cannot come back. - mask_error_details is on. Deliberate messages pass through untouched; anything else is a bug and is replaced by a generic message. - ErrorHandlingMiddleware no longer transforms. Its default promoted our errors to JSON-RPC protocol errors labelled "Internal error", losing is_error. - One vocabulary. RMES had six codes of its own; they map onto the shared set. - An empty result is no longer an error: search_melodi_modalities returns an empty list and a count. - get_insee_document no longer copies raw exception text into its results. That text is a returned value, so masking could never redact it. Also fixes a crash: _fetch_html referenced a variable removed when the client gained a base_url, so every insee.fr timeout, 404 and network error raised NameError instead of the intended message. Function names now lead with a verb: _build_failed_document, _match_exact, _match_prefix, _strip_graph_base, compute_current_date_iso. BREAKING CHANGE: the `error` field is removed from RMES_list_graphs, RMES_describe_resource and RMES_run_sparql. A client reading `output.error` must instead check is_error on the call result and read the message. --- src/mcpdiffusion/config/tool_metadata.py | 8 +- src/mcpdiffusion/core/errors.py | 43 +++--- src/mcpdiffusion/models/rmes.py | 23 ---- src/mcpdiffusion/server.py | 8 +- src/mcpdiffusion/services/insee_document.py | 61 +++++---- src/mcpdiffusion/services/melodi.py | 35 ++--- src/mcpdiffusion/services/rmes.py | 123 +++++++----------- .../tools/insee_search_chiffrecle.py | 6 +- .../tools/insee_search_conjoncture.py | 6 +- .../tools/insee_search_documents.py | 6 +- 10 files changed, 130 insertions(+), 189 deletions(-) diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py index 65ffc20..a8124d1 100644 --- a/src/mcpdiffusion/config/tool_metadata.py +++ b/src/mcpdiffusion/config/tool_metadata.py @@ -15,7 +15,7 @@ # Fixme: I'd put such a generic function in a separate module # just a preference, not mandatory -def current_date_iso() -> str: +def compute_current_date_iso() -> str: """Return today's date as ISO-8601.""" return date.today().isoformat() @@ -210,7 +210,7 @@ def current_date_iso() -> str: "List of publications: `{ id, score, titre, soustitre, chapo, " "anneediffusion, zone, theme, url }`. Feed `url` to `get_insee_document`.\n" "\n" - f"Current date is {current_date_iso()}.\n" + f"Current date is {compute_current_date_iso()}.\n" ), "tool_metadata": {"version": "6.0", "author": "mirlon"}, } @@ -260,7 +260,7 @@ def current_date_iso() -> str: "A list of publications: `{ id, score, titre, soustitre, chapo, " "anneediffusion, zone, theme, url }`.\n" "\n" - f"Current date is {current_date_iso()}.\n" + f"Current date is {compute_current_date_iso()}.\n" ), "tool_metadata": {"version": "5.0", "author": "mirlon"}, } @@ -293,7 +293,7 @@ def current_date_iso() -> str: "3. Follow up with `search_insee_documents` or `search_insee_conjoncture` " "only if the user needs deeper tables or historic series.\n" "\n" - f"Current date is {current_date_iso()}.\n" + f"Current date is {compute_current_date_iso()}.\n" ), "tool_metadata": {"version": "5.0", "author": "mirlon"}, } diff --git a/src/mcpdiffusion/core/errors.py b/src/mcpdiffusion/core/errors.py index eaa87e8..1eadb86 100644 --- a/src/mcpdiffusion/core/errors.py +++ b/src/mcpdiffusion/core/errors.py @@ -1,14 +1,11 @@ -"""Standardized error conventions for MCP tools.""" -from __future__ import annotations +"""The single error type tools and services raise.""" from typing import Literal from fastmcp.exceptions import ToolError - ErrorCode = Literal[ "INVALID_INPUT", - "EMPTY_RESULT", "BACKEND_UNAVAILABLE", "UPSTREAM_ERROR", "PARSE_ERROR", @@ -17,22 +14,26 @@ "UNKNOWN", ] -# Fixme: fail returns None while it always raises which can fool type checkers -def fail( - code: ErrorCode, - message: str, - retryable: bool = False, -) -> None: - """Raise a standardized tool error. - `message` should be actionable: name the offending parameter, suggest - the next step, include the shortest useful excerpt of the upstream error. +class AppToolError(ToolError): + """A failure the caller is meant to read and act on. + + Subclasses ToolError, so the message survives `mask_error_details`. Every other exception is + a bug and gets replaced by a generic message. The code and retryability are attributes for + logging and tests, and are rendered into the message because that text is all the caller gets. + + The message must name what failed, why, and what to do next — the offending parameter, or the + tool that produces a valid value. """ - # Fixme: the prefix can be built once using an f string instead of being re-assigned - prefix = f"[{code}] " - if retryable: - # Fixme: it might be better to leverage a separate error flag for information like retryable - # so the information is easier to identify vs baked into a string - # this can be achieved by subclassing ToolError, I guess - prefix = f"[{code}, retryable] " - raise ToolError(prefix + message) + + def __init__( + self, + code: ErrorCode, + message: str, + *, + retryable: bool = False, + ) -> None: + self.code = code + self.retryable = retryable + marker = f"{code}, retryable" if retryable else code + super().__init__(f"[{marker}] {message}") diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index 073fc0e..a57dda0 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -37,26 +37,6 @@ class GraphCategoryChoice(StrEnum): AUTRE = "autre" -# --- Error types --- - -# Fixme: I am afraid these error models might compete with what is defined in 'core/errors.py' -# We should provide uniform errors accros the application to simply parsing for clients -class SparqlErrorType(StrEnum): - INVALID_QUERY_FORM = "INVALID_QUERY_FORM" - TIMEOUT = "TIMEOUT" - SYNTAX_ERROR = "SYNTAX_ERROR" - HTTP_ERROR = "HTTP_ERROR" - NETWORK_ERROR = "NETWORK_ERROR" - EMPTY_QUERY = "EMPTY_QUERY" - - -class SparqlError(BaseModel): - type: SparqlErrorType - message: str - query: str - endpoint_message: Optional[str] = None - - class GraphRow(BaseModel): graph: str triples: int @@ -101,7 +81,6 @@ class CategoryBucket(BaseModel): class ListGraphsOutput(BaseModel): total_graphs_matched: int categories: list[CategoryBucket] - error: Optional[SparqlError] = None # --- RMES_describe_resource --- @@ -134,7 +113,6 @@ class DescribeResourceOutput(BaseModel): uri: str properties: list[ResourceProperty] count: int - error: Optional[SparqlError] = None # --- RMES_run_sparql --- @@ -163,4 +141,3 @@ class RunSparqlOutput(BaseModel): variables: Optional[list[str]] = None bindings: Optional[list[dict[str, Any]]] = None turtle: Optional[str] = None - error: Optional[SparqlError] = None diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 4a245fe..1892c0f 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -20,6 +20,9 @@ mcp = FastMCP( "INSEE-mcp-diffusion", + # Only AppToolError messages reach the caller; anything else is a bug and is replaced + # by a generic message. + mask_error_details=True, lifespan=build_lifespan( es_host=settings.es_host, es_tls_verify=settings.es_tls_verify, @@ -40,7 +43,10 @@ # None of them logs how many results a tool returned. If empty results become hard to diagnose, add an # `on_call_tool` middleware that inspects the ToolResult, or have the tool report it with `ctx.info`. mcp.add_middleware( - ErrorHandlingMiddleware(), + # transform_errors would promote our ToolErrors to JSON-RPC protocol errors labelled + # "Internal error", losing is_error and the message the LLM is meant to act on. We only + # want the logging and error counting. + ErrorHandlingMiddleware(transform_errors=False), ) mcp.add_middleware( SlidingWindowRateLimitingMiddleware( diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 2d7f415..1141dd3 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -8,15 +8,19 @@ from trafilatura import extract from trafilatura.settings import Extractor +import logging + import httpx -from ..core.errors import fail +from ..core.errors import AppToolError from ..models.insee import ( DocumentResult, GetInseeDocumentInput, GetInseeDocumentOutput, ) +logger = logging.getLogger(__name__) + _TRAFILATURA_OPTIONS = Extractor( output_format="markdown", links=True, @@ -104,42 +108,47 @@ def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: async def _fetch_html(url: str, http_client: httpx.AsyncClient) -> str: # A relative path resolves against the client's base_url; an absolute one overrides it. + target = http_client.base_url.join(url) try: response = await http_client.get(url, follow_redirects=True) response.raise_for_status() return response.text - # Fixme: the error handling is not correctly designed, at a global scale - # Fixme: for example, here, the 'fail' invocation raises a ToolError nesting any information within a string - # so an error is logged twice and the client ultimately receives an error string - # he can hardly react on automatically except httpx.TimeoutException as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", - f"insee.fr timed out fetching {full_url}: {exc}", + f"insee.fr timed out fetching {target}: {exc}", retryable=True, ) - raise except httpx.HTTPStatusError as exc: if exc.response.status_code == 404: - fail( + raise AppToolError( "NOT_FOUND", - f"INSEE document not found at {full_url} (HTTP 404). " + f"INSEE document not found at {target} (HTTP 404). " "Verify the URL with `search_insee_documents`.", ) else: - fail( + raise AppToolError( "UPSTREAM_ERROR", - f"insee.fr returned HTTP {exc.response.status_code} for {full_url}.", + f"insee.fr returned HTTP {exc.response.status_code} for {target}.", retryable=(500 <= exc.response.status_code < 600), ) - raise except httpx.HTTPError as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", - f"Network error fetching {full_url}: {exc}", + f"Network error fetching {target}: {exc}", retryable=True, ) - raise + + +def _build_failed_document(url: object, message: str) -> DocumentResult: + return DocumentResult( + id=str(url), + status="error", + markdown_content=None, + sommaire=None, + truncated=False, + error=message, + ) async def get_insee_document( @@ -148,7 +157,7 @@ async def get_insee_document( http_client: httpx.AsyncClient, ) -> GetInseeDocumentOutput: if not params.list_of_url: - fail( + raise AppToolError( "INVALID_INPUT", "list_of_url must contain at least one URL. " "Use `search_insee_documents` to find URLs first.", @@ -181,16 +190,12 @@ async def get_insee_document( error=None, ) ) - except Exception as exc: - results.append( - DocumentResult( - id=str(url), - status="error", - markdown_content=None, - sommaire=None, - truncated=False, - error=f"{type(exc).__name__}: {str(exc)[:500]}", - ) - ) + except AppToolError as exc: + # A typed failure is written for the caller, so it is safe to pass on. + results.append(_build_failed_document(url, str(exc))) + except Exception: + # Anything else is a bug: log it here, tell the caller only that this URL failed. + logger.exception("Unexpected failure fetching %s", url) + results.append(_build_failed_document(url, "[UNKNOWN] Could not fetch this document.")) return GetInseeDocumentOutput(results=results, count=len(results)) diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 194b090..3f81fdf 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -8,7 +8,7 @@ from elasticsearch import ConnectionError as ESConnectionError from elasticsearch import TransportError -from ..core.errors import fail +from ..core.errors import AppToolError from ..models.melodi import ( ColumnResult, DatasetSearchResult, @@ -37,17 +37,16 @@ async def get_melodi_observations( response.raise_for_status() # Fixme: the following problematic error handling pattern has already been adressed except httpx.TimeoutException as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"Melodi API timed out calling {url}: {exc}. Try again or narrow the query.", retryable=True, ) - raise except httpx.HTTPStatusError as exc: status = exc.response.status_code body_excerpt = (exc.response.text or "")[:500] if status == 400: - fail( + raise AppToolError( "INVALID_INPUT", f"Melodi API rejected the query (HTTP 400). " f"Columns/values passed: {params.dict_of_columns_and_values}. " @@ -55,38 +54,35 @@ async def get_melodi_observations( "Verify modality codes with `search_melodi_modalities`.", ) elif status == 404: - fail( + raise AppToolError( "NOT_FOUND", f"Melodi dataset {params.dataset_id!r} not found (HTTP 404). " "Check the dataset_id with `search_melodi_datasets`.", ) else: - fail( + raise AppToolError( "UPSTREAM_ERROR", f"Melodi API returned HTTP {status}: {body_excerpt}", retryable=(500 <= status < 600), ) - raise except httpx.HTTPError as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"Could not reach Melodi API at {url}: {exc}", retryable=True, ) - raise try: payload = response.json() except ValueError as exc: - fail( + raise AppToolError( "PARSE_ERROR", f"Melodi API returned non-JSON response: {exc}", ) - raise observations = payload.get("observations") if isinstance(payload, dict) else None if not isinstance(observations, list): - fail( + raise AppToolError( "PARSE_ERROR", "Melodi API response did not contain an 'observations' list.", ) @@ -205,13 +201,12 @@ async def search_melodi_datasets( body=body, ) except (ESConnectionError, TransportError) as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"Melodi datasets search backend unreachable: {exc}. " "Verify ES_HOST and try again.", retryable=True, ) - raise results: list[DatasetSearchResult] = [] for hit in ds_res.get("hits", {}).get("hits", []): @@ -284,13 +279,12 @@ async def search_melodi_modalities( }, ) except (ESConnectionError, TransportError) as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"Melodi columns search backend unreachable: {exc}. " "Verify ES_HOST and try again.", retryable=True, ) - raise results: list[ColumnResult] = [] for hit in ds_column.get("hits", {}).get("hits", []): @@ -320,13 +314,4 @@ async def search_melodi_modalities( ) ) - if not results: - fail( - "EMPTY_RESULT", - f"No modalities matched for dataset_id={params.dataset_id!r}, " - f"columns_id={params.columns_id!r}, " - f"french_query={params.french_query!r}. " - "Verify the dataset_id and column ids with `search_melodi_datasets`, " - "then try a broader French query.", - ) return SearchMelodiModalitiesOutput(results=results) diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 3f1460d..29ebe04 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -12,6 +12,7 @@ import httpx +from ..core.errors import AppToolError from ..models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, GRAPH_BASE, @@ -24,8 +25,6 @@ ListGraphsOutput, ResourceProperty, RunSparqlOutput, - SparqlError, - SparqlErrorType, RunSparqlInput, DescribeResourceInput, ListGraphsInput, @@ -79,12 +78,12 @@ def __init__(self, key: str, label: str, description: str, match: CategoryMatche self.match = match -def _exact(*paths: str) -> CategoryMatcher: +def _match_exact(*paths: str) -> CategoryMatcher: allowed = set(paths) return lambda path: path in allowed -def _prefix(prefix: str) -> CategoryMatcher: +def _match_prefix(prefix: str) -> CategoryMatcher: return lambda path: path.startswith(prefix) @@ -98,7 +97,7 @@ def _prefix(prefix: str) -> CategoryMatcher: "(pertinence, precision, actualite, coherence...) sous forme de " "sdmx-mm:ReportedAttribute. Tous ces graphes ont un schema identique." ), - match=_prefix("qualite/rapport/"), + match=_match_prefix("qualite/rapport/"), ), _CategoryRule( key="qualite_referentiels", @@ -107,7 +106,7 @@ def _prefix(prefix: str) -> CategoryMatcher: "Vocabulaire SIMS-FR (simsv2fr), documents annexes (documents) et referentiel " "territorial (territoires) associes aux rapports qualite." ), - match=_exact("qualite/documents", "qualite/simsv2fr", "qualite/territoires"), + match=_match_exact("qualite/documents", "qualite/simsv2fr", "qualite/territoires"), ), _CategoryRule( key="codes_concepts_generiques", @@ -118,7 +117,7 @@ def _prefix(prefix: str) -> CategoryMatcher: "notes explicatives xkos. Ce n'est PAS une nomenclature metier -- voir " "'nomenclatures' pour NAF/PCS/COICOP/etc." ), - match=_exact("codes", "codes/nomenclatures"), + match=_match_exact("codes", "codes/nomenclatures"), ), _CategoryRule( key="nomenclatures", @@ -130,7 +129,7 @@ def _prefix(prefix: str) -> CategoryMatcher: "juridiques (CJ), emplois (EAP/EMB par annee), tables de correspondance entre " "versions (ex: nafr2-cpfr21)." ), - match=_prefix("codes/"), + match=_match_prefix("codes/"), ), _CategoryRule( key="operations_statistiques", @@ -140,19 +139,19 @@ def _prefix(prefix: str) -> CategoryMatcher: "d'enquetes/collectes de l'Insee. C'est la cible (sdmx-mm:target) de chaque " "rapport qualite." ), - match=_exact("operations"), + match=_match_exact("operations"), ), _CategoryRule( key="demographie", label="Demographie", description="Populations legales par annee (popleg).", - match=_prefix("demo/"), + match=_match_prefix("demo/"), ), _CategoryRule( key="geographie", label="Geographie", description="Code officiel geographique (COG) : communes, decoupages administratifs.", - match=_prefix("geo/"), + match=_match_prefix("geo/"), ), _CategoryRule( key="organisations", @@ -161,25 +160,25 @@ def _prefix(prefix: str) -> CategoryMatcher: "Organismes producteurs de statistiques (services statistiques ministeriels...) " "et unites organisationnelles internes de l'Insee." ), - match=_prefix("organisations"), + match=_match_prefix("organisations"), ), _CategoryRule( key="concepts", label="Concepts et definitions statistiques", description="Themes statistiques et definitions de notions utilisees dans les publications.", - match=_prefix("concepts"), + match=_match_prefix("concepts"), ), _CategoryRule( key="produits", label="Produits / indicateurs statistiques", description="Indicateurs statistiques publies (StatisticalIndicator).", - match=_exact("produits"), + match=_match_exact("produits"), ), _CategoryRule( key="catalogue", label="Catalogue DCAT", description="Metadonnees de catalogage (dcat:Dataset, dcat:CatalogRecord).", - match=_exact("catalogue"), + match=_match_exact("catalogue"), ), _CategoryRule( key="ontologies", @@ -189,7 +188,7 @@ def _prefix(prefix: str) -> CategoryMatcher: "qui structurent les autres graphes. A consulter pour comprendre le schema " "d'un graphe de donnees, pas pour y chercher des donnees elles-memes." ), - match=_prefix("def/"), + match=_match_prefix("def/"), ), ] @@ -207,14 +206,14 @@ def _prefix(prefix: str) -> CategoryMatcher: _ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] -def _relative_path(graph_uri: str) -> str: +def _strip_graph_base(graph_uri: str) -> str: if graph_uri.startswith(GRAPH_BASE): return graph_uri[len(GRAPH_BASE):] return graph_uri def _categorize(graph_uri: str) -> _CategoryRule: - path = _relative_path(graph_uri) + path = _strip_graph_base(graph_uri) for cat in CATEGORY_DEFS: if cat.match(path): return cat @@ -252,12 +251,6 @@ def _accept_header(query_form: str) -> str: return "text/turtle" -def _error_payload(error_type: SparqlErrorType, message: str, query: str, **extra: Any) -> dict[str, Any]: - payload = {"type": error_type, "message": message, "query": query} - payload.update(extra) - return {"error": payload} - - # --------------------------------------------------------------------------- # Low-level SPARQL execution # --------------------------------------------------------------------------- @@ -273,11 +266,10 @@ async def _execute_sparql( query_form = _detect_query_form(query) if query_form == "UNKNOWN": - return _error_payload( - SparqlErrorType.INVALID_QUERY_FORM, + raise AppToolError( + "INVALID_QUERY", "Impossible de detecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requete. " "Verifie la syntaxe SPARQL (pas GraphQL).", - query, ) effective_query, limit_added = _ensure_limit(query, query_form, max_rows) @@ -294,45 +286,45 @@ async def _execute_sparql( response.raise_for_status() except httpx.TimeoutException: - return _error_payload( - SparqlErrorType.TIMEOUT, - f"Le endpoint n'a pas repondu en moins de {timeout}s. " + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Le endpoint RMES n'a pas repondu en moins de {timeout}s. " "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " "evite les scans sans filtre sur tous les graphes).", - query, + retryable=True, ) except httpx.HTTPStatusError as exc: status = exc.response.status_code body = exc.response.text[:2000] if status == 400: - return _error_payload( - SparqlErrorType.SYNTAX_ERROR, - "Le endpoint a rejete la requete (erreur de syntaxe SPARQL probable).", - query, - endpoint_message=body, + raise AppToolError( + "INVALID_QUERY", + f"Le endpoint RMES a rejete la requete (erreur de syntaxe SPARQL probable) : {body}", ) - return _error_payload( - SparqlErrorType.HTTP_ERROR, - f"Le endpoint a repondu {status}.", - query, - endpoint_message=body, + raise AppToolError( + "UPSTREAM_ERROR", + f"Le endpoint RMES a repondu {status} : {body}", + retryable=(500 <= status < 600), ) except httpx.RequestError as exc: - logger.warning("Erreur reseau vers %s: %s", endpoint, exc) - return _error_payload( - SparqlErrorType.NETWORK_ERROR, + raise AppToolError( + "BACKEND_UNAVAILABLE", f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", - query, + retryable=True, ) if accept == "text/turtle": return {"format": "turtle", "limit_added": limit_added, "data": response.text} - # Fixme: if the parsing of the response fails, it will lead to an unhandled exception - # as this line of code is not wrapped within the try except block - result = response.json() + try: + result = response.json() + except ValueError as exc: + raise AppToolError( + "PARSE_ERROR", + f"Le endpoint RMES a renvoye une reponse non-JSON : {exc}", + ) if limit_added: result.setdefault("_meta", {})["limit_added"] = max_rows result["_meta"]["hint"] = ( @@ -347,7 +339,7 @@ async def _get_raw_graph_rows( *, sparql_client: httpx.AsyncClient, endpoint: str, -) -> dict[str, Any]: +) -> list[dict[str, Any]]: now = time.time() if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: query = ( @@ -365,8 +357,6 @@ async def _get_raw_graph_rows( sparql_client=sparql_client, endpoint=endpoint, ) - if "error" in result: - return result rows = [ {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} for b in result["results"]["bindings"] @@ -374,7 +364,7 @@ async def _get_raw_graph_rows( _GRAPH_CACHE["data"] = rows _GRAPH_CACHE["ts"] = now - return {"rows": _GRAPH_CACHE["data"]} + return _GRAPH_CACHE["data"] # --------------------------------------------------------------------------- @@ -411,16 +401,10 @@ async def list_graphs( sparql_client: httpx.AsyncClient, endpoint: str, ) -> ListGraphsOutput: - raw = await _get_raw_graph_rows( - sparql_client=sparql_client, endpoint=endpoint, + rows = await _get_raw_graph_rows( + sparql_client=sparql_client, + endpoint=endpoint, ) - if "error" in raw: - return ListGraphsOutput( - total_graphs_matched=0, - categories=[], - error=SparqlError(**raw["error"]), - ) - rows = raw["rows"] expand = params.expand if params.contains: @@ -492,11 +476,6 @@ async def describe_resource( sparql_client=sparql_client, endpoint=endpoint, ) - if "error" in result: - return DescribeResourceOutput( - uri=params.uri, properties=[], count=0, error=SparqlError(**result["error"]) - ) - properties = _parse_bindings_to_properties(result["results"]["bindings"]) return DescribeResourceOutput(uri=params.uri, properties=properties, count=len(properties)) @@ -508,12 +487,9 @@ async def run_sparql( endpoint: str, ) -> RunSparqlOutput: if not params.full_sparql_query or not params.full_sparql_query.strip(): - return RunSparqlOutput( - error=SparqlError( - type=SparqlErrorType.EMPTY_QUERY, - message="La requete est vide.", - query=params.full_sparql_query, - ) + raise AppToolError( + "INVALID_INPUT", + "La requete SPARQL est vide. Fournis une requete SELECT, ASK, CONSTRUCT ou DESCRIBE.", ) max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) @@ -522,9 +498,6 @@ async def run_sparql( sparql_client=sparql_client, endpoint=endpoint, ) - if "error" in result: - return RunSparqlOutput(error=SparqlError(**result["error"])) - if result.get("format") == "turtle": return RunSparqlOutput( format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"] diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index d130bf1..bc564ee 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -6,7 +6,7 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_CHIFFRECLEF -from ..core.errors import fail +from ..core.errors import AppToolError from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( SearchInseeChiffrecleInput, @@ -56,12 +56,10 @@ async def search_insee_chiffrecle( index=index, ) except (ESConnectionError, TransportError) as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"INSEE documents search backend unreachable: {exc}. " "Verify ES_HOST and try again.", retryable=True, ) - # Fixme: we saw that the fail function did already raise - raise return SearchInseeChiffrecleOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index a26328a..f9654b6 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -7,7 +7,7 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_CONJONCTURE -from ..core.errors import fail +from ..core.errors import AppToolError from ..data.themes import DICT_THEME_CONJ from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( @@ -59,12 +59,10 @@ async def search_insee_conjoncture( index=index, ) except (ESConnectionError, TransportError) as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"INSEE conjoncture search backend unreachable: {exc}. " "Verify ES_HOST and try again.", retryable=True, ) - # Fixme: as stated, this raise is dead - raise return SearchInseeConjonctureOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index 1277d92..0d897d5 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -6,7 +6,7 @@ from fastmcp import Context, FastMCP from ..config.tool_metadata import SEARCH_DOCUMENTS -from ..core.errors import fail +from ..core.errors import AppToolError from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( SearchInseeDocumentsInput, @@ -54,12 +54,10 @@ async def search_insee_documents( index=index, ) except (ESConnectionError, TransportError) as exc: - fail( + raise AppToolError( "BACKEND_UNAVAILABLE", f"INSEE documents search backend unreachable: {exc}. " "Verify ES_HOST and try again.", retryable=True, ) - # Fixme: dead raise - raise return SearchInseeDocumentsOutput(results=hits, count=len(hits)) From 60308322d849c007851308a355bc0000440e8a08 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Fri, 4 Sep 2026 12:13:52 +0200 Subject: [PATCH 14/55] build: adopt ruff for linting and formatting 120 findings across the package, most of them rules we had written down but had no way to enforce. - UP045 x26: `Optional[X]` where python.md mandates `X | None` - I001 x11: unsorted imports, including one I introduced by hand - E501 x3, plus 59 in data/indicators.py which is literal statistics awaiting a live source and is ignored per-file until then - F401 x1: register_extras_send_feedback is imported and never registered, so send_feedback is not exposed at all. Marked with an explicit noqa and a note, rather than deleted, until we decide to wire it up or drop it. B904 flagged 20 raises inside except clauses without `from exc`. Adding it turned out to buy nothing: Python preserves the original as __context__ either way, so the cause is in the traceback regardless and `from exc` only changes the wording. What was actually missing is that ErrorHandlingMiddleware was discarding tracebacks, so no cause reached the log at all. include_traceback is now on, `from exc` is dropped, and B904 is ignored with the reason recorded. python.md now explains that the formatter collapses anything fitting on one line and never splits it, so a trailing comma is what keeps a literal exploded. Without that the multiline rule reads as broken. Quotes need no rule: the formatter normalises them. error.md's chaining rule now states its real weight - server-side hygiene, nothing crossing the wire - instead of implying it preserves lost information. --- .claude/rules/error.md | 4 +- .claude/rules/python.md | 6 +- .pre-commit-config.yaml | 13 + pyproject.toml | 24 ++ src/mcpdiffusion/config/tool_metadata.py | 30 +- src/mcpdiffusion/data/indicators.py | 354 +++++++++++++++--- src/mcpdiffusion/data/themes.py | 3 +- src/mcpdiffusion/infra/__init__.py | 2 +- src/mcpdiffusion/infra/lifespan.py | 6 +- src/mcpdiffusion/models/feedback.py | 1 + src/mcpdiffusion/models/insee.py | 73 ++-- src/mcpdiffusion/models/melodi.py | 10 +- src/mcpdiffusion/models/rmes.py | 26 +- src/mcpdiffusion/server.py | 11 +- src/mcpdiffusion/services/feedback.py | 5 +- src/mcpdiffusion/services/insee_document.py | 25 +- src/mcpdiffusion/services/insee_search.py | 19 +- src/mcpdiffusion/services/melodi.py | 42 +-- src/mcpdiffusion/services/rmes.py | 34 +- src/mcpdiffusion/tools/__init__.py | 18 +- .../tools/extras_send_feedback.py | 1 + src/mcpdiffusion/tools/insee_get_document.py | 1 + src/mcpdiffusion/tools/insee_get_homepage.py | 2 + .../tools/insee_search_chiffrecle.py | 5 +- .../tools/insee_search_conjoncture.py | 5 +- .../tools/insee_search_documents.py | 5 +- .../tools/melodi_get_observations.py | 1 + .../tools/melodi_search_datasets.py | 1 + .../tools/melodi_search_modalities.py | 1 + .../tools/rmes_describe_resource.py | 1 + src/mcpdiffusion/tools/rmes_list_graphs.py | 1 + src/mcpdiffusion/tools/rmes_run_sparql.py | 11 +- tests/conftest.py | 5 + tests/test_feedback_service.py | 1 + tests/test_insee_document_service.py | 52 ++- tests/test_insee_search_service.py | 70 +++- tests/test_melodi_service.py | 242 ++++++++---- tests/test_middleware.py | 17 +- tests/test_rmes_service.py | 89 +++-- tests/test_rmes_tools.py | 93 ++--- uv.lock | 117 ++++++ 41 files changed, 993 insertions(+), 434 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.claude/rules/error.md b/.claude/rules/error.md index 341baf8..d8ad9b9 100644 --- a/.claude/rules/error.md +++ b/.claude/rules/error.md @@ -30,7 +30,9 @@ step — the offending parameter, or the tool that produces a valid value. Never (it swallows `CancelledError`). - Translate once, at the boundary owning the dependency. Never re-wrap an already typed error. - Never swallow: no empty `except`, no default on failure, no log-and-continue. -- Chain with `raise ... from exc`. +- Chain with `raise ... from exc`. Server-side hygiene only: `__cause__` never crosses the wire, so it + leaks nothing, and Python keeps the original either way — this just states that it was the cause rather + than an error raised while handling one. ## Use what FastMCP provides diff --git a/.claude/rules/python.md b/.claude/rules/python.md index d332c82..aca17da 100644 --- a/.claude/rules/python.md +++ b/.claude/rules/python.md @@ -35,8 +35,10 @@ Spell names out. The reader should not have to look up what something holds. ## Layout -- Dicts, lists and other objects: multiline, one entry per line, including as call arguments. -- Signatures and calls with two or more arguments: multiline, one per line. +- Dicts, lists and other objects: multiline, one entry per line, including as call arguments. The + formatter collapses anything that fits on one line and never splits it for you — a **trailing comma + on the last entry** is what keeps it exploded, so write one. +- Signatures and calls with two or more arguments: multiline, one per line, with the same trailing comma. - Calls with more than two arguments name each one. Positional only where keywords are forbidden (`getattr`, `dict`, `join`). - Imports at the top of the module, never inside a function. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d252341 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: local + hooks: + - id: ruff-lint + name: ruff lint + entry: uv run ruff check --fix + language: system + types: [ python ] + - id: ruff-format + name: ruff format + entry: uv run ruff format + language: system + types: [ python ] diff --git a/pyproject.toml b/pyproject.toml index 5b1dd14..b71f74a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,8 +28,32 @@ packages = ["src/mcpdiffusion"] dev = [ "pytest>=9.1.1", "pytest-asyncio>=1.4.0", + "pre-commit>=4.5.1", + "ruff>=0.15.4", ] +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +# B904 is off: Python keeps the original exception either way, so `raise ... from exc` only changes the +# wording in a traceback. include_traceback on ErrorHandlingMiddleware is what makes causes recoverable. +ignore = ["B904"] +select = [ + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear - catches common bug patterns + "UP", # pyupgrade - enforces modern syntax, including `str | None` over `Optional[str]` +] + +[tool.ruff.lint.per-file-ignores] +# Long literal statistics, pending replacement by a live source. +"src/mcpdiffusion/data/indicators.py" = ["E501"] + +[tool.ruff.format] +docstring-code-format = true + [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py index a8124d1..b587082 100644 --- a/src/mcpdiffusion/config/tool_metadata.py +++ b/src/mcpdiffusion/config/tool_metadata.py @@ -10,6 +10,7 @@ - Tool descriptions describe the *final* schemas; rewrite in lockstep when schemas change. """ + from datetime import date @@ -77,8 +78,8 @@ def compute_current_date_iso() -> str: "\n" "TIPS\n" "- Matching is lexical. Make `french_query` explicit and rich in French " - "synonyms: e.g. `\"indice des prix a la consommation\"`, " - "`\"deces par departement\"`, `\"prenoms des nouveau-nes\"`.\n" + 'synonyms: e.g. `"indice des prix a la consommation"`, ' + '`"deces par departement"`, `"prenoms des nouveau-nes"`.\n' "- Use `start_year` / `end_year` to narrow the temporal range. Leaving " "both at default covers all years.\n" "\n" @@ -107,7 +108,7 @@ def compute_current_date_iso() -> str: "\n" "INPUT\n" "- `dataset_id` -- from a previous search result.\n" - "- `columns_id` -- which columns to search (e.g. `[\"PRICES\", \"GEO\"]`).\n" + '- `columns_id` -- which columns to search (e.g. `["PRICES", "GEO"]`).\n' "- `french_query` -- natural-language query in French.\n" "\n" "OUTPUT\n" @@ -143,7 +144,7 @@ def compute_current_date_iso() -> str: "\n" "INPUT\n" "- `list_of_url` -- list of relative URLs to fetch (e.g. " - "`[\"/fr/statistiques/4277658?sommaire=4318291\"]`).\n" + '`["/fr/statistiques/4277658?sommaire=4318291"]`).\n' "- `include_sommaire` -- also parse the page's table-of-contents " "section. Use once to discover the structure of a multi-section " "publication, then turn it off for subsequent requests on the same page.\n" @@ -218,14 +219,15 @@ def compute_current_date_iso() -> str: SEARCH_CHIFFRECLEF = { "tool_name": "search_insee_chiffrecle", "tool_description": "Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, \n" - "comparaisons regionales/departementales et statistiques factuelles simples.\n" - "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" - "prix par categorie, comparaisons geographiques (region, departement, commune).\n" - "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" - "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" - "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" - "ou search_insee_documents selon le contexte).\n" - "Retourne directement les tableaux synthetiques prets a l'emploi.\n", + "comparaisons regionales/departementales et statistiques factuelles simples.\n" + "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" + "prix par categorie, comparaisons geographiques (region, departement, commune).\n" + "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', " + "'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" + "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" + "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" + "ou search_insee_documents selon le contexte).\n" + "Retourne directement les tableaux synthetiques prets a l'emploi.\n", "tool_metadata": {"version": "5.0", "author": "mirlon"}, } @@ -311,7 +313,7 @@ def compute_current_date_iso() -> str: "avec un compteur et quelques URIs d'exemple par categorie -- pas la liste plate " "des 700+ graphes. Choisis une categorie precise dans le parametre `category` " "pour cibler une famille, ou utilise `contains` pour une recherche libre par " - "sous-chaine. Une categorie \"autre\" recueille tout graphe ne correspondant a " + 'sous-chaine. Une categorie "autre" recueille tout graphe ne correspondant a ' "aucune famille connue." ), "tool_metadata": {"version": "5.0", "author": "mirlon"}, @@ -342,7 +344,7 @@ def compute_current_date_iso() -> str: "Bonnes pratiques :\n" "- Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou " " VALUES ?g { } plutot que de scanner tous les graphes.\n" - "- Toujours ajouter FILTER(lang(?label) = \"fr\") sur les litteraux SKOS pour eviter " + '- Toujours ajouter FILTER(lang(?label) = "fr") sur les litteraux SKOS pour eviter ' " les doublons multilingues.\n" "- Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee " " automatiquement (indique dans la reponse via `limit_added`/`hint`).\n" diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/indicators.py index 4411d50..1b919b0 100644 --- a/src/mcpdiffusion/data/indicators.py +++ b/src/mcpdiffusion/data/indicators.py @@ -5,63 +5,299 @@ # Fixme: This seems like hardcoded, stale statistics... I don't know if this is normal DICT_KV = [ {"cle": "clé", "alias": "alias", "valeur": "valeur"}, - {"cle": "estimation de population France", "alias": "", "valeur": "Au 1er janvier 2026, la population résidant en France est estimée à 69,1 millions d'habitants."}, - {"cle": "population légale France", "alias": "", "valeur": "Au 1er janvier 2023, la population de la France hors Mayotte s'établit officiellement à 68 094 000 habitants."}, - {"cle": "immigrés France", "alias": "", "valeur": "En 2025, 8,0 millions d'immigrés vivent en France, soit 11,6 % de la population totale."}, - {"cle": "population étrangère France", "alias": "", "valeur": "En 2025, la population étrangère vivant en France s'élève à 6,3 millions de personnes, soit 9,1 % de la population totale."}, - {"cle": "naissances France", "alias": "", "valeur": "En 2025, le nombre de naissances en France est estimé à 645 000, soit une baisse de -2,1 % par rapport à 2024."}, - {"cle": "indicateur conjoncturel de fécondité", "alias": "", "valeur": "En 2025, l'indicateur conjoncturel de fécondité (ICF) continue de diminuer. Il s'établit à 1,56 enfant par femme (1,53 en France métropolitaine), après 1,61 en 2024 (1,58 en France métropolitaine)."}, - {"cle": "décès France", "alias": "", "valeur": "En 2025, le nombre de décès en France est estimé à 651 000, en hausse de 1,5 % par rapport à 2024, après +0,3 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile)."}, - {"cle": "espérance de vie France", "alias": "", "valeur": "En 2025, l'espérance de vie à la naissance s'élève à 85,9 ans pour les femmes et à 80,3 ans pour les hommes. Elle augmente en 2025, de +0,1 an pour les femmes comme pour les hommes, pour atteindre un niveau historiquement élevé."}, - {"cle": "mariages France", "alias": "", "valeur": "En 2025, le nombre de mariages célébrés en France est estimé à 251 000, dont 244 000 entre personnes de sexe différent et 7 000 entre personnes de même sexe. Le nombre de mariages augmente de 1,4 % par rapport à 2024, après +2,7 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile), alors que la tendance était plutôt à la baisse avant la crise sanitaire."}, - {"cle": "ménages France", "alias": "", "valeur": "En 2023, la France hors Mayotte compte 31,3 millions de ménages."}, - {"cle": "divorces France", "alias": "", "valeur": "128 043 divorces en 2016. Note : jusqu'en 2016, les divorces étaient des décisions de justice prononcées par un juge ; depuis 2017, les divorces par consentement mutuel passent par un acte notarié et ne sont plus comptabilisés de la même façon."}, - {"cle": "inflation", "alias": "Indice des prix à la consommation – IPC ", "valeur": "En juin 2026, les prix à la consommation (IPC) augmentent de 1,8 % sur un an. Sur un mois, l'indice des prix à la consommation diminue de 0,3 %."}, - {"cle": "Chômage BIT ", "alias": "", "valeur": "Au premier trimestre 2026, le taux de chômage en France (hors Mayotte) augmente de 0,2 point et atteint 8,1 % . Le nombre de chômeurs est de 2,6 millions de personnes."}, - {"cle": "emploi BIT", "alias": "", "valeur": "En moyenne sur l'année 2025, parmi les personnes âgées de 15 à 64 ans vivant en France, 69,3 % sont en emploi au sens du Bureau international du travail (BIT)."}, - {"cle": "PIB trimestriel", "alias": "croissance trimestrielle", "valeur": "Au premier trimestre 2026, le produit intérieur brut (PIB) en volume se replie légèrement (-0,1 %)."}, - {"cle": "PIB annuel", "alias": "croissance annuelle", "valeur": "En 2025, le PIB croît de 0,8 % en volume aux prix de l'année précédente."}, - {"cle": "Dépenses de consommation des ménages en biens", "alias": "", "valeur": "En mai 2026, les dépenses de consommation des ménages en biens rebondissent sur un mois (+0,5 % en volume après -0,5 % en avril). Les volumes sont mesurés aux prix de l'année précédente chaînés (en milliards d'euros 2020) et corrigés des variations saisonnières et des effets des jours ouvrables (CVS-CJO)."}, - {"cle": "Climat des affaires", "alias": "", "valeur": "En juin 2026, l'indicateur synthétique du climat des affaires, calculé à partir des réponses des chefs d'entreprise des principaux secteurs d'activité marchands rebondit très légèrement, à 94, en deçà de son niveau moyen."}, - {"cle": "climat de l'emploi", "alias": "", "valeur": "En juin 2026, l'indicateur du climat de l'emploi perd de nouveau trois points (après arrondi) et s'établit à 89, son niveau le plus bas depuis juin 2013 (hors crise sanitaire)."}, - {"cle": "production manufacturière", "alias": "Indice de la production industrielle - IPI", "valeur": "En mai 2026, après deux mois de hausse, la production se replie nettement dans l'industrie manufacturière (-1,0 % après +0,6 % en avril 2026). Dans l'ensemble de l'industrie, elle se replie aussi mais plus légèrement (-0,1 % après +0,3 %)."}, - {"cle": "niveau de vie", "alias": "", "valeur": "En 2024, en France métropolitaine, le niveau de vie médian de la population s'élève à 26 740 euros annuels. Il correspond à un revenu disponible de 2 228 euros par mois pour une personne seule."}, - {"cle": "pouvoir d'achat", "alias": "", "valeur": "En 2025, le pouvoir d'achat du revenu disponible (RDB) des ménages se replie de 0,4 % après une hausse de 2,7 % en 2024. Ramené au niveau individuel et en tenant compte de l'évolution de la taille des ménages, le pouvoir d'achat baisse de 0,7 % après une hausse de 2,2 % en 2024"}, - {"cle": "balance commerciale", "alias": "", "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l'activité en 2025, à hauteur de -0,2 point de PIB, après l'avoir fortement soutenue en 2023 et 2024. "}, - {"cle": "pauvreté monétaire", "alias": "", "valeur": "En 2024, 9,8 millions de personnes vivent avec un niveau de vie inférieur au seuil de pauvreté monétaire, soit 15,4 % de la population vivant dans un logement ordinaire en France métropolitaine."}, - {"cle": "patrimoine", "alias": "", "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine. "}, - {"cle": "état santé", "alias": "", "valeur": "En 2024, deux tiers des personnes âgées de 16 ans ou plus se déclarent en bonne ou très bonne santé. À l'opposé, près de 10 % jugent leur état de santé mauvais voire très mauvais."}, - {"cle": "prestation handicap", "alias": "", "valeur": "Selon leur âge et leur situation, les personnes en situation de handicap ou de perte d'autonomie peuvent prétendre à différentes prestations. Fin 2023, 44 000 personnes ont un droit ouvert à l'allocation compensatrice pour tierce personne (ACTP) et 407 000 à la prestation de compensation du handicap (PCH). Par ailleurs, 1,4 million de personnes de 60 ans ou plus ont perçu l'allocation personnalisée d'autonomie (APA) au titre du mois de décembre 2023."}, - {"cle": "dépenses liées à la culture", "alias": "", "valeur": "En 2025, les dépenses liées à la culture, au sport et aux loisirs s'élèvent à 108 milliards d'euros. Les services récréatifs, sportifs et culturels rassemblent 45 % de ces dépenses."}, - {"cle": "Parc de logements", "alias": "", "valeur": "Au 1er janvier 2025, la France hors Mayotte compte 38,4 millions de logements. 82,5 % des logements sont des résidences principales et 54,4 % des logements individuels (maisons)."}, - {"cle": "logements vacants", "alias": "", "valeur": "Après avoir fortement augmenté entre 2005 et 2019, la part des logements vacants diminue, passant de 8,1 % en 2019 à 7,7 % en 2025 ; en 2025, 3,0 millions de logements sont vacants."}, - {"cle": "résidences secondaires ou logements occasionnels", "alias": "", "valeur": "Au 1er janvier 2025, 3,8 millions de logements sont des résidences secondaires ou des logements occasionnels ; après avoir augmenté entre 2011 et 2017, leur part dans l'ensemble du parc est stable."}, - {"cle": "ménages sont propriétaires de leur résidence principale", "alias": "", "valeur": "Au 1er janvier 2025, 57,4 % des ménages sont propriétaires de leur résidence principale."}, - {"cle": "smic", "alias": "Salaire minimum interprofessionnel de croissance", "valeur": "Depuis le 1er janvier 2026, le Smic brut s'élève à 12,02 euros par heure, soit 1 823,03 euros par mois pour 151,67 heures de travail."}, - {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur privé", "alias": "", "valeur": "En 2023, le salaire mensuel moyen en équivalent temps plein (EQTP) dans le secteur privé est de 2 730 euros, nets de cotisations et contributions sociales."}, - {"cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur public", "alias": "", "valeur": "Dans la fonction publique, tous statuts confondus, un salarié gagne en moyenne 2 650 euros nets par mois en EQTP en 2023."}, - {"cle": "revenus non salariés", "alias": "", "valeur": "En 2023, hors agriculture, les non-salariés classiques (micro-entrepreneurs exclus) retirent en moyenne 4 040 euros par mois de leur activité non salariée. Cette moyenne recouvre de fortes disparités selon la nature des emplois."}, - {"cle": "salaires horaires", "alias": "", "valeur": "Au premier trimestre 2026, les salaires horaires augmentent de 0,3 % sur le trimestre et de 2,0 % sur un an"}, - {"cle": "coût horaire du travail", "alias": "Indice du coût du travail – ICT", "valeur": "Au premier trimestre 2026, le coût horaire du travail (salaires, cotisations et taxes, déduction faite des exonérations et subventions) de l'ensemble du secteur marchand non agricole (hors services aux ménages) freine significativement, dans le sillage des salaires : +0,5 % sur le trimestre et + 2,3 % sur un an."}, - {"cle": "création entreprises", "alias": "", "valeur": "En 2025, 1 165 800 entreprises ont été créées en France, dont 758 500 sous forme d'entrepreneurs individuels ayant adopté le régime de la microentreprise (micro-entrepreneurs)."}, - {"cle": "défaillances d'entreprises", "alias": "", "valeur": "En 2025, 68 872 unités légales ont été en situation de défaillance."}, - {"cle": "entreprises marchandes non agricoles et non financières en France", "alias": "", "valeur": "En 2023, en France, les secteurs marchands non agricoles et non financiers (incluant toutefois les exploitations forestières, les auxiliaires de services financiers et d'assurance et les holdings) comptent 5,2 millions d'entreprises. Ces entreprises emploient 15,9 millions de salariés en équivalent temps plein (EQTP)."}, - {"cle": "exploitations agricoles", "alias": "", "valeur": "Dans le secteur agricole, l'usage est de compter plutôt des exploitations agricoles ; en 2023, la France métropolitaine en compte 349 600 et la main d'œuvre agricole s'élève à 663 200 EQTP."}, - {"cle": "commerce", "alias": "", "valeur": "En 2023, le commerce rassemble 739 128 entreprises. Elles réalisent un chiffre d'affaires de 1 728 milliards d'euros et dégagent une valeur ajoutée (VA) de 272 milliards d'euros. Fin 2024, 3,4 millions de personnes occupent un emploi salarié dans le commerce."}, - {"cle": "industrie", "alias": "", "valeur": "En 2023, l'industrie rassemble 322 386 entreprises. Elles réalisent un chiffre d'affaire de 1 544 milliards d'euros et dégagent une valeur ajoutée (VA) de 368 milliards d'euros. Fin 2024, 3,3 millions de personnes occupent un emploi salarié dans l'industrie."}, - {"cle": "construction", "alias": "", "valeur": "En 2023, la construction rassemble 587 898 entreprises. Elles réalisent un chiffre d'affaires de 405 milliards d'euros et dégagent une valeur ajoutée (VA) de 128 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans la construction."}, - {"cle": "services", "alias": "", "valeur": "En 2023, les services principalement marchands non financiers comptent plus de 2,3 millions d'entreprises. Ces entreprises réalisent un chiffre d'affaires de 995 milliards d'euros et dégagent une valeur ajoutée (VA) de 475 milliards d'euros. Fin 2024, 7,5 millions de personnes (y compris les intérimaires) occupent un emploi salarié dans les services principalement marchands non financiers."}, - {"cle": "transports", "alias": "", "valeur": "En 2023, les transports et l'entreposage rassemblent 193 101 entreprises. Elles réalisent un chiffre d'affaires de 267 milliards d'euros et dégagent une valeur ajoutée (VA) de 102 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans les transports et l'entreposage."}, - {"cle": "entreprises de l'économie sociale", "alias": "", "valeur": "Les entreprises de l'économie sociale se caractérisent par leur famille de l'économie sociale, à la fois privé et à caractère essentiellement non lucratif. En 2022, elles représentent 9,8 % de l'emploi salarié total en équivalent temps plein. Les associations emploient 73 % de ce volume de travail salarié ; 14 % est employé par les coopératives, 6 % par les mutuelles, 5 % par les fondations et 3 % par les autres organismes privés à but non-lucratif."}, - {"cle": "Population quartiers prioritaires de la politique de la ville", "alias": "QPV", "valeur": "Les quartiers prioritaires de la politique de la ville (QPV) tels que définis par le décret n° 2015-1138 du 14 septembre 2015 regroupent 7,9 % de la population en 2020."}, - {"cle": "Population unités urbaines", "alias": "", "valeur": "Les unités urbaines rassemblent toujours plus d'habitants. En 2022, en France métropolitaine, elles représentent 78,8 % de la population, soit 51,9 millions d'habitants. À l'exception de l'unité urbaine de Paris qui concentre près de 11 millions d'habitants, les 10 plus grandes unités urbaines françaises comptent chacune entre 0,5 et 2 millions d'habitants."}, - {"cle": "mode déplacement domicile travail", "alias": "", "valeur": "Pour se rendre au travail, les personnes en emploi se déplacent majoritairement en voiture ou en deux-roues motorisés (71 % en 2022). 15 % des personnes en emploi empruntent les transports en commun."}, - {"cle": "dépense nationale protection de l'environnement", "alias": "", "valeur": "En 2022, la dépense nationale en faveur de la protection de l'environnement s'élève à 63,7 milliards d'euros (Md€). Elle est dédiée à la protection de l'air, de la biodiversité et des paysages, la collecte et traitement des déchets, la protection et dépollution des sols et des eaux, la lutte contre le bruit et d'autres activités de protection de l'environnement (frais de fonctionnement de l'administration publique et des opérateurs chargés des questions environnementales notamment). Les entreprises sont les principaux financeurs des dépenses de protection de l'environnement (22,6 Md€, soit 35 %), devant les administrations publiques (État et ses ministères, collectivités locales, organismes publics) (22,2 Md€, soit 35 %) et les ménages (18,1 Md€, soit 28 %)."}, - {"cle": "indice de référence des loyers", "alias": "IRL", "valeur": "Au deuxième trimestre 2026, l'indice de référence des loyers s'établit à 148,37. Sur un an, il augmente de 1,15 % après +0,78 % au trimestre précédent."}, - {"cle": "indice des loyers commerciaux", "alias": "ILC", "valeur": "Au premier trimestre 2026, l'indice des loyers commerciaux s'établit à 135,26. Sur un an, il baisse de 0,45 % (après -0,50 % au trimestre précédent)."}, - {"cle": "indice des loyers des activités tertiaires", "alias": "ILAT", "valeur": "Au premier trimestre 2026, l'indice des loyers des activités tertiaires s'établit à 137,42. Sur un an, il augmente de 0,09 % (après -0,06 % au trimestre précédent)."}, - {"cle": "indice du coût de la construction", "alias": "ICC", "valeur": "L'indice du coût de la construction (ICC) s'établit à 2 084 au premier trimestre 2026. Il est en hausse de 1,26 % sur un trimestre (après +0,10 % au trimestre précédent). Sur un an, il baisse de 2,89 % (après -2,37 % au trimestre précédent)."}, - {"cle": "index du bâtiment tous corps d'état", "alias": "BT01 ; index bâtiment BT01", "valeur": "En mai 2026, l'index Bâtiment BT01 « Tous corps d'état » s'établit à 137,9, en référence 100 en 2010."}, - {"cle": "index général des travaux publics", "alias": "TP01 ; index travaux publics TP01", "valeur": "En mai 2026, l'index Travaux publics TP01 « Index général tous travaux » s'établit à 140,4, en référence 100 en 2010."}, - {"cle": "index ingénierie", "alias": "ING ; indice ING", "valeur": "En mai 2026, l'index divers de la construction ING « Ingénierie » s'établit à 138,3, en référence 100 en 2010."}, + { + "cle": "estimation de population France", + "alias": "", + "valeur": "Au 1er janvier 2026, la population résidant en France est estimée à 69,1 millions d'habitants.", + }, + { + "cle": "population légale France", + "alias": "", + "valeur": "Au 1er janvier 2023, la population de la France hors Mayotte s'établit officiellement à 68 094 000 habitants.", + }, + { + "cle": "immigrés France", + "alias": "", + "valeur": "En 2025, 8,0 millions d'immigrés vivent en France, soit 11,6 % de la population totale.", + }, + { + "cle": "population étrangère France", + "alias": "", + "valeur": "En 2025, la population étrangère vivant en France s'élève à 6,3 millions de personnes, soit 9,1 % de la population totale.", + }, + { + "cle": "naissances France", + "alias": "", + "valeur": "En 2025, le nombre de naissances en France est estimé à 645 000, soit une baisse de -2,1 % par rapport à 2024.", + }, + { + "cle": "indicateur conjoncturel de fécondité", + "alias": "", + "valeur": "En 2025, l'indicateur conjoncturel de fécondité (ICF) continue de diminuer. Il s'établit à 1,56 enfant par femme (1,53 en France métropolitaine), après 1,61 en 2024 (1,58 en France métropolitaine).", + }, + { + "cle": "décès France", + "alias": "", + "valeur": "En 2025, le nombre de décès en France est estimé à 651 000, en hausse de 1,5 % par rapport à 2024, après +0,3 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile).", + }, + { + "cle": "espérance de vie France", + "alias": "", + "valeur": "En 2025, l'espérance de vie à la naissance s'élève à 85,9 ans pour les femmes et à 80,3 ans pour les hommes. Elle augmente en 2025, de +0,1 an pour les femmes comme pour les hommes, pour atteindre un niveau historiquement élevé.", + }, + { + "cle": "mariages France", + "alias": "", + "valeur": "En 2025, le nombre de mariages célébrés en France est estimé à 251 000, dont 244 000 entre personnes de sexe différent et 7 000 entre personnes de même sexe. Le nombre de mariages augmente de 1,4 % par rapport à 2024, après +2,7 % entre 2023 et 2024 (en tenant compte du fait que 2024 est une année bissextile), alors que la tendance était plutôt à la baisse avant la crise sanitaire.", + }, + { + "cle": "ménages France", + "alias": "", + "valeur": "En 2023, la France hors Mayotte compte 31,3 millions de ménages.", + }, + { + "cle": "divorces France", + "alias": "", + "valeur": "128 043 divorces en 2016. Note : jusqu'en 2016, les divorces étaient des décisions de justice prononcées par un juge ; depuis 2017, les divorces par consentement mutuel passent par un acte notarié et ne sont plus comptabilisés de la même façon.", + }, + { + "cle": "inflation", + "alias": "Indice des prix à la consommation – IPC ", + "valeur": "En juin 2026, les prix à la consommation (IPC) augmentent de 1,8 % sur un an. Sur un mois, l'indice des prix à la consommation diminue de 0,3 %.", + }, + { + "cle": "Chômage BIT ", + "alias": "", + "valeur": "Au premier trimestre 2026, le taux de chômage en France (hors Mayotte) augmente de 0,2 point et atteint 8,1 % . Le nombre de chômeurs est de 2,6 millions de personnes.", + }, + { + "cle": "emploi BIT", + "alias": "", + "valeur": "En moyenne sur l'année 2025, parmi les personnes âgées de 15 à 64 ans vivant en France, 69,3 % sont en emploi au sens du Bureau international du travail (BIT).", + }, + { + "cle": "PIB trimestriel", + "alias": "croissance trimestrielle", + "valeur": "Au premier trimestre 2026, le produit intérieur brut (PIB) en volume se replie légèrement (-0,1 %).", + }, + { + "cle": "PIB annuel", + "alias": "croissance annuelle", + "valeur": "En 2025, le PIB croît de 0,8 % en volume aux prix de l'année précédente.", + }, + { + "cle": "Dépenses de consommation des ménages en biens", + "alias": "", + "valeur": "En mai 2026, les dépenses de consommation des ménages en biens rebondissent sur un mois (+0,5 % en volume après -0,5 % en avril). Les volumes sont mesurés aux prix de l'année précédente chaînés (en milliards d'euros 2020) et corrigés des variations saisonnières et des effets des jours ouvrables (CVS-CJO).", + }, + { + "cle": "Climat des affaires", + "alias": "", + "valeur": "En juin 2026, l'indicateur synthétique du climat des affaires, calculé à partir des réponses des chefs d'entreprise des principaux secteurs d'activité marchands rebondit très légèrement, à 94, en deçà de son niveau moyen.", + }, + { + "cle": "climat de l'emploi", + "alias": "", + "valeur": "En juin 2026, l'indicateur du climat de l'emploi perd de nouveau trois points (après arrondi) et s'établit à 89, son niveau le plus bas depuis juin 2013 (hors crise sanitaire).", + }, + { + "cle": "production manufacturière", + "alias": "Indice de la production industrielle - IPI", + "valeur": "En mai 2026, après deux mois de hausse, la production se replie nettement dans l'industrie manufacturière (-1,0 % après +0,6 % en avril 2026). Dans l'ensemble de l'industrie, elle se replie aussi mais plus légèrement (-0,1 % après +0,3 %).", + }, + { + "cle": "niveau de vie", + "alias": "", + "valeur": "En 2024, en France métropolitaine, le niveau de vie médian de la population s'élève à 26 740 euros annuels. Il correspond à un revenu disponible de 2 228 euros par mois pour une personne seule.", + }, + { + "cle": "pouvoir d'achat", + "alias": "", + "valeur": "En 2025, le pouvoir d'achat du revenu disponible (RDB) des ménages se replie de 0,4 % après une hausse de 2,7 % en 2024. Ramené au niveau individuel et en tenant compte de l'évolution de la taille des ménages, le pouvoir d'achat baisse de 0,7 % après une hausse de 2,2 % en 2024", + }, + { + "cle": "balance commerciale", + "alias": "", + "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l'activité en 2025, à hauteur de -0,2 point de PIB, après l'avoir fortement soutenue en 2023 et 2024. ", + }, + { + "cle": "pauvreté monétaire", + "alias": "", + "valeur": "En 2024, 9,8 millions de personnes vivent avec un niveau de vie inférieur au seuil de pauvreté monétaire, soit 15,4 % de la population vivant dans un logement ordinaire en France métropolitaine.", + }, + { + "cle": "patrimoine", + "alias": "", + "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine. ", + }, + { + "cle": "état santé", + "alias": "", + "valeur": "En 2024, deux tiers des personnes âgées de 16 ans ou plus se déclarent en bonne ou très bonne santé. À l'opposé, près de 10 % jugent leur état de santé mauvais voire très mauvais.", + }, + { + "cle": "prestation handicap", + "alias": "", + "valeur": "Selon leur âge et leur situation, les personnes en situation de handicap ou de perte d'autonomie peuvent prétendre à différentes prestations. Fin 2023, 44 000 personnes ont un droit ouvert à l'allocation compensatrice pour tierce personne (ACTP) et 407 000 à la prestation de compensation du handicap (PCH). Par ailleurs, 1,4 million de personnes de 60 ans ou plus ont perçu l'allocation personnalisée d'autonomie (APA) au titre du mois de décembre 2023.", + }, + { + "cle": "dépenses liées à la culture", + "alias": "", + "valeur": "En 2025, les dépenses liées à la culture, au sport et aux loisirs s'élèvent à 108 milliards d'euros. Les services récréatifs, sportifs et culturels rassemblent 45 % de ces dépenses.", + }, + { + "cle": "Parc de logements", + "alias": "", + "valeur": "Au 1er janvier 2025, la France hors Mayotte compte 38,4 millions de logements. 82,5 % des logements sont des résidences principales et 54,4 % des logements individuels (maisons).", + }, + { + "cle": "logements vacants", + "alias": "", + "valeur": "Après avoir fortement augmenté entre 2005 et 2019, la part des logements vacants diminue, passant de 8,1 % en 2019 à 7,7 % en 2025 ; en 2025, 3,0 millions de logements sont vacants.", + }, + { + "cle": "résidences secondaires ou logements occasionnels", + "alias": "", + "valeur": "Au 1er janvier 2025, 3,8 millions de logements sont des résidences secondaires ou des logements occasionnels ; après avoir augmenté entre 2011 et 2017, leur part dans l'ensemble du parc est stable.", + }, + { + "cle": "ménages sont propriétaires de leur résidence principale", + "alias": "", + "valeur": "Au 1er janvier 2025, 57,4 % des ménages sont propriétaires de leur résidence principale.", + }, + { + "cle": "smic", + "alias": "Salaire minimum interprofessionnel de croissance", + "valeur": "Depuis le 1er janvier 2026, le Smic brut s'élève à 12,02 euros par heure, soit 1 823,03 euros par mois pour 151,67 heures de travail.", + }, + { + "cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur privé", + "alias": "", + "valeur": "En 2023, le salaire mensuel moyen en équivalent temps plein (EQTP) dans le secteur privé est de 2 730 euros, nets de cotisations et contributions sociales.", + }, + { + "cle": "salaire mensuel moyen en équivalent temps plein (EQTP) secteur public", + "alias": "", + "valeur": "Dans la fonction publique, tous statuts confondus, un salarié gagne en moyenne 2 650 euros nets par mois en EQTP en 2023.", + }, + { + "cle": "revenus non salariés", + "alias": "", + "valeur": "En 2023, hors agriculture, les non-salariés classiques (micro-entrepreneurs exclus) retirent en moyenne 4 040 euros par mois de leur activité non salariée. Cette moyenne recouvre de fortes disparités selon la nature des emplois.", + }, + { + "cle": "salaires horaires", + "alias": "", + "valeur": "Au premier trimestre 2026, les salaires horaires augmentent de 0,3 % sur le trimestre et de 2,0 % sur un an", + }, + { + "cle": "coût horaire du travail", + "alias": "Indice du coût du travail – ICT", + "valeur": "Au premier trimestre 2026, le coût horaire du travail (salaires, cotisations et taxes, déduction faite des exonérations et subventions) de l'ensemble du secteur marchand non agricole (hors services aux ménages) freine significativement, dans le sillage des salaires : +0,5 % sur le trimestre et + 2,3 % sur un an.", + }, + { + "cle": "création entreprises", + "alias": "", + "valeur": "En 2025, 1 165 800 entreprises ont été créées en France, dont 758 500 sous forme d'entrepreneurs individuels ayant adopté le régime de la microentreprise (micro-entrepreneurs).", + }, + { + "cle": "défaillances d'entreprises", + "alias": "", + "valeur": "En 2025, 68 872 unités légales ont été en situation de défaillance.", + }, + { + "cle": "entreprises marchandes non agricoles et non financières en France", + "alias": "", + "valeur": "En 2023, en France, les secteurs marchands non agricoles et non financiers (incluant toutefois les exploitations forestières, les auxiliaires de services financiers et d'assurance et les holdings) comptent 5,2 millions d'entreprises. Ces entreprises emploient 15,9 millions de salariés en équivalent temps plein (EQTP).", + }, + { + "cle": "exploitations agricoles", + "alias": "", + "valeur": "Dans le secteur agricole, l'usage est de compter plutôt des exploitations agricoles ; en 2023, la France métropolitaine en compte 349 600 et la main d'œuvre agricole s'élève à 663 200 EQTP.", + }, + { + "cle": "commerce", + "alias": "", + "valeur": "En 2023, le commerce rassemble 739 128 entreprises. Elles réalisent un chiffre d'affaires de 1 728 milliards d'euros et dégagent une valeur ajoutée (VA) de 272 milliards d'euros. Fin 2024, 3,4 millions de personnes occupent un emploi salarié dans le commerce.", + }, + { + "cle": "industrie", + "alias": "", + "valeur": "En 2023, l'industrie rassemble 322 386 entreprises. Elles réalisent un chiffre d'affaire de 1 544 milliards d'euros et dégagent une valeur ajoutée (VA) de 368 milliards d'euros. Fin 2024, 3,3 millions de personnes occupent un emploi salarié dans l'industrie.", + }, + { + "cle": "construction", + "alias": "", + "valeur": "En 2023, la construction rassemble 587 898 entreprises. Elles réalisent un chiffre d'affaires de 405 milliards d'euros et dégagent une valeur ajoutée (VA) de 128 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans la construction.", + }, + { + "cle": "services", + "alias": "", + "valeur": "En 2023, les services principalement marchands non financiers comptent plus de 2,3 millions d'entreprises. Ces entreprises réalisent un chiffre d'affaires de 995 milliards d'euros et dégagent une valeur ajoutée (VA) de 475 milliards d'euros. Fin 2024, 7,5 millions de personnes (y compris les intérimaires) occupent un emploi salarié dans les services principalement marchands non financiers.", + }, + { + "cle": "transports", + "alias": "", + "valeur": "En 2023, les transports et l'entreposage rassemblent 193 101 entreprises. Elles réalisent un chiffre d'affaires de 267 milliards d'euros et dégagent une valeur ajoutée (VA) de 102 milliards d'euros. Fin 2024, 1,5 million de personnes occupent un emploi salarié dans les transports et l'entreposage.", + }, + { + "cle": "entreprises de l'économie sociale", + "alias": "", + "valeur": "Les entreprises de l'économie sociale se caractérisent par leur famille de l'économie sociale, à la fois privé et à caractère essentiellement non lucratif. En 2022, elles représentent 9,8 % de l'emploi salarié total en équivalent temps plein. Les associations emploient 73 % de ce volume de travail salarié ; 14 % est employé par les coopératives, 6 % par les mutuelles, 5 % par les fondations et 3 % par les autres organismes privés à but non-lucratif.", + }, + { + "cle": "Population quartiers prioritaires de la politique de la ville", + "alias": "QPV", + "valeur": "Les quartiers prioritaires de la politique de la ville (QPV) tels que définis par le décret n° 2015-1138 du 14 septembre 2015 regroupent 7,9 % de la population en 2020.", + }, + { + "cle": "Population unités urbaines", + "alias": "", + "valeur": "Les unités urbaines rassemblent toujours plus d'habitants. En 2022, en France métropolitaine, elles représentent 78,8 % de la population, soit 51,9 millions d'habitants. À l'exception de l'unité urbaine de Paris qui concentre près de 11 millions d'habitants, les 10 plus grandes unités urbaines françaises comptent chacune entre 0,5 et 2 millions d'habitants.", + }, + { + "cle": "mode déplacement domicile travail", + "alias": "", + "valeur": "Pour se rendre au travail, les personnes en emploi se déplacent majoritairement en voiture ou en deux-roues motorisés (71 % en 2022). 15 % des personnes en emploi empruntent les transports en commun.", + }, + { + "cle": "dépense nationale protection de l'environnement", + "alias": "", + "valeur": "En 2022, la dépense nationale en faveur de la protection de l'environnement s'élève à 63,7 milliards d'euros (Md€). Elle est dédiée à la protection de l'air, de la biodiversité et des paysages, la collecte et traitement des déchets, la protection et dépollution des sols et des eaux, la lutte contre le bruit et d'autres activités de protection de l'environnement (frais de fonctionnement de l'administration publique et des opérateurs chargés des questions environnementales notamment). Les entreprises sont les principaux financeurs des dépenses de protection de l'environnement (22,6 Md€, soit 35 %), devant les administrations publiques (État et ses ministères, collectivités locales, organismes publics) (22,2 Md€, soit 35 %) et les ménages (18,1 Md€, soit 28 %).", + }, + { + "cle": "indice de référence des loyers", + "alias": "IRL", + "valeur": "Au deuxième trimestre 2026, l'indice de référence des loyers s'établit à 148,37. Sur un an, il augmente de 1,15 % après +0,78 % au trimestre précédent.", + }, + { + "cle": "indice des loyers commerciaux", + "alias": "ILC", + "valeur": "Au premier trimestre 2026, l'indice des loyers commerciaux s'établit à 135,26. Sur un an, il baisse de 0,45 % (après -0,50 % au trimestre précédent).", + }, + { + "cle": "indice des loyers des activités tertiaires", + "alias": "ILAT", + "valeur": "Au premier trimestre 2026, l'indice des loyers des activités tertiaires s'établit à 137,42. Sur un an, il augmente de 0,09 % (après -0,06 % au trimestre précédent).", + }, + { + "cle": "indice du coût de la construction", + "alias": "ICC", + "valeur": "L'indice du coût de la construction (ICC) s'établit à 2 084 au premier trimestre 2026. Il est en hausse de 1,26 % sur un trimestre (après +0,10 % au trimestre précédent). Sur un an, il baisse de 2,89 % (après -2,37 % au trimestre précédent).", + }, + { + "cle": "index du bâtiment tous corps d'état", + "alias": "BT01 ; index bâtiment BT01", + "valeur": "En mai 2026, l'index Bâtiment BT01 « Tous corps d'état » s'établit à 137,9, en référence 100 en 2010.", + }, + { + "cle": "index général des travaux publics", + "alias": "TP01 ; index travaux publics TP01", + "valeur": "En mai 2026, l'index Travaux publics TP01 « Index général tous travaux » s'établit à 140,4, en référence 100 en 2010.", + }, + { + "cle": "index ingénierie", + "alias": "ING ; indice ING", + "valeur": "En mai 2026, l'index divers de la construction ING « Ingénierie » s'établit à 138,3, en référence 100 en 2010.", + }, ] diff --git a/src/mcpdiffusion/data/themes.py b/src/mcpdiffusion/data/themes.py index 1b30b18..a43534b 100644 --- a/src/mcpdiffusion/data/themes.py +++ b/src/mcpdiffusion/data/themes.py @@ -74,7 +74,8 @@ "Les inscrits a France Travail", ], "Wages and labour costs": [ - "Indice du cout horaire du travail revise - Tous salaries (ICHT, ICHTrev-TS) - Publication arretee depuis le 06/10/2023", + "Indice du cout horaire du travail revise - Tous salaries (ICHT, ICHTrev-TS)" + " - Publication arretee depuis le 06/10/2023", "Indice du cout du travail (ICT) - Resultats detailles", "Indice du cout du travail (ICT) - Estimation flash", "Salaires de base - Comparaison France-Allemagne", diff --git a/src/mcpdiffusion/infra/__init__.py b/src/mcpdiffusion/infra/__init__.py index 6333438..a2a266c 100644 --- a/src/mcpdiffusion/infra/__init__.py +++ b/src/mcpdiffusion/infra/__init__.py @@ -1,3 +1,3 @@ """Infrastructure: external clients (ES, HTTP, SPARQL).""" -# Fixme: each of the dependency functions in that folder does not provide proper typing which is a pity \ No newline at end of file +# Fixme: each of the dependency functions in that folder does not provide proper typing which is a pity diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/infra/lifespan.py index dbbc99d..48774f7 100644 --- a/src/mcpdiffusion/infra/lifespan.py +++ b/src/mcpdiffusion/infra/lifespan.py @@ -5,15 +5,13 @@ from typing import Any from elasticsearch import AsyncElasticsearch -from httpx import AsyncClient, Timeout from fastmcp.server.lifespan import lifespan +from httpx import AsyncClient, Timeout logger = logging.getLogger(__name__) # insee.fr serves different markup to unknown agents, so the scraper has to look like a browser. -INSEE_USER_AGENT = ( - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" -) +INSEE_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" # The APIs have no such requirement, so they get an honest identity. MELODI_USER_AGENT = "McpDiffusion/0.1" SPARQL_USER_AGENT = "MCP-RMeS/2.0" diff --git a/src/mcpdiffusion/models/feedback.py b/src/mcpdiffusion/models/feedback.py index 1af8115..1ad83e8 100644 --- a/src/mcpdiffusion/models/feedback.py +++ b/src/mcpdiffusion/models/feedback.py @@ -1,4 +1,5 @@ """Pydantic schemas for the feedback tool.""" + from __future__ import annotations from pydantic import BaseModel, Field diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 2f99c01..f65a45b 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -1,8 +1,8 @@ """Pydantic schemas for INSEE.fr tools.""" + from __future__ import annotations from enum import StrEnum -from typing import Optional from pydantic import BaseModel, Field @@ -52,32 +52,27 @@ class ThemeConjoncture(StrEnum): # --- Shared output model --- + class DocumentHit(BaseModel): """Whitelisted publication record returned by INSEE.fr search tools.""" + id: str = Field(description="Elasticsearch document id.") score: float = Field(description="Relevance score from Elasticsearch.") - titre: Optional[str] = None - soustitre: Optional[str] = None - chapo: Optional[str] = None - anneediffusion: Optional[str] = Field( - default=None, description="Publication year as indexed." - ) - zone: Optional[str] = Field( - default=None, description="Geographic zone (e.g. 'France', 'Bretagne')." - ) - theme: Optional[str] = None - collection_libelle: Optional[str] = Field( + titre: str | None = None + soustitre: str | None = None + chapo: str | None = None + anneediffusion: str | None = Field(default=None, description="Publication year as indexed.") + zone: str | None = Field(default=None, description="Geographic zone (e.g. 'France', 'Bretagne').") + theme: str | None = None + collection_libelle: str | None = Field( default=None, - description="Collection the publication belongs to " - "(e.g. 'Insee Premiere', 'Informations rapides').", + description="Collection the publication belongs to (e.g. 'Insee Premiere', 'Informations rapides').", ) - idproduit: Optional[str] = Field( + idproduit: str | None = Field( default=None, description="INSEE product identifier (often equal to the ES id).", ) - url: str = Field( - description="Relative URL ready to feed into `get_insee_document`." - ) + url: str = Field(description="Relative URL ready to feed into `get_insee_document`.") # --- search_insee_documents --- @@ -91,18 +86,15 @@ class SearchInseeDocumentsInput(BaseModel): default=INSEETheme.ALL, description="Optional top-level INSEE theme used to restrict the search. Default: ALL.", ) - year_of_reference: Optional[int] = Field( + year_of_reference: int | None = Field( default=None, - description=( - "Hard filter on publication year (e.g. 2024). Leave null to " - "search all years." - ), + description=("Hard filter on publication year (e.g. 2024). Leave null to search all years."), ) geo_niveau: INSEEGeo = Field( default=INSEEGeo.FRANCE, description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", ) - geo_keyword: Optional[str] = Field( + geo_keyword: str | None = Field( default=None, description=( "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " @@ -118,6 +110,7 @@ class SearchInseeDocumentsInput(BaseModel): le=20, ) + # Fixme: the same model shape is used 3 times class SearchInseeDocumentsOutput(BaseModel): results: list[DocumentHit] @@ -126,23 +119,21 @@ class SearchInseeDocumentsOutput(BaseModel): # --- search_insee_chiffrecle --- + class SearchInseeChiffrecleInput(BaseModel): query: str = Field( description="Natural-language search query describing the statistics to retrieve.", examples=["population de Lyon", "taux de chomage 2024", "PIB France"], ) - year_of_reference: Optional[int] = Field( + year_of_reference: int | None = Field( default=None, - description=( - "Hard filter on publication year (e.g. 2024). Leave null to " - "search all years." - ), + description=("Hard filter on publication year (e.g. 2024). Leave null to search all years."), ) geo_niveau: INSEEGeo = Field( default=INSEEGeo.FRANCE, description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", ) - geo_keyword: Optional[str] = Field( + geo_keyword: str | None = Field( default=None, description=( "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " @@ -164,6 +155,7 @@ class SearchInseeChiffrecleOutput(BaseModel): # --- search_insee_conjoncture --- + class SearchInseeConjonctureInput(BaseModel): query: str = Field( description=( @@ -172,14 +164,14 @@ class SearchInseeConjonctureInput(BaseModel): ), examples=["consommation", "hotel", "PIB"], ) - theme_conjoncture: Optional[ThemeConjoncture] = Field( + theme_conjoncture: ThemeConjoncture | None = Field( default=None, description=( "Optional broad category to restrict the search. Each category " "contains multiple sub-themes. Leave null to search across all." ), ) - year_of_reference: Optional[int] = Field( + year_of_reference: int | None = Field( default=None, description=( "Hard filter on publication year (e.g. 2024). Leave null to " @@ -202,12 +194,10 @@ class SearchInseeConjonctureOutput(BaseModel): # --- get_insee_document --- + class GetInseeDocumentInput(BaseModel): list_of_url: list[str] = Field( - description=( - "List of relative URLs to retrieve (e.g. " - "'/fr/statistiques/4277658?sommaire=4318291')." - ), + description=("List of relative URLs to retrieve (e.g. '/fr/statistiques/4277658?sommaire=4318291')."), examples=[["/fr/statistiques/4277658?sommaire=4318291"]], ) include_sommaire: bool = Field( @@ -231,8 +221,8 @@ class GetInseeDocumentInput(BaseModel): class DocumentResult(BaseModel): id: str = Field(description="The input URL that produced this entry.") status: str = Field(description="'success' or 'error'.") - markdown_content: Optional[str] = None - sommaire: Optional[dict[str, dict[str, str]]] = Field( + markdown_content: str | None = None + sommaire: dict[str, dict[str, str]] | None = Field( default=None, description=( "Parsed table of contents as " @@ -244,7 +234,7 @@ class DocumentResult(BaseModel): default=False, description="True if markdown_content was clipped due to size.", ) - error: Optional[str] = Field( + error: str | None = Field( default=None, description="Human-readable error message when status == 'error'.", ) @@ -257,15 +247,14 @@ class GetInseeDocumentOutput(BaseModel): # --- get_insee_homepage --- + class KeyValueIndicator(BaseModel): key: str = Field(description="Indicator name (e.g. 'smic', 'PIB annuel').") alias: str = Field( default="", description="Optional alias / alternative name for the indicator.", ) - value: str = Field( - description="Pre-computed textual description of the latest figure." - ) + value: str = Field(description="Pre-computed textual description of the latest figure.") class KeyIndicatorsOutput(BaseModel): diff --git a/src/mcpdiffusion/models/melodi.py b/src/mcpdiffusion/models/melodi.py index d8ef8dc..a5cdc33 100644 --- a/src/mcpdiffusion/models/melodi.py +++ b/src/mcpdiffusion/models/melodi.py @@ -1,13 +1,14 @@ """Pydantic schemas for Melodi tools.""" + from __future__ import annotations from typing import Any from pydantic import BaseModel, Field - # --- get_melodi_observations --- + class GetMelodiObservationsInput(BaseModel): dataset_id: str = Field( description="Identifier of the Melodi dataset (from search_melodi_datasets).", @@ -51,6 +52,7 @@ class GetMelodiObservationsOutput(BaseModel): # --- search_melodi_datasets --- + class SearchMelodiDatasetsInput(BaseModel): french_query: str = Field( description=( @@ -91,10 +93,7 @@ class DatasetDescription(BaseModel): class DatasetSearchResult(BaseModel): dataset_id: str dataset_columns: str = Field( - description=( - "Pipe-separated list of available columns formatted as " - "'COLUMN_ID Label'." - ) + description=("Pipe-separated list of available columns formatted as 'COLUMN_ID Label'.") ) dataset_description: DatasetDescription dataset_score: float @@ -106,6 +105,7 @@ class SearchMelodiDatasetsOutput(BaseModel): # --- search_melodi_modalities --- + class SearchMelodiModalitiesInput(BaseModel): dataset_id: str = Field( description="Identifier of the Melodi dataset (from search_melodi_datasets).", diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index a57dda0..ee2bc4c 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -1,12 +1,12 @@ """Pydantic schemas for RMES (SPARQL) tools.""" + from __future__ import annotations from enum import StrEnum -from typing import Any, Literal, Optional +from typing import Any, Literal from pydantic import BaseModel, Field - # --- Shared RMES constants exposed to tools --- # Fixme: a lot of values in here belongs in settings @@ -20,6 +20,7 @@ # --- Graph taxonomy --- + class GraphCategoryChoice(StrEnum): ALL = "ALL" QUALITE_RAPPORTS = "qualite_rapports" @@ -44,8 +45,9 @@ class GraphRow(BaseModel): # --- RMES_list_graphs --- + class ListGraphsInput(BaseModel): - contains: Optional[str] = Field( + contains: str | None = Field( default=None, description=( "Filtre les graphes dont l'URI contient cette sous-chaine (insensible a la " @@ -75,7 +77,7 @@ class CategoryBucket(BaseModel): count: int total_triples: int examples: list[str] - graphs: Optional[list[GraphRow]] = None + graphs: list[GraphRow] | None = None class ListGraphsOutput(BaseModel): @@ -85,6 +87,7 @@ class ListGraphsOutput(BaseModel): # --- RMES_describe_resource --- + class DescribeResourceInput(BaseModel): uri: str = Field( description="URI complete de la ressource RDF a decrire.", @@ -105,8 +108,8 @@ class ResourceProperty(BaseModel): direction: Literal["outgoing", "incoming"] predicate: str value: str - value_type: Optional[str] = None - lang: Optional[str] = None + value_type: str | None = None + lang: str | None = None class DescribeResourceOutput(BaseModel): @@ -117,6 +120,7 @@ class DescribeResourceOutput(BaseModel): # --- RMES_run_sparql --- + class RunSparqlInput(BaseModel): full_sparql_query: str = Field( description="Requete SPARQL complete (SELECT / ASK / CONSTRUCT / DESCRIBE).", @@ -136,8 +140,8 @@ class RunSparqlInput(BaseModel): class RunSparqlOutput(BaseModel): format: Literal["json", "turtle"] = "json" - limit_added: Optional[int] = None - hint: Optional[str] = None - variables: Optional[list[str]] = None - bindings: Optional[list[dict[str, Any]]] = None - turtle: Optional[str] = None + limit_added: int | None = None + hint: str | None = None + variables: list[str] | None = None + bindings: list[dict[str, Any]] | None = None + turtle: str | None = None diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 1892c0f..9d6baa3 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -43,10 +43,13 @@ # None of them logs how many results a tool returned. If empty results become hard to diagnose, add an # `on_call_tool` middleware that inspects the ToolResult, or have the tool report it with `ctx.info`. mcp.add_middleware( - # transform_errors would promote our ToolErrors to JSON-RPC protocol errors labelled - # "Internal error", losing is_error and the message the LLM is meant to act on. We only - # want the logging and error counting. - ErrorHandlingMiddleware(transform_errors=False), + # include_traceback puts the original cause in the server log, which is the only place it is + # recoverable. transform_errors would promote our ToolErrors to JSON-RPC protocol errors labelled + # "Internal error", losing is_error and the message the caller is meant to act on. + ErrorHandlingMiddleware( + transform_errors=False, + include_traceback=True, + ), ) mcp.add_middleware( SlidingWindowRateLimitingMiddleware( diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py index cada1bc..ccc6a8e 100644 --- a/src/mcpdiffusion/services/feedback.py +++ b/src/mcpdiffusion/services/feedback.py @@ -1,4 +1,5 @@ """Business logic for the feedback tool.""" + from __future__ import annotations from datetime import datetime @@ -42,9 +43,7 @@ async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: # just hope this is not ultimately fed to an LLM # Fixme: a big flaw is that feedback.md is versioned, so user feedback might be fed into git # Fixme: beware the data is lost on each restart - f"## {timestamp} — {params.username}\n\n" - f"{params.feedback}\n\n" - "---\n\n" + f"## {timestamp} — {params.username}\n\n{params.feedback}\n\n---\n\n" ) # Fixme: this call is blocking the event loop diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 1141dd3..82342ff 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -1,17 +1,16 @@ """Business logic for get_insee_document tool.""" + from __future__ import annotations +import logging from collections import defaultdict from urllib.parse import urljoin, urlparse +import httpx from bs4 import BeautifulSoup from trafilatura import extract from trafilatura.settings import Extractor -import logging - -import httpx - from ..core.errors import AppToolError from ..models.insee import ( DocumentResult, @@ -37,15 +36,15 @@ def _as_relative(url: str) -> str: p = urlparse(url) return f"{p.path}?{p.query}" if p.query else p.path + # Fixme: some complex composed types are involved multiple times - ex: list[dict[str, str] -# it might be better to leverage Pydantic and create meaningful type aliases - ex TableOfContentParams = list[dict[str, str] +# it might be better to leverage Pydantic and create meaningful type aliases +# - ex TableOfContentParams = list[dict[str, str]] def _parse_sommaire(html: str, base_url: str) -> list[dict[str, str]]: soup = BeautifulSoup(html, "lxml") results: list[dict[str, str]] = [] - sommaire_section = soup.find( - lambda t: t.has_attr("class") and any("sommaire" in c for c in t["class"]) - ) + sommaire_section = soup.find(lambda t: t.has_attr("class") and any("sommaire" in c for c in t["class"])) if not sommaire_section: return [] @@ -67,9 +66,7 @@ def _parse_sommaire(html: str, base_url: str) -> list[dict[str, str]]: title = a.get_text(strip=True) absolute = urljoin(base_url, a.get("href", "")) rel_url = _as_relative(absolute) - results.append( - {"category": category_name, "title": title, "url": rel_url} - ) + results.append({"category": category_name, "title": title, "url": rel_url}) else: a = top_li.find("a") if not a: @@ -123,8 +120,7 @@ async def _fetch_html(url: str, http_client: httpx.AsyncClient) -> str: if exc.response.status_code == 404: raise AppToolError( "NOT_FOUND", - f"INSEE document not found at {target} (HTTP 404). " - "Verify the URL with `search_insee_documents`.", + f"INSEE document not found at {target} (HTTP 404). Verify the URL with `search_insee_documents`.", ) else: raise AppToolError( @@ -159,8 +155,7 @@ async def get_insee_document( if not params.list_of_url: raise AppToolError( "INVALID_INPUT", - "list_of_url must contain at least one URL. " - "Use `search_insee_documents` to find URLs first.", + "list_of_url must contain at least one URL. Use `search_insee_documents` to find URLs first.", ) results: list[DocumentResult] = [] diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index f835213..cfe42af 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -3,6 +3,7 @@ Centralizes query building, collection filtering, and search execution for search_insee_documents, search_insee_conjoncture, and search_insee_chiffrecle. """ + from __future__ import annotations from collections.abc import Iterable @@ -25,7 +26,7 @@ def _coerce_hit_value(value) -> str | None: return str(value) -# Build query +# Build query def build_text_clauses( query: str | None, year_of_reference: int | None, @@ -52,9 +53,7 @@ def build_text_clauses( fuzziness="AUTO", ) ) - should.append( - Q("match_phrase", titre={"query": query, "boost": 1}) - ) + should.append(Q("match_phrase", titre={"query": query, "boost": 1})) if year_of_reference: filters.append( @@ -96,9 +95,7 @@ def apply_collection_filters( if must_only_rapides: filters.append(Q("term", collection_libelle="Informations rapides")) elif must_not_rapides: - filters.append( - Q("bool", must_not=[Q("term", collection_libelle="Informations rapides")]) - ) + filters.append(Q("bool", must_not=[Q("term", collection_libelle="Informations rapides")])) # Fixme: the 1st check seems useless if theme and theme != "ALL": @@ -127,13 +124,13 @@ def apply_collection_filters( fuzziness="AUTO", ) ) - should.append( - Q("match_phrase", zone={"query": geo_keyword, "boost": 5}) - ) + should.append(Q("match_phrase", zone={"query": geo_keyword, "boost": 5})) return filters, should -# Execute search with built query + +# Execute search with built query + async def execute_search( *, diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 3f81fdf..22d844c 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -1,12 +1,12 @@ """Business logic for Melodi tools (observations, datasets, modalities).""" + from __future__ import annotations from typing import Any import httpx -from elasticsearch import AsyncElasticsearch +from elasticsearch import AsyncElasticsearch, TransportError from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError from ..core.errors import AppToolError from ..models.melodi import ( @@ -86,9 +86,6 @@ async def get_melodi_observations( "PARSE_ERROR", "Melodi API response did not contain an 'observations' list.", ) - # Fixme: this not not an appropriate fix - # this piece of code is unreachable since 'fail' raises already before - raise # pragma: no cover if params.list_of_year: years_str = {str(y) for y in params.list_of_year} @@ -98,9 +95,7 @@ async def get_melodi_observations( obs for obs in observations # Fixme: 'TIME_PERIOD' could be sanitized - if (obs.get("dimensions", {}) - .get("TIME_PERIOD", "") - .split("-")[0]) in years_str + if (obs.get("dimensions", {}).get("TIME_PERIOD", "").split("-")[0]) in years_str ] sliced = observations[: params.number_of_results] @@ -110,9 +105,11 @@ async def get_melodi_observations( count=len(sliced), ) + # Fixme: I believe this is not the correct place (inside the service) to place a raw complex query # the code might benefit having a repository layer to encapsulate data access + async def search_melodi_datasets( params: SearchMelodiDatasetsInput, *, @@ -121,21 +118,9 @@ async def search_melodi_datasets( ) -> SearchMelodiDatasetsOutput: filters: list[dict[str, Any]] = [] if params.start_year: - filters.append({ - "range": { - "metadata.temporal.endPeriod": { - "gte": f"{params.start_year}-01-01" - } - } - }) + filters.append({"range": {"metadata.temporal.endPeriod": {"gte": f"{params.start_year}-01-01"}}}) if params.end_year: - filters.append({ - "range": { - "metadata.temporal.startPeriod": { - "lte": f"{params.end_year}-12-31" - } - } - }) + filters.append({"range": {"metadata.temporal.startPeriod": {"lte": f"{params.end_year}-12-31"}}}) body = { "size": params.number_of_results, @@ -203,8 +188,7 @@ async def search_melodi_datasets( except (ESConnectionError, TransportError) as exc: raise AppToolError( "BACKEND_UNAVAILABLE", - f"Melodi datasets search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", + f"Melodi datasets search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) @@ -281,20 +265,14 @@ async def search_melodi_modalities( except (ESConnectionError, TransportError) as exc: raise AppToolError( "BACKEND_UNAVAILABLE", - f"Melodi columns search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", + f"Melodi columns search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) results: list[ColumnResult] = [] for hit in ds_column.get("hits", {}).get("hits", []): modalities: list[Modality] = [] - inner_hits = ( - hit.get("inner_hits", {}) - .get("modalities", {}) - .get("hits", {}) - .get("hits", []) - ) + inner_hits = hit.get("inner_hits", {}).get("modalities", {}).get("hits", {}).get("hits", []) for m in inner_hits: src = m.get("_source", {}) label = src.get("label", {}) or {} diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 29ebe04..5f3d71f 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -3,6 +3,7 @@ Contains: taxonomy, categorization, SPARQL execution, graph cache, and high-level operations for the three RMES tools. """ + from __future__ import annotations import logging @@ -16,18 +17,18 @@ from ..models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, GRAPH_BASE, - MAX_ROW_LIMIT, MAX_QUERY_TIMEOUT_SECONDS, + MAX_ROW_LIMIT, CategoryBucket, + DescribeResourceInput, DescribeResourceOutput, GraphCategoryChoice, GraphRow, + ListGraphsInput, ListGraphsOutput, ResourceProperty, - RunSparqlOutput, RunSparqlInput, - DescribeResourceInput, - ListGraphsInput, + RunSparqlOutput, ) # Fixme: follow a clear convention for logger names @@ -67,6 +68,7 @@ # Fixme: this is too broad of a type CategoryMatcher = Any # Callable[[str], bool] + # Fixme: you can use an immutable (frozen) dataclass instead - ex: annotate the class with '@dataclass(frozen=True)' class _CategoryRule: __slots__ = ("key", "label", "description", "match") @@ -208,7 +210,7 @@ def _match_prefix(prefix: str) -> CategoryMatcher: def _strip_graph_base(graph_uri: str) -> str: if graph_uri.startswith(GRAPH_BASE): - return graph_uri[len(GRAPH_BASE):] + return graph_uri[len(GRAPH_BASE) :] return graph_uri @@ -255,6 +257,7 @@ def _accept_header(query_form: str) -> str: # Low-level SPARQL execution # --------------------------------------------------------------------------- + async def _execute_sparql( query: str, timeout: float, @@ -343,8 +346,7 @@ async def _get_raw_graph_rows( now = time.time() if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: query = ( - "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } " - "GROUP BY ?g ORDER BY DESC(?nbTriples)" + "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } GROUP BY ?g ORDER BY DESC(?nbTriples)" ) # Fixme: note that while this request runs (async nature), # other concurrent requests can still enter the current block @@ -358,8 +360,7 @@ async def _get_raw_graph_rows( endpoint=endpoint, ) rows = [ - {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} - for b in result["results"]["bindings"] + {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} for b in result["results"]["bindings"] ] _GRAPH_CACHE["data"] = rows _GRAPH_CACHE["ts"] = now @@ -371,6 +372,7 @@ async def _get_raw_graph_rows( # High-level tool operations # --------------------------------------------------------------------------- + def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: buckets: dict[str, CategoryBucket] = {} for row in rows: @@ -472,8 +474,11 @@ async def describe_resource( }} LIMIT {MAX_ROW_LIMIT} """ result = await _execute_sparql( - query, timeout=DEFAULT_QUERY_TIMEOUT_SECONDS, max_rows=MAX_ROW_LIMIT, - sparql_client=sparql_client, endpoint=endpoint, + query, + timeout=DEFAULT_QUERY_TIMEOUT_SECONDS, + max_rows=MAX_ROW_LIMIT, + sparql_client=sparql_client, + endpoint=endpoint, ) properties = _parse_bindings_to_properties(result["results"]["bindings"]) @@ -494,8 +499,11 @@ async def run_sparql( max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) result = await _execute_sparql( - params.full_sparql_query, timeout=params.timeout, max_rows=max_rows, - sparql_client=sparql_client, endpoint=endpoint, + params.full_sparql_query, + timeout=params.timeout, + max_rows=max_rows, + sparql_client=sparql_client, + endpoint=endpoint, ) if result.get("format") == "turtle": diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 881613d..6d0d42e 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -4,24 +4,28 @@ wires all of them in one place; to disable a tool, comment out its import and the corresponding call below. """ + from __future__ import annotations from fastmcp import FastMCP from ..config.settings import Settings -from .melodi_get_observations import register_get_melodi_observations -from .melodi_search_datasets import register_search_melodi_datasets -from .melodi_search_modalities import register_search_melodi_modalities +# Imported but never registered: send_feedback is not exposed. Decide whether to wire it up or +# drop it, then remove this import or the noqa. +from .extras_send_feedback import register_extras_send_feedback # noqa: F401 from .insee_get_document import register_get_insee_document from .insee_get_homepage import register_get_insee_homepage -from .insee_search_documents import register_search_insee_documents -from .insee_search_conjoncture import register_search_insee_conjoncture from .insee_search_chiffrecle import register_search_insee_chiffreclef -from .rmes_list_graphs import register_rmes_list_graphs +from .insee_search_conjoncture import register_search_insee_conjoncture +from .insee_search_documents import register_search_insee_documents +from .melodi_get_observations import register_get_melodi_observations +from .melodi_search_datasets import register_search_melodi_datasets +from .melodi_search_modalities import register_search_melodi_modalities from .rmes_describe_resource import register_rmes_describe_resource +from .rmes_list_graphs import register_rmes_list_graphs from .rmes_run_sparql import register_rmes_run_sparql -from .extras_send_feedback import register_extras_send_feedback + # Fixme: there might be better pattern instead of iterating with if statements on tool groups def register_tools(mcp: FastMCP, settings: Settings) -> None: diff --git a/src/mcpdiffusion/tools/extras_send_feedback.py b/src/mcpdiffusion/tools/extras_send_feedback.py index f8d3117..503292f 100644 --- a/src/mcpdiffusion/tools/extras_send_feedback.py +++ b/src/mcpdiffusion/tools/extras_send_feedback.py @@ -1,4 +1,5 @@ """Tool: send_feedback -- thin registration layer.""" + from __future__ import annotations from fastmcp import FastMCP diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 61889df..1da3c09 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -1,4 +1,5 @@ """Tool: get_insee_document -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index 389d01d..9941e35 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -1,4 +1,5 @@ """Tool: get_insee_homepage -- thin registration layer.""" + from __future__ import annotations from fastmcp import FastMCP @@ -7,6 +8,7 @@ from ..data.indicators import DICT_KV from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator + # Fixme: the tool contains no service which is kind of breaking the convention I saw earlier # this correlates unit testing to the tool mechanics def register_get_insee_homepage(mcp: FastMCP) -> None: diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index bc564ee..efc3c83 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -1,4 +1,5 @@ """Tool: search_insee_chiffrecle -- thin registration layer.""" + from __future__ import annotations from elasticsearch import ConnectionError as ESConnectionError @@ -18,6 +19,7 @@ execute_search, ) + # Fixme: the orchestration present in that function belongs in a service # Indeed, the approach from one tool to another is inconsistent def register_search_insee_chiffreclef(mcp: FastMCP, *, index: str) -> None: @@ -58,8 +60,7 @@ async def search_insee_chiffrecle( except (ESConnectionError, TransportError) as exc: raise AppToolError( "BACKEND_UNAVAILABLE", - f"INSEE documents search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", + f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) return SearchInseeChiffrecleOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index f9654b6..deca8e1 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -1,4 +1,5 @@ """Tool: search_insee_conjoncture -- thin registration layer.""" + from __future__ import annotations from elasticsearch import ConnectionError as ESConnectionError @@ -20,6 +21,7 @@ execute_search, ) + # Fixme: again, a lot of code in that tool that should belong in the service def register_search_insee_conjoncture(mcp: FastMCP, *, index: str) -> None: @mcp.tool( @@ -61,8 +63,7 @@ async def search_insee_conjoncture( except (ESConnectionError, TransportError) as exc: raise AppToolError( "BACKEND_UNAVAILABLE", - f"INSEE conjoncture search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", + f"INSEE conjoncture search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) return SearchInseeConjonctureOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index 0d897d5..6d6cbf9 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -1,4 +1,5 @@ """Tool: search_insee_documents -- thin registration layer.""" + from __future__ import annotations from elasticsearch import ConnectionError as ESConnectionError @@ -18,6 +19,7 @@ execute_search, ) + # Fixme: state clear conventions between what goes to a tool and what do not # most of the code might belong in the service def register_search_insee_documents(mcp: FastMCP, *, index: str) -> None: @@ -56,8 +58,7 @@ async def search_insee_documents( except (ESConnectionError, TransportError) as exc: raise AppToolError( "BACKEND_UNAVAILABLE", - f"INSEE documents search backend unreachable: {exc}. " - "Verify ES_HOST and try again.", + f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) return SearchInseeDocumentsOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index 76975fb..def3b5e 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -1,4 +1,5 @@ """Tool: get_melodi_observations -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py index c3ec01a..5942ddd 100644 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ b/src/mcpdiffusion/tools/melodi_search_datasets.py @@ -1,4 +1,5 @@ """Tool: search_melodi_datasets -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py index 6074f92..5ef2643 100644 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ b/src/mcpdiffusion/tools/melodi_search_modalities.py @@ -1,4 +1,5 @@ """Tool: search_melodi_modalities -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index b62562e..7cc2c0a 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -1,4 +1,5 @@ """Tool: RMES_describe_resource -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/rmes_list_graphs.py b/src/mcpdiffusion/tools/rmes_list_graphs.py index a2d31a4..b293f59 100644 --- a/src/mcpdiffusion/tools/rmes_list_graphs.py +++ b/src/mcpdiffusion/tools/rmes_list_graphs.py @@ -1,4 +1,5 @@ """Tool: RMES_list_graphs -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index 494a0b9..e5918d6 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -1,4 +1,5 @@ """Tool: RMES_run_sparql -- thin registration layer.""" + from __future__ import annotations from fastmcp import Context, FastMCP @@ -16,18 +17,18 @@ def register_rmes_run_sparql(mcp: FastMCP, *, endpoint: str) -> None: # the service plus a hardoded description # Fixme: this is also not the right place for a query description=RMES_RUN_SPARQL["tool_description"] + "\n" + KNOWN_VOCABULARIES_NOTE + "\n\n" - "Exemple -- recherche de codes NAF contenant \"extraction\" :\n" + 'Exemple -- recherche de codes NAF contenant "extraction" :\n' "PREFIX skos: \n" "SELECT ?s ?label WHERE {\n" " GRAPH {\n" " ?s skos:prefLabel ?label .\n" - " FILTER(lang(?label) = \"fr\")\n" - " FILTER(CONTAINS(LCASE(STR(?label)), \"extraction\"))\n" + ' FILTER(lang(?label) = "fr")\n' + ' FILTER(CONTAINS(LCASE(STR(?label)), "extraction"))\n' " }\n" "} LIMIT 10\n" "\n" - "Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format=\"turtle\"`, champ `turtle`) " - "plutot que des lignes (`format=\"json\"`, champs `variables`/`bindings`).", + 'Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format="turtle"`, champ `turtle`) ' + 'plutot que des lignes (`format="json"`, champs `variables`/`bindings`).', meta=RMES_RUN_SPARQL["tool_metadata"], ) async def run_sparql_tool(params: RunSparqlInput, ctx: Context) -> RunSparqlOutput: diff --git a/tests/conftest.py b/tests/conftest.py index 1677541..41dc8bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """Shared fixtures and helpers for all test files.""" + from __future__ import annotations import json @@ -21,6 +22,7 @@ # Helpers: fake httpx responses # --------------------------------------------------------------------------- + def _json_response(body: dict[str, Any], status: int = 200) -> httpx.Response: return httpx.Response( status_code=status, @@ -57,6 +59,7 @@ def _out(call_tool_result) -> dict[str, Any]: # Fake httpx.AsyncClient # --------------------------------------------------------------------------- + class FakeAsyncClient: """Drop-in replacement for httpx.AsyncClient.""" @@ -81,6 +84,7 @@ async def post(self, url, **kwargs): # Fixtures: RMES server & client # --------------------------------------------------------------------------- + @pytest.fixture def _fake_sparql_client(): """Shared FakeAsyncClient whose handler is set by mock_sparql.""" @@ -113,6 +117,7 @@ def rmes_client(rmes_mcp: FastMCP) -> Client: # Fixture: mock SPARQL endpoint # --------------------------------------------------------------------------- + @pytest.fixture def mock_sparql(_fake_sparql_client): """Return a callable that sets up the fake SPARQL endpoint.""" diff --git a/tests/test_feedback_service.py b/tests/test_feedback_service.py index acd7668..8db4372 100644 --- a/tests/test_feedback_service.py +++ b/tests/test_feedback_service.py @@ -1,4 +1,5 @@ """Unit tests for mcpdiffusion.services.feedback.""" + from __future__ import annotations from mcpdiffusion.models.feedback import SendFeedbackInput diff --git a/tests/test_insee_document_service.py b/tests/test_insee_document_service.py index e1a2955..bddab1f 100644 --- a/tests/test_insee_document_service.py +++ b/tests/test_insee_document_service.py @@ -1,19 +1,19 @@ """Unit tests for mcpdiffusion.services.insee_document.""" + from __future__ import annotations import httpx import pytest from fastmcp.exceptions import ToolError -from mcpdiffusion.core.errors import fail from mcpdiffusion.config.settings import Settings from mcpdiffusion.models.insee import GetInseeDocumentInput from mcpdiffusion.services.insee_document import ( _as_relative, + _fetch_html, _format_sommaire, _parse_sommaire, _truncate, - _fetch_html, get_insee_document, ) from tests.conftest import FakeAsyncClient @@ -25,6 +25,7 @@ # _as_relative # =================================================================== + class TestAsRelative: def test_path_only(self): assert _as_relative("https://www.insee.fr/fr/statistiques/123") == "/fr/statistiques/123" @@ -41,6 +42,7 @@ def test_already_relative(self): # _truncate # =================================================================== + class TestTruncate: def test_short_text_not_truncated(self): text, truncated = _truncate("Short text") @@ -72,6 +74,7 @@ def test_preserves_head_and_tail(self): # _parse_sommaire # =================================================================== + class TestParseSommaire: def test_empty_html_returns_empty(self): assert _parse_sommaire("", "https://www.insee.fr") == [] @@ -131,6 +134,7 @@ def test_no_ul_inside_sommaire_returns_empty(self): # _format_sommaire # =================================================================== + class TestFormatSommaire: def test_groups_by_category(self): items = [ @@ -154,11 +158,16 @@ def test_empty_category(self): # _fetch_html # =================================================================== + class TestFetchHtml: async def test_success_returns_html(self): - fake = FakeAsyncClient(lambda url, **kw: httpx.Response( - 200, content=b"OK", request=httpx.Request("GET", url), - )) + fake = FakeAsyncClient( + lambda url, **kw: httpx.Response( + 200, + content=b"OK", + request=httpx.Request("GET", url), + ) + ) result = await _fetch_html("/fr/stat/1", _SETTINGS, fake) assert result == "OK" @@ -210,6 +219,7 @@ def handler(url, **kw): # get_insee_document # =================================================================== + class TestGetInseeDocument: async def test_empty_url_list_raises(self): params = GetInseeDocumentInput(list_of_url=[]) @@ -222,7 +232,9 @@ def handler(url, **kw): params = GetInseeDocumentInput(list_of_url=["/fr/stat/1"]) result = await get_insee_document( - params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + params, + http_client=FakeAsyncClient(handler), + settings=_SETTINGS, ) assert result.count == 1 @@ -231,9 +243,13 @@ def handler(url, **kw): async def test_success_returns_markdown(self): html = "

Important paragraph.

" - fake = FakeAsyncClient(lambda url, **kw: httpx.Response( - 200, content=html.encode(), request=httpx.Request("GET", url), - )) + fake = FakeAsyncClient( + lambda url, **kw: httpx.Response( + 200, + content=html.encode(), + request=httpx.Request("GET", url), + ) + ) params = GetInseeDocumentInput( list_of_url=["/fr/stat/1"], include_sommaire=False, @@ -246,10 +262,13 @@ async def test_success_returns_markdown(self): assert result.results[0].error is None async def test_multiple_urls(self): - fake = FakeAsyncClient(lambda url, **kw: httpx.Response( - 200, content=b"

Content

", - request=httpx.Request("GET", url), - )) + fake = FakeAsyncClient( + lambda url, **kw: httpx.Response( + 200, + content=b"

Content

", + request=httpx.Request("GET", url), + ) + ) params = GetInseeDocumentInput( list_of_url=["/fr/stat/1", "/fr/stat/2"], include_sommaire=False, @@ -264,7 +283,8 @@ def handler(url, **kw): call_count[0] += 1 if call_count[0] == 1: return httpx.Response( - 200, content=b"

OK

", + 200, + content=b"

OK

", request=httpx.Request("GET", url), ) raise httpx.TimeoutException("timed out") @@ -274,7 +294,9 @@ def handler(url, **kw): include_sommaire=False, ) result = await get_insee_document( - params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + params, + http_client=FakeAsyncClient(handler), + settings=_SETTINGS, ) assert result.count == 2 assert result.results[0].status == "success" diff --git a/tests/test_insee_search_service.py b/tests/test_insee_search_service.py index 5d82bbc..ef0d76b 100644 --- a/tests/test_insee_search_service.py +++ b/tests/test_insee_search_service.py @@ -1,4 +1,5 @@ """Unit tests for mcpdiffusion.services.insee_search (pure logic, no ES).""" + from __future__ import annotations from mcpdiffusion.services.insee_search import ( @@ -7,11 +8,11 @@ build_text_clauses, ) - # =================================================================== # _coerce_hit_value # =================================================================== + class TestCoerceHitValue: def test_none_returns_none(self): assert _coerce_hit_value(None) is None @@ -36,6 +37,7 @@ def test_single_element_list(self): # build_text_clauses # =================================================================== + class TestBuildTextClauses: def test_no_arguments_returns_empty_lists(self): must, filters, should, must_not = build_text_clauses(None, None) @@ -61,7 +63,9 @@ def test_query_and_year_combined(self): def test_keywords_add_should_clauses(self): must, filters, should, must_not = build_text_clauses( - None, None, keywords=["eco", "stats"], + None, + None, + keywords=["eco", "stats"], ) assert len(should) == 2 @@ -71,7 +75,9 @@ def test_empty_keywords_ignored(self): def test_query_with_keywords(self): must, filters, should, must_not = build_text_clauses( - "chomage", None, keywords=["emploi"], + "chomage", + None, + keywords=["emploi"], ) assert len(must) == 1 assert len(should) == 2 # match_phrase + keyword @@ -81,77 +87,111 @@ def test_query_with_keywords(self): # apply_collection_filters # =================================================================== + class TestApplyCollectionFilters: def test_must_only_rapides(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=True, + [], + must_not_rapides=False, + must_only_rapides=True, ) assert len(filters) == 1 def test_must_not_rapides(self): filters, should = apply_collection_filters( - [], must_not_rapides=True, must_only_rapides=False, + [], + must_not_rapides=True, + must_only_rapides=False, ) assert len(filters) == 1 def test_no_rapides_filter_when_both_false(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, + [], + must_not_rapides=False, + must_only_rapides=False, ) assert filters == [] assert should == [] def test_chiffre_clef_adds_filter(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, chiffre_clef=True, + [], + must_not_rapides=False, + must_only_rapides=False, + chiffre_clef=True, ) assert len(filters) == 1 def test_valid_theme_adds_filter(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, theme="Demographie", + [], + must_not_rapides=False, + must_only_rapides=False, + theme="Demographie", ) assert len(filters) == 1 def test_theme_all_ignored(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, theme="ALL", + [], + must_not_rapides=False, + must_only_rapides=False, + theme="ALL", ) assert filters == [] def test_unknown_theme_ignored(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, theme="NotATheme", + [], + must_not_rapides=False, + must_only_rapides=False, + theme="NotATheme", ) assert filters == [] def test_valid_geo_niveau(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, geo_niveau="COMMUNE", + [], + must_not_rapides=False, + must_only_rapides=False, + geo_niveau="COMMUNE", ) assert len(filters) == 1 def test_unknown_geo_niveau_ignored(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, geo_niveau="MARS", + [], + must_not_rapides=False, + must_only_rapides=False, + geo_niveau="MARS", ) assert filters == [] def test_geo_keyword_adds_two_should_clauses(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, geo_keyword="Paris", + [], + must_not_rapides=False, + must_only_rapides=False, + geo_keyword="Paris", ) assert len(should) == 2 def test_geo_keyword_all_ignored(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, geo_keyword="all", + [], + must_not_rapides=False, + must_only_rapides=False, + geo_keyword="all", ) assert should == [] def test_geo_keyword_all_case_insensitive(self): filters, should = apply_collection_filters( - [], must_not_rapides=False, must_only_rapides=False, geo_keyword="ALL", + [], + must_not_rapides=False, + must_only_rapides=False, + geo_keyword="ALL", ) assert should == [] diff --git a/tests/test_melodi_service.py b/tests/test_melodi_service.py index a27885c..14b69e7 100644 --- a/tests/test_melodi_service.py +++ b/tests/test_melodi_service.py @@ -1,4 +1,5 @@ """Unit tests for mcpdiffusion.services.melodi.""" + from __future__ import annotations import json @@ -33,6 +34,7 @@ # Helpers # --------------------------------------------------------------------------- + def _json_http_response(payload: dict, url: str = "https://api.test") -> httpx.Response: return httpx.Response( 200, @@ -59,6 +61,7 @@ def search(self, **kwargs): # get_melodi_observations # =================================================================== + class TestGetMelodiObservations: async def test_success(self): payload = {"observations": [{"v": 1}, {"v": 2}, {"v": 3}]} @@ -71,14 +74,17 @@ async def test_success(self): assert result.count == 3 async def test_year_filtering(self): - payload = {"observations": [ - {"dimensions": {"TIME_PERIOD": "2020-01"}, "v": 1}, - {"dimensions": {"TIME_PERIOD": "2021-06"}, "v": 2}, - {"dimensions": {"TIME_PERIOD": "2022-12"}, "v": 3}, - ]} + payload = { + "observations": [ + {"dimensions": {"TIME_PERIOD": "2020-01"}, "v": 1}, + {"dimensions": {"TIME_PERIOD": "2021-06"}, "v": 2}, + {"dimensions": {"TIME_PERIOD": "2022-12"}, "v": 3}, + ] + } fake = FakeAsyncClient(lambda url, **kw: _json_http_response(payload, url)) params = GetMelodiObservationsInput( - dataset_id="DS_TEST", list_of_year=[2020, 2022], + dataset_id="DS_TEST", + list_of_year=[2020, 2022], ) result = await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) @@ -101,7 +107,9 @@ def handler(url, **kw): params = GetMelodiObservationsInput(dataset_id="DS_TEST") with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): await get_melodi_observations( - params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + params, + http_client=FakeAsyncClient(handler), + settings=_SETTINGS, ) async def test_404_raises_tool_error(self): @@ -112,7 +120,9 @@ def handler(url, **kw): params = GetMelodiObservationsInput(dataset_id="DS_NONEXIST") with pytest.raises(ToolError, match="NOT_FOUND"): await get_melodi_observations( - params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + params, + http_client=FakeAsyncClient(handler), + settings=_SETTINGS, ) async def test_400_raises_tool_error(self): @@ -123,15 +133,20 @@ def handler(url, **kw): params = GetMelodiObservationsInput(dataset_id="DS_TEST") with pytest.raises(ToolError, match="INVALID_INPUT"): await get_melodi_observations( - params, http_client=FakeAsyncClient(handler), settings=_SETTINGS, + params, + http_client=FakeAsyncClient(handler), + settings=_SETTINGS, ) async def test_non_json_response_raises(self): - fake = FakeAsyncClient(lambda url, **kw: httpx.Response( - 200, content=b"not json", - headers={"content-type": "text/plain"}, - request=httpx.Request("GET", url), - )) + fake = FakeAsyncClient( + lambda url, **kw: httpx.Response( + 200, + content=b"not json", + headers={"content-type": "text/plain"}, + request=httpx.Request("GET", url), + ) + ) params = GetMelodiObservationsInput(dataset_id="DS_TEST") with pytest.raises(ToolError, match="PARSE_ERROR"): await get_melodi_observations(params, http_client=fake, settings=_SETTINGS) @@ -157,24 +172,31 @@ async def test_empty_year_filter_returns_all(self): # search_melodi_datasets # =================================================================== + class TestSearchMelodiDatasets: async def test_success(self): es_response = { - "hits": {"hits": [{ - "_id": "DS_IPC", - "_score": 10.5, - "_source": { - "columns": "COL1 Label1 | COL2 Label2", - "metadata": { - "description": {"content": "Price index", "lang": "fr"}, - }, - }, - }]}, + "hits": { + "hits": [ + { + "_id": "DS_IPC", + "_score": 10.5, + "_source": { + "columns": "COL1 Label1 | COL2 Label2", + "metadata": { + "description": {"content": "Price index", "lang": "fr"}, + }, + }, + } + ] + }, } params = SearchMelodiDatasetsInput(french_query="prix") result = await search_melodi_datasets( - params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + params, + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, ) assert len(result.results) == 1 @@ -198,18 +220,23 @@ async def test_es_connection_error_raises(self): async def test_description_list_takes_first(self): es_response = { - "hits": {"hits": [{ - "_id": "DS_1", "_score": 1.0, - "_source": { - "columns": "", - "metadata": { - "description": [ - {"content": "First", "lang": "fr"}, - {"content": "Second", "lang": "en"}, - ], - }, - }, - }]}, + "hits": { + "hits": [ + { + "_id": "DS_1", + "_score": 1.0, + "_source": { + "columns": "", + "metadata": { + "description": [ + {"content": "First", "lang": "fr"}, + {"content": "Second", "lang": "en"}, + ], + }, + }, + } + ] + }, } result = await search_melodi_datasets( SearchMelodiDatasetsInput(french_query="test"), @@ -220,10 +247,15 @@ async def test_description_list_takes_first(self): async def test_description_missing_defaults(self): es_response = { - "hits": {"hits": [{ - "_id": "DS_1", "_score": 1.0, - "_source": {"columns": "", "metadata": {}}, - }]}, + "hits": { + "hits": [ + { + "_id": "DS_1", + "_score": 1.0, + "_source": {"columns": "", "metadata": {}}, + } + ] + }, } result = await search_melodi_datasets( SearchMelodiDatasetsInput(french_query="test"), @@ -235,15 +267,20 @@ async def test_description_missing_defaults(self): async def test_description_dict_kept_as_is(self): es_response = { - "hits": {"hits": [{ - "_id": "DS_1", "_score": 1.0, - "_source": { - "columns": "", - "metadata": { - "description": {"content": "Direct dict", "lang": "en"}, - }, - }, - }]}, + "hits": { + "hits": [ + { + "_id": "DS_1", + "_score": 1.0, + "_source": { + "columns": "", + "metadata": { + "description": {"content": "Direct dict", "lang": "en"}, + }, + }, + } + ] + }, } result = await search_melodi_datasets( SearchMelodiDatasetsInput(french_query="test"), @@ -257,28 +294,43 @@ async def test_description_dict_kept_as_is(self): # search_melodi_modalities # =================================================================== + class TestSearchMelodiModalities: async def test_success_with_inner_hits(self): es_response = { - "hits": {"hits": [{ - "_source": {"code": "PRICES", "text": "Price types"}, - "inner_hits": { - "modalities": {"hits": {"hits": [{ - "_score": 5.0, - "_source": { - "code": "D", - "label": {"en": "Unit value", "fr": "Valeur unitaire"}, + "hits": { + "hits": [ + { + "_source": {"code": "PRICES", "text": "Price types"}, + "inner_hits": { + "modalities": { + "hits": { + "hits": [ + { + "_score": 5.0, + "_source": { + "code": "D", + "label": {"en": "Unit value", "fr": "Valeur unitaire"}, + }, + } + ] + } + }, }, - }]}}, - }, - }]}, + } + ] + }, } params = SearchMelodiModalitiesInput( - dataset_id="DS_IPC", columns_id=["PRICES"], french_query="prix", + dataset_id="DS_IPC", + columns_id=["PRICES"], + french_query="prix", ) result = await search_melodi_modalities( - params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + params, + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, ) assert len(result.results) == 1 @@ -291,7 +343,9 @@ async def test_success_with_inner_hits(self): async def test_empty_results_raises_tool_error(self): es = FakeElasticsearch(response={"hits": {"hits": []}}) params = SearchMelodiModalitiesInput( - dataset_id="DS_X", columns_id=["COL"], french_query="unknown", + dataset_id="DS_X", + columns_id=["COL"], + french_query="unknown", ) with pytest.raises(ToolError, match="EMPTY_RESULT"): @@ -300,7 +354,9 @@ async def test_empty_results_raises_tool_error(self): async def test_es_error_raises(self): es = FakeElasticsearch(error=ESConnectionError("down")) params = SearchMelodiModalitiesInput( - dataset_id="DS_X", columns_id=["COL"], french_query="test", + dataset_id="DS_X", + columns_id=["COL"], + french_query="test", ) with pytest.raises(ToolError, match="BACKEND_UNAVAILABLE"): @@ -308,17 +364,25 @@ async def test_es_error_raises(self): async def test_no_inner_hits_returns_empty_modalities(self): es_response = { - "hits": {"hits": [{ - "_source": {"code": "GEO", "text": "Geography"}, - "inner_hits": {"modalities": {"hits": {"hits": []}}}, - }]}, + "hits": { + "hits": [ + { + "_source": {"code": "GEO", "text": "Geography"}, + "inner_hits": {"modalities": {"hits": {"hits": []}}}, + } + ] + }, } params = SearchMelodiModalitiesInput( - dataset_id="DS_1", columns_id=["GEO"], french_query="france", + dataset_id="DS_1", + columns_id=["GEO"], + french_query="france", ) result = await search_melodi_modalities( - params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + params, + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, ) assert len(result.results) == 1 @@ -326,22 +390,36 @@ async def test_no_inner_hits_returns_empty_modalities(self): async def test_missing_label_defaults_to_empty(self): es_response = { - "hits": {"hits": [{ - "_source": {"code": "COL", "text": "Column"}, - "inner_hits": { - "modalities": {"hits": {"hits": [{ - "_score": 1.0, - "_source": {"code": "X", "label": None}, - }]}}, - }, - }]}, + "hits": { + "hits": [ + { + "_source": {"code": "COL", "text": "Column"}, + "inner_hits": { + "modalities": { + "hits": { + "hits": [ + { + "_score": 1.0, + "_source": {"code": "X", "label": None}, + } + ] + } + }, + }, + } + ] + }, } params = SearchMelodiModalitiesInput( - dataset_id="DS_1", columns_id=["COL"], french_query="test", + dataset_id="DS_1", + columns_id=["COL"], + french_query="test", ) result = await search_melodi_modalities( - params, es=FakeElasticsearch(response=es_response), settings=_SETTINGS, + params, + es=FakeElasticsearch(response=es_response), + settings=_SETTINGS, ) mod = result.results[0].matching_modalities[0] diff --git a/tests/test_middleware.py b/tests/test_middleware.py index bf7b864..c9b29c6 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,7 +1,8 @@ """Unit tests for mcpdiffusion.core.middleware (RateLimitMiddleware).""" + from __future__ import annotations -import pytest +from mcpdiffusion.core.middleware import RateLimitMiddleware from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import PlainTextResponse @@ -9,13 +10,12 @@ from starlette.testclient import TestClient from mcpdiffusion.config.settings import Settings -from mcpdiffusion.core.middleware import RateLimitMiddleware - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _make_app(rate_limit: int = 3) -> Starlette: """Create a minimal Starlette app with a DI-configured RateLimitMiddleware.""" @@ -36,6 +36,7 @@ async def homepage(request: Request) -> PlainTextResponse: # Tests # --------------------------------------------------------------------------- + class TestRateLimitAllowed: """Requests within the limit should pass through normally.""" @@ -109,10 +110,12 @@ def test_different_paths_share_same_counter(self): TZ="Europe/Paris", _env_file=None, ) - app = Starlette(routes=[ - Route("/a", lambda r: PlainTextResponse("a")), - Route("/b", lambda r: PlainTextResponse("b")), - ]) + app = Starlette( + routes=[ + Route("/a", lambda r: PlainTextResponse("a")), + Route("/b", lambda r: PlainTextResponse("b")), + ] + ) app.add_middleware(RateLimitMiddleware, settings=settings) client = TestClient(app) diff --git a/tests/test_rmes_service.py b/tests/test_rmes_service.py index 0ffb7f7..4dcc287 100644 --- a/tests/test_rmes_service.py +++ b/tests/test_rmes_service.py @@ -1,4 +1,5 @@ """Unit tests for mcpdiffusion.services.rmes (pure logic, no MCP layer).""" + from __future__ import annotations import time @@ -22,11 +23,11 @@ ) from tests.conftest import FakeAsyncClient, _json_response - # =================================================================== # _detect_query_form # =================================================================== + class TestDetectQueryForm: def test_select(self): assert _detect_query_form("SELECT ?s WHERE { ?s ?p ?o }") == "SELECT" @@ -53,10 +54,7 @@ def test_with_prefixes(self): assert _detect_query_form(query) == "SELECT" def test_prefix_containing_select_keyword(self): - query = ( - "PREFIX select: \n" - "ASK { ?s select:prop ?o }" - ) + query = "PREFIX select: \nASK { ?s select:prop ?o }" assert _detect_query_form(query) == "ASK" def test_unknown_form(self): @@ -73,6 +71,7 @@ def test_only_prefixes(self): # _ensure_limit # =================================================================== + class TestEnsureLimit: def test_adds_limit_to_select_without_limit(self): query = "SELECT ?s WHERE { ?s ?p ?o }" @@ -121,6 +120,7 @@ def test_case_insensitive_limit_detection(self): # _accept_header # =================================================================== + class TestAcceptHeader: def test_select_returns_json(self): assert _accept_header("SELECT") == "application/sparql-results+json" @@ -139,6 +139,7 @@ def test_describe_returns_turtle(self): # _relative_path # =================================================================== + class TestRelativePath: def test_strips_graph_base(self): assert _relative_path(f"{GRAPH_BASE}codes/naf2025") == "codes/naf2025" @@ -152,6 +153,7 @@ def test_returns_as_is_without_base(self): # _categorize # =================================================================== + class TestCategorize: def test_nomenclature(self): cat = _categorize(f"{GRAPH_BASE}codes/naf2025") @@ -225,6 +227,7 @@ def test_specific_rules_take_precedence(self): # _error_payload # =================================================================== + class TestErrorPayload: def test_basic_payload(self): result = _error_payload(SparqlErrorType.TIMEOUT, "timed out", "SELECT 1") @@ -235,7 +238,9 @@ def test_basic_payload(self): def test_extra_fields(self): result = _error_payload( - SparqlErrorType.SYNTAX_ERROR, "bad", "SELECT", + SparqlErrorType.SYNTAX_ERROR, + "bad", + "SELECT", endpoint_message="parse error at line 1", ) assert result["error"]["endpoint_message"] == "parse error at line 1" @@ -245,13 +250,17 @@ def test_extra_fields(self): # _execute_sparql (async, mocked HTTP via DI) # =================================================================== + class TestExecuteSparql: async def test_unknown_form_returns_error_without_http_call(self): called = [] fake = FakeAsyncClient(lambda url, **kw: called.append(1) or _json_response({})) result = await _execute_sparql( - "INSERT DATA {

}", timeout=10, max_rows=100, sparql_client=fake, + "INSERT DATA {

}", + timeout=10, + max_rows=100, + sparql_client=fake, ) assert "error" in result @@ -263,7 +272,10 @@ async def test_select_success(self): fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) result = await _execute_sparql( - "SELECT ?x WHERE { ?x ?p ?o } LIMIT 1", timeout=10, max_rows=100, sparql_client=fake, + "SELECT ?x WHERE { ?x ?p ?o } LIMIT 1", + timeout=10, + max_rows=100, + sparql_client=fake, ) assert "error" not in result @@ -274,7 +286,10 @@ async def test_select_without_limit_adds_meta(self): fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) result = await _execute_sparql( - "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=50, sparql_client=fake, + "SELECT ?x WHERE { ?x ?p ?o }", + timeout=10, + max_rows=50, + sparql_client=fake, ) assert result["_meta"]["limit_added"] == 50 @@ -282,13 +297,19 @@ async def test_select_without_limit_adds_meta(self): async def test_construct_returns_turtle(self): turtle = " ." - fake = FakeAsyncClient(lambda url, **kw: httpx.Response( - 200, content=turtle.encode(), headers={"content-type": "text/turtle"}, - request=httpx.Request("POST", url), - )) + fake = FakeAsyncClient( + lambda url, **kw: httpx.Response( + 200, + content=turtle.encode(), + headers={"content-type": "text/turtle"}, + request=httpx.Request("POST", url), + ) + ) result = await _execute_sparql( - "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1", timeout=10, max_rows=100, + "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1", + timeout=10, + max_rows=100, sparql_client=fake, ) @@ -302,7 +323,10 @@ def handler(url, **kw): fake = FakeAsyncClient(handler) result = await _execute_sparql( - "SELECT ?x WHERE { ?x ?p ?o }", timeout=5, max_rows=100, sparql_client=fake, + "SELECT ?x WHERE { ?x ?p ?o }", + timeout=5, + max_rows=100, + sparql_client=fake, ) assert result["error"]["type"] == SparqlErrorType.TIMEOUT @@ -327,7 +351,10 @@ def handler(url, **kw): fake = FakeAsyncClient(handler) result = await _execute_sparql( - "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100, sparql_client=fake, + "SELECT ?x WHERE { ?x ?p ?o }", + timeout=10, + max_rows=100, + sparql_client=fake, ) assert result["error"]["type"] == SparqlErrorType.HTTP_ERROR @@ -339,7 +366,10 @@ def handler(url, **kw): fake = FakeAsyncClient(handler) result = await _execute_sparql( - "SELECT ?x WHERE { ?x ?p ?o }", timeout=10, max_rows=100, sparql_client=fake, + "SELECT ?x WHERE { ?x ?p ?o }", + timeout=10, + max_rows=100, + sparql_client=fake, ) assert result["error"]["type"] == SparqlErrorType.NETWORK_ERROR @@ -349,6 +379,7 @@ def handler(url, **kw): # _get_raw_graph_rows (async, mocked HTTP + cache) # =================================================================== + class TestGetRawGraphRows: @pytest.fixture(autouse=True) def reset_cache(self): @@ -359,9 +390,11 @@ def reset_cache(self): async def test_returns_rows_on_success(self): body = { "head": {"vars": ["g", "nbTriples"]}, - "results": {"bindings": [ - {"g": {"value": "http://rdf.insee.fr/graphes/codes/naf2025"}, "nbTriples": {"value": "100"}}, - ]}, + "results": { + "bindings": [ + {"g": {"value": "http://rdf.insee.fr/graphes/codes/naf2025"}, "nbTriples": {"value": "100"}}, + ] + }, } fake = FakeAsyncClient(lambda url, **kw: _json_response(body)) @@ -386,9 +419,11 @@ async def test_uses_cache_on_second_call(self): call_count = [] body = { "head": {"vars": ["g", "nbTriples"]}, - "results": {"bindings": [ - {"g": {"value": "http://rdf.insee.fr/graphes/geo/cog"}, "nbTriples": {"value": "50"}}, - ]}, + "results": { + "bindings": [ + {"g": {"value": "http://rdf.insee.fr/graphes/geo/cog"}, "nbTriples": {"value": "50"}}, + ] + }, } def handler(url, **kw): @@ -406,9 +441,11 @@ def handler(url, **kw): async def test_cache_expires_after_ttl(self): body = { "head": {"vars": ["g", "nbTriples"]}, - "results": {"bindings": [ - {"g": {"value": "http://rdf.insee.fr/graphes/foo"}, "nbTriples": {"value": "1"}}, - ]}, + "results": { + "bindings": [ + {"g": {"value": "http://rdf.insee.fr/graphes/foo"}, "nbTriples": {"value": "1"}}, + ] + }, } call_count = [] diff --git a/tests/test_rmes_tools.py b/tests/test_rmes_tools.py index d212908..5d938fb 100644 --- a/tests/test_rmes_tools.py +++ b/tests/test_rmes_tools.py @@ -3,6 +3,7 @@ All HTTP calls to the real SPARQL endpoint are mocked via monkeypatch on `mcpdiffusion.infra.sparql.get_sparql_client`, so these tests run offline. """ + from __future__ import annotations import httpx @@ -10,11 +11,11 @@ from tests.conftest import _json_response, _out, _text_response - # =================================================================== # Tests: tool registration & discovery # =================================================================== + class TestToolDiscovery: async def test_three_rmes_tools_registered(self, rmes_client: Client): async with rmes_client: @@ -45,17 +46,17 @@ async def test_three_rmes_tools_registered(self, rmes_client: Client): class TestRunSparql: - async def test_select_query_returns_bindings( - self, rmes_client: Client, mock_sparql - ): + async def test_select_query_returns_bindings(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response(SPARQL_SELECT_RESPONSE)) async with rmes_client: raw = await rmes_client.call_tool( "RMES_run_sparql", - {"params": { - "full_sparql_query": "SELECT ?s ?label WHERE { ?s skos:prefLabel ?label } LIMIT 1", - }}, + { + "params": { + "full_sparql_query": "SELECT ?s ?label WHERE { ?s skos:prefLabel ?label } LIMIT 1", + } + }, ) result = _out(raw) @@ -64,27 +65,25 @@ async def test_select_query_returns_bindings( assert len(result["bindings"]) == 1 assert result["bindings"][0]["label"]["value"] == "Agriculture" - async def test_construct_query_returns_turtle( - self, rmes_client: Client, mock_sparql - ): + async def test_construct_query_returns_turtle(self, rmes_client: Client, mock_sparql): turtle_data = " ." mock_sparql(lambda url, **kw: _text_response(turtle_data)) async with rmes_client: raw = await rmes_client.call_tool( "RMES_run_sparql", - {"params": { - "full_sparql_query": "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1", - }}, + { + "params": { + "full_sparql_query": "CONSTRUCT { ?s ?p ?o } WHERE { ?s ?p ?o } LIMIT 1", + } + }, ) result = _out(raw) assert result["format"] == "turtle" assert "" in result["turtle"] - async def test_empty_query_returns_error( - self, rmes_client: Client, mock_sparql - ): + async def test_empty_query_returns_error(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response({})) async with rmes_client: @@ -97,9 +96,7 @@ async def test_empty_query_returns_error( assert result["error"] is not None assert result["error"]["type"] == "EMPTY_QUERY" - async def test_limit_auto_added_when_missing( - self, rmes_client: Client, mock_sparql - ): + async def test_limit_auto_added_when_missing(self, rmes_client: Client, mock_sparql): captured_queries = [] def handler(url, **kw): @@ -111,10 +108,12 @@ def handler(url, **kw): async with rmes_client: raw = await rmes_client.call_tool( "RMES_run_sparql", - {"params": { - "full_sparql_query": "SELECT ?s WHERE { ?s ?p ?o }", - "max_rows": 50, - }}, + { + "params": { + "full_sparql_query": "SELECT ?s WHERE { ?s ?p ?o }", + "max_rows": 50, + } + }, ) result = _out(raw) @@ -148,14 +147,13 @@ def handler(url, **kw): class TestListGraphs: - async def test_list_graphs_default( - self, rmes_client: Client, mock_sparql - ): + async def test_list_graphs_default(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response(GRAPH_LIST_SPARQL_RESPONSE)) async with rmes_client: raw = await rmes_client.call_tool( - "RMES_list_graphs", {"params": {}}, + "RMES_list_graphs", + {"params": {}}, ) result = _out(raw) @@ -165,9 +163,7 @@ async def test_list_graphs_default( assert "nomenclatures" in category_keys assert "geographie" in category_keys - async def test_list_graphs_filter_by_contains( - self, rmes_client: Client, mock_sparql - ): + async def test_list_graphs_filter_by_contains(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response(GRAPH_LIST_SPARQL_RESPONSE)) async with rmes_client: @@ -181,9 +177,7 @@ async def test_list_graphs_filter_by_contains( assert result["categories"][0]["category"] == "nomenclatures" assert result["categories"][0]["graphs"] is not None - async def test_list_graphs_filter_by_category( - self, rmes_client: Client, mock_sparql - ): + async def test_list_graphs_filter_by_category(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response(GRAPH_LIST_SPARQL_RESPONSE)) async with rmes_client: @@ -196,9 +190,7 @@ async def test_list_graphs_filter_by_category( assert result["total_graphs_matched"] == 1 assert all(c["category"] == "geographie" for c in result["categories"]) - async def test_list_graphs_sparql_error( - self, rmes_client: Client, mock_sparql - ): + async def test_list_graphs_sparql_error(self, rmes_client: Client, mock_sparql): def handler(url, **kw): raise httpx.TimeoutException("timed out") @@ -206,7 +198,8 @@ def handler(url, **kw): async with rmes_client: raw = await rmes_client.call_tool( - "RMES_list_graphs", {"params": {}}, + "RMES_list_graphs", + {"params": {}}, ) result = _out(raw) @@ -245,9 +238,7 @@ def handler(url, **kw): class TestDescribeResource: - async def test_describe_resource_returns_properties( - self, rmes_client: Client, mock_sparql - ): + async def test_describe_resource_returns_properties(self, rmes_client: Client, mock_sparql): mock_sparql(lambda url, **kw: _json_response(DESCRIBE_SPARQL_RESPONSE)) async with rmes_client: @@ -265,9 +256,7 @@ async def test_describe_resource_returns_properties( assert labels[0]["lang"] == "fr" assert labels[0]["direction"] == "outgoing" - async def test_describe_resource_with_graph_filter( - self, rmes_client: Client, mock_sparql - ): + async def test_describe_resource_with_graph_filter(self, rmes_client: Client, mock_sparql): captured_queries = [] def handler(url, **kw): @@ -279,18 +268,18 @@ def handler(url, **kw): async with rmes_client: await rmes_client.call_tool( "RMES_describe_resource", - {"params": { - "uri": "http://id.insee.fr/codes/naf2025/section/A", - "graph": "http://rdf.insee.fr/graphes/codes/naf2025", - }}, + { + "params": { + "uri": "http://id.insee.fr/codes/naf2025/section/A", + "graph": "http://rdf.insee.fr/graphes/codes/naf2025", + } + }, ) assert "VALUES ?g" in captured_queries[0] assert "codes/naf2025" in captured_queries[0] - async def test_describe_resource_sparql_error( - self, rmes_client: Client, mock_sparql - ): + async def test_describe_resource_sparql_error(self, rmes_client: Client, mock_sparql): def handler(url, **kw): raise httpx.TimeoutException("timed out") @@ -307,9 +296,7 @@ def handler(url, **kw): assert result["error"] is not None assert result["error"]["type"] == "TIMEOUT" - async def test_describe_resource_empty_result( - self, rmes_client: Client, mock_sparql - ): + async def test_describe_resource_empty_result(self, rmes_client: Client, mock_sparql): empty_response = { "head": {"vars": ["g", "direction", "p", "o"]}, "results": {"bindings": []}, diff --git a/uv.lock b/uv.lock index 41da271..b2a8f7d 100644 --- a/uv.lock +++ b/uv.lock @@ -350,6 +350,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.5.1" @@ -596,6 +605,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/1b/349e07ad184d64e81109e85a3557d7e05631fa3d05344169114ba743c4d3/dateparser-1.4.2-py3-none-any.whl", hash = "sha256:752f3d49d477cf7f60a7a9c8bcb19c882496ede0e377d5a3d80014cdfeca7050", size = 316546, upload-time = "2026-08-04T12:11:01.396Z" }, ] +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -738,6 +756,15 @@ server = [ { name = "websockets" }, ] +[[package]] +name = "filelock" +version = "3.32.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/a0/50c2c0ce5e74d7721bbb1b19a26ebd339aac5878553a6e35308c2f31f935/filelock-3.32.5.tar.gz", hash = "sha256:f6a6a28f743f9b95ce19db5abe0f376f75eb56517dff21e1a4751e2657d3e83d", size = 222838, upload-time = "2026-08-31T18:56:34.729Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/d2/b70a31e13d04456d28493f31d2aa087e99eeb2767ef0293b2625727ccb8c/filelock-3.32.5-py3-none-any.whl", hash = "sha256:142cd9fa77a872c5e78c62329a0d15278fadc686eb89e760017968961a4fd6b2", size = 100003, upload-time = "2026-08-31T18:56:33.078Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -928,6 +955,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + [[package]] name = "idna" version = "3.19" @@ -1277,8 +1313,10 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "ruff" }, ] [package.metadata] @@ -1298,8 +1336,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.1.1" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, + { name = "ruff", specifier = ">=0.15.4" }, ] [[package]] @@ -1419,6 +1459,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "openapi-pydantic" version = "0.5.1" @@ -1479,6 +1528,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pre-commit" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/89/1f3e8e1fc3e97de0fa963495832f581f025f29471602a309e48808244292/pre_commit-4.6.2.tar.gz", hash = "sha256:8f5d7bfb021ecdbcd9d49d89847082dd24172ccde534390081a679ad046e2441", size = 198670, upload-time = "2026-08-10T22:07:18.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, +] + [[package]] name = "propcache" version = "0.5.2" @@ -1789,6 +1854,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-discovery" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/96/0f93e27c9f60a650838f2118159aa115fd5732c0716247917b7ba7ede665/python_discovery-1.6.0.tar.gz", hash = "sha256:6393b4eae1be8b2182670635e7baff89ac21cb9f8e86fd1ff40c7b1144febb4c", size = 82849, upload-time = "2026-08-28T17:30:02.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/5e/21abf578182fb15006a57faf3711a1e659e29d600d19b6e557eae908c81d/python_discovery-1.6.0-py3-none-any.whl", hash = "sha256:d4e244cf17b8b29819ed78003d55fbacf86eda23425b075454fff9271b79377a", size = 38451, upload-time = "2026-08-28T17:30:01.236Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.3" @@ -2129,6 +2206,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, ] +[[package]] +name = "ruff" +version = "0.16.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/7c/6adb35d70e7c027e308274557901c7e00fb3407750faf3620c184ae058cb/ruff-0.16.6.tar.gz", hash = "sha256:dcf8a73d2ff77e99dde91244b4da16feba7f14e6beeb4015dee7c5a909e99050", size = 4921251, upload-time = "2026-09-03T16:57:29.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/28/9cc1b79639e284ec103f43c88c644db4eb58cbd0ea1ca11f1193435369ac/ruff-0.16.6-py3-none-linux_armv6l.whl", hash = "sha256:61c368c26bf8e973e5ab14a2772de587bc068ea3f9a277f673380749b4898fb8", size = 10015638, upload-time = "2026-09-03T16:56:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/627d342ef727ea7794edf74fe23d60a074b02c3acc2e9436684e782286ca/ruff-0.16.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ecf4f068e2e123e43a26e9db4e19524cc56563912404e83bbfca375757e45a32", size = 10220762, upload-time = "2026-09-03T16:56:44.681Z" }, + { url = "https://files.pythonhosted.org/packages/43/d9/b75668ce41e4c8d073d18d6d08672ba6906ce45d5c06ea4fdb2e84ce3853/ruff-0.16.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:99b62ea33baf130f50368798d841f0d95527b6d817bf31817b65dd058f1d314c", size = 9835082, upload-time = "2026-09-03T16:56:47.142Z" }, + { url = "https://files.pythonhosted.org/packages/99/97/123ab10b05cde889c107c20f5a9774955104b5552796a2a8584b089ae8eb/ruff-0.16.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fbf89013f2bb3f6835a6038ff658dc8a1b38c98dc8e724b964168ad4e881876", size = 9949304, upload-time = "2026-09-03T16:56:49.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/58/a4a2c59dd2e5b85929c912d9cac3056eb9ee8c7e75e9b9fe3e109174966b/ruff-0.16.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56a67065e22efa6bc4d498299d3bb06c0c90aace8fac2068b5a12f9dc4d8d51d", size = 9840612, upload-time = "2026-09-03T16:56:52.368Z" }, + { url = "https://files.pythonhosted.org/packages/61/6a/ff8c8626a786c4f49d48ced4a752dadbca65f5263005f9c2416578194694/ruff-0.16.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e25cc89174874b176a157e4428d66761c2c0c006654419bf384f967f361ff1b1", size = 10543465, upload-time = "2026-09-03T16:56:55.089Z" }, + { url = "https://files.pythonhosted.org/packages/ad/bb/c47535923365f337b82e28192e4e9eef2176511007cfd99a62fc22df5dad/ruff-0.16.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0700580ed5303723cb3c11c2f1d2a8913ce77b7ea86646dddb887f5417a9ba70", size = 11267576, upload-time = "2026-09-03T16:56:57.791Z" }, + { url = "https://files.pythonhosted.org/packages/ba/50/e5119a5212b5cd63b51e1f4b25e7bd636a6668fc069a3160b108ad7e3c16/ruff-0.16.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15f1d0b6e165a6e56567befb6629f8209271311d990bae0f37e6d065035ef5f3", size = 10781993, upload-time = "2026-09-03T16:57:00.666Z" }, + { url = "https://files.pythonhosted.org/packages/8b/98/083d8b4ef3c51a0d19db84367791cbe9f44e4b53343d19dfa83556e1cd9a/ruff-0.16.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d72c591a96986ee4268860e2b7235082129ca5e4cb9cbba653a4b57c11893757", size = 10317748, upload-time = "2026-09-03T16:57:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/9a/29/68f7ff2c5ad95f19f00627ac2de95644e25fe47371ea60b2db1fd952315e/ruff-0.16.6-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:65a006baa18f33324325814c864daef03541d51564b98c517610ea756ab7003e", size = 10540096, upload-time = "2026-09-03T16:57:06.182Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/79a8f6de85968641d68a7863aeec577551924ef066a990a48ff93167beab/ruff-0.16.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:cd02a7bf1a21a8735228a3e8c95a9dc5cf86bd2a52194f4aaae2a5755b4de0f4", size = 10100494, upload-time = "2026-09-03T16:57:09.194Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e8/b81a22d9b90c00b892ccf2fa2ac36fa95de4c13ab85aea3e73795cfe4651/ruff-0.16.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:31b36f1e5ad85e0737f09d2be4e512e2e283583c14015da3b9dc07359ac0fc88", size = 9843663, upload-time = "2026-09-03T16:57:12.168Z" }, + { url = "https://files.pythonhosted.org/packages/39/aa/54f516ec5e5a11c4afdceb1c454ebb054ffb96e4f4a1705580b4346abd35/ruff-0.16.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:61029b4ab4aa723fd3064fab96b1d814492596bf0c792679fffcbde1e1679953", size = 10282461, upload-time = "2026-09-03T16:57:15.077Z" }, + { url = "https://files.pythonhosted.org/packages/52/0b/38d0aa8aa32372b96dc44f97b22e576c4147808271aab7b2cb1e353d4445/ruff-0.16.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9ac8998457832c2061709d900856b7ad271dace0cb41f346588d540162bfa718", size = 10728808, upload-time = "2026-09-03T16:57:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e5/9e274e24eeb027640ffc7442f21239f16d17f47acec15ae34f32e03a5c79/ruff-0.16.6-py3-none-win32.whl", hash = "sha256:0b87d9d16fcb63e8018423ca1d50b7260f15cb2da33e30db4baad4183a948c25", size = 10049212, upload-time = "2026-09-03T16:57:20.55Z" }, + { url = "https://files.pythonhosted.org/packages/22/31/72472449414223ed1a2da236b992adbb1a2ae59e34794574810f60ce068e/ruff-0.16.6-py3-none-win_amd64.whl", hash = "sha256:10d21c51c3495d8eaea7b703a16592117ea6eb1d649e36335aa965ff1173eb39", size = 10556402, upload-time = "2026-09-03T16:57:23.501Z" }, + { url = "https://files.pythonhosted.org/packages/fc/07/d781f8f8e1ac24bef9f3269cf62ffb1407ca24c3a8f12e5e22874f90528c/ruff-0.16.6-py3-none-win_arm64.whl", hash = "sha256:7a976c79b958f94e50a022a19f0f8c87387448020935ec14fc74331bd0a7f2c5", size = 10412850, upload-time = "2026-09-03T16:57:26.416Z" }, +] + [[package]] name = "secretstorage" version = "3.5.0" @@ -2304,6 +2406,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, ] +[[package]] +name = "virtualenv" +version = "21.7.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/1c/69faa2e6a83484e2a8227bce5cfaa183941c5720f99c48f204931d286b07/virtualenv-21.7.8.tar.gz", hash = "sha256:1dc49c790072a9072cb1803f9bd62aa69cd583077cada32390f75505cdc64c9b", size = 5347580, upload-time = "2026-09-01T13:36:13.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/34/88d507d4a4030fa559788de9c690a214f9a4053aa1d91cfb60e9b36127c2/virtualenv-21.7.8-py3-none-any.whl", hash = "sha256:3040eb3cbf5d32b10ffd57d167e6a162237ad82ba7d8cf1400a1efed593d85ac", size = 5324617, upload-time = "2026-09-01T13:36:11.248Z" }, +] + [[package]] name = "watchfiles" version = "1.2.0" From a220556e214ba4ad1b39d7fca7ad24b27219f5da Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Fri, 4 Sep 2026 12:20:42 +0200 Subject: [PATCH 15/55] docs(core): reconcile the error rules with the code error.md still told the reader to write `raise ... from exc` after we removed every one of them and ignored B904. It now says not to, and why, so nobody adds them back believing the ignore was an oversight. It also read as forbidding the Elasticsearch client's own max_retries and retry_on_timeout, which RetryMiddleware does not replace: one retries the HTTP connection, the other the MCP request. The rule now separates them. uvicorn moves to module scope. It is a declared dependency, so importing it inside __main__ bought no isolation and was the only import in a function body, against python.md. --- .claude/rules/error.md | 9 +++++---- src/mcpdiffusion/server.py | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.claude/rules/error.md b/.claude/rules/error.md index d8ad9b9..9535c09 100644 --- a/.claude/rules/error.md +++ b/.claude/rules/error.md @@ -30,9 +30,9 @@ step — the offending parameter, or the tool that produces a valid value. Never (it swallows `CancelledError`). - Translate once, at the boundary owning the dependency. Never re-wrap an already typed error. - Never swallow: no empty `except`, no default on failure, no log-and-continue. -- Chain with `raise ... from exc`. Server-side hygiene only: `__cause__` never crosses the wire, so it - leaks nothing, and Python keeps the original either way — this just states that it was the cause rather - than an error raised while handling one. +- Do not write `raise ... from exc`. Python keeps the original as `__context__` either way, so the cause + is in the traceback regardless; chaining only changes the wording. `ErrorHandlingMiddleware` runs with + `include_traceback=True`, which is what actually puts the cause in the log. B904 is ignored for this. ## Use what FastMCP provides @@ -41,4 +41,5 @@ step — the offending parameter, or the tool that produces a valid value. Never generic message; `ToolError` subclasses keep theirs. - `ErrorHandlingMiddleware` catches, logs and converts every exception. Register it first, so it sees the rest of the chain. Failures are logged there, not by the code that raises — see `logging.md`. -- `RetryMiddleware` handles transient failures with backoff. Do not write a retry loop. +- `RetryMiddleware` handles transient failures with backoff. Do not write a retry loop. A client's own + retry settings are different and stay where they are — the Elasticsearch client retries internally. diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 9d6baa3..58eccc1 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -2,6 +2,7 @@ import logging +import uvicorn from fastmcp import FastMCP from fastmcp.server.middleware.error_handling import ErrorHandlingMiddleware from fastmcp.server.middleware.logging import LoggingMiddleware @@ -74,8 +75,6 @@ ) if __name__ == "__main__": - import uvicorn - uvicorn.run( app, host=settings.mcp_host, From ff84d82154242c7e82cd9628150e48547673b1b6 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Fri, 4 Sep 2026 12:45:04 +0200 Subject: [PATCH 16/55] fix(insee): describe what get_insee_homepage actually returns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The description promised `mainIndicators` with a per-indicator link to pass to `get_insee_document`, plus `lastArticles` and `keyGraphics`. The tool returns `indicators` and `count` and no links at all, so the model was instructed to perform a chain it could not perform, on every call. get_insee_document pointed at those same links, and now points at the search tools instead. Cleanup around it, behaviour unchanged at 59 indicators: - DICT_KV becomes KEY_INDICATORS, which is python.md's own example of the distinction - the header row {"cle": "clé", ...} is deleted along with the runtime filter that removed it by comparing three literals on every call - trailing whitespace lives in the data rather than in a .strip() per read - the mapping moves to services/insee_indicators.py, so it is testable without an MCP server - the tool is sync: it reads a literal and performs no I/O The figures themselves are untouched. They are frozen literals that assert their own dates, which is a decision for the data owners, so it is recorded as a `# Business rule:` along with the promised-but-never-built output shape - the best evidence of what the tool was originally meant to be. --- src/mcpdiffusion/config/tool_metadata.py | 19 ++++++------- src/mcpdiffusion/data/indicators.py | 22 ++++++++------- src/mcpdiffusion/services/insee_indicators.py | 18 +++++++++++++ src/mcpdiffusion/tools/insee_get_homepage.py | 27 ++++--------------- 4 files changed, 46 insertions(+), 40 deletions(-) create mode 100644 src/mcpdiffusion/services/insee_indicators.py diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py index b587082..a48a052 100644 --- a/src/mcpdiffusion/config/tool_metadata.py +++ b/src/mcpdiffusion/config/tool_metadata.py @@ -130,8 +130,8 @@ def compute_current_date_iso() -> str: "tool_description": ( "Fetch and parse a single INSEE publication from a known URL and " "return its full text in markdown. Use ONLY when you already have one " - "or more explicit URLs (e.g. from `search_insee_documents` or from " - "the `link` fields returned by `get_insee_homepage`).\n" + "or more explicit URLs, from `search_insee_documents`, " + "`search_insee_conjoncture` or `search_insee_chiffrecle`.\n" "\n" "WHEN TO USE\n" "- You have a concrete URL of the form `/fr/statistiques/` or " @@ -267,6 +267,9 @@ def compute_current_date_iso() -> str: "tool_metadata": {"version": "5.0", "author": "mirlon"}, } +# Business rule: this description calls the figures "latest" and makes the tool the preferred FIRST step, +# but they are frozen literals (see data/indicators.py). Whether the wording softens or the data becomes +# live is the same decision. Left as-is deliberately. GET_HOMEPAGE = { "tool_name": "get_insee_homepage", "tool_description": ( @@ -283,17 +286,15 @@ def compute_current_date_iso() -> str: "or `search_insee_conjoncture` with `year_of_reference`.\n" "\n" "OUTPUT\n" - "- `mainIndicators` -- each with name, value, description and a link " - "to the underlying official product (pass the link to `get_insee_document`).\n" - "- `lastArticles` -- recent short articles with title, date, " - "collection and link.\n" - "- `keyGraphics` -- selection of recent graphical publications.\n" + "- `indicators` -- each with `key` (indicator name), `alias` (alternative name, often empty) " + "and `value`, a full sentence in French stating the figure and the period it covers.\n" + "- `count` -- number of indicators returned.\n" "\n" "WORKFLOW\n" "1. Call this tool.\n" - "2. Present the indicator value + description + link.\n" + "2. Present the indicator value, quoting the period it states.\n" "3. Follow up with `search_insee_documents` or `search_insee_conjoncture` " - "only if the user needs deeper tables or historic series.\n" + "if the user needs deeper tables, historic series, or a source document.\n" "\n" f"Current date is {compute_current_date_iso()}.\n" ), diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/indicators.py index 1b919b0..5435b73 100644 --- a/src/mcpdiffusion/data/indicators.py +++ b/src/mcpdiffusion/data/indicators.py @@ -1,10 +1,14 @@ """Curated INSEE key indicators (homepage data).""" -# Fixme: I am wondering whether this really belongs in the source code or in a separate file or database -# Fixme: The 1st entry seems like a header (contains no real data), is that normal? -# Fixme: This seems like hardcoded, stale statistics... I don't know if this is normal -DICT_KV = [ - {"cle": "clé", "alias": "alias", "valeur": "valeur"}, +# Business rule: these figures are frozen literals — nothing refreshes them, so the server reports whatever +# was true when this file was last edited, while each sentence asserts its own date. Whether to fetch +# insee.fr live, derive them from the Elasticsearch index, or keep a curated list with a visible +# "last updated" is a decision for the data owners. Behaviour preserved until then. +# +# The tool description used to promise `mainIndicators` with a per-indicator link to pass to +# `get_insee_document`, plus `lastArticles` and `keyGraphics`. None of that was ever produced. It is +# recorded here because it says what the tool was meant to be, and is worth raising in that decision. +KEY_INDICATORS = [ { "cle": "estimation de population France", "alias": "", @@ -62,11 +66,11 @@ }, { "cle": "inflation", - "alias": "Indice des prix à la consommation – IPC ", + "alias": "Indice des prix à la consommation – IPC", "valeur": "En juin 2026, les prix à la consommation (IPC) augmentent de 1,8 % sur un an. Sur un mois, l'indice des prix à la consommation diminue de 0,3 %.", }, { - "cle": "Chômage BIT ", + "cle": "Chômage BIT", "alias": "", "valeur": "Au premier trimestre 2026, le taux de chômage en France (hors Mayotte) augmente de 0,2 point et atteint 8,1 % . Le nombre de chômeurs est de 2,6 millions de personnes.", }, @@ -118,7 +122,7 @@ { "cle": "balance commerciale", "alias": "", - "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l'activité en 2025, à hauteur de -0,2 point de PIB, après l'avoir fortement soutenue en 2023 et 2024. ", + "valeur": "En 2025, les exportations en volume restent soutenues (+2,3 % après +3,2 % en 2024), tandis que les importations se redressent nettement (+2,8 % après -0,6 %). De ce fait, les échanges extérieurs pèsent sur la croissance de l'activité en 2025, à hauteur de -0,2 point de PIB, après l'avoir fortement soutenue en 2023 et 2024.", }, { "cle": "pauvreté monétaire", @@ -128,7 +132,7 @@ { "cle": "patrimoine", "alias": "", - "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine. ", + "valeur": "Début 2024, la moitié des ménages vivant en France déclarent un patrimoine brut supérieur à 205 100 euros. La moitié la mieux dotée en patrimoine brut possède collectivement 93 % de la masse totale de patrimoine.", }, { "cle": "état santé", diff --git a/src/mcpdiffusion/services/insee_indicators.py b/src/mcpdiffusion/services/insee_indicators.py new file mode 100644 index 0000000..ed88bf3 --- /dev/null +++ b/src/mcpdiffusion/services/insee_indicators.py @@ -0,0 +1,18 @@ +"""Business logic for the INSEE key indicators tool.""" + +from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator + + +def build_key_indicators(entries: list[dict[str, str]]) -> KeyIndicatorsOutput: + indicators = [ + KeyValueIndicator( + key=entry["cle"], + alias=entry["alias"], + value=entry["valeur"], + ) + for entry in entries + ] + return KeyIndicatorsOutput( + indicators=indicators, + count=len(indicators), + ) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index 9941e35..fd1b74f 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -1,35 +1,18 @@ """Tool: get_insee_homepage -- thin registration layer.""" -from __future__ import annotations - from fastmcp import FastMCP from ..config.tool_metadata import GET_HOMEPAGE -from ..data.indicators import DICT_KV -from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator +from ..data.indicators import KEY_INDICATORS +from ..models.insee import KeyIndicatorsOutput +from ..services.insee_indicators import build_key_indicators -# Fixme: the tool contains no service which is kind of breaking the convention I saw earlier -# this correlates unit testing to the tool mechanics def register_get_insee_homepage(mcp: FastMCP) -> None: @mcp.tool( name=GET_HOMEPAGE["tool_name"], description=GET_HOMEPAGE["tool_description"], meta=GET_HOMEPAGE["tool_metadata"], ) - # Fixme: this is an async function with nothing to await - async def get_insee_homepage() -> KeyIndicatorsOutput: - indicators = [ - KeyValueIndicator( - key=entry["cle"].strip(), - alias=entry["alias"].strip(), - value=entry["valeur"].strip(), - ) - for entry in DICT_KV - if not ( - entry["cle"].strip() == "clé" - and entry["alias"].strip() == "alias" - and entry["valeur"].strip() == "valeur" - ) - ] - return KeyIndicatorsOutput(indicators=indicators, count=len(indicators)) + def get_insee_homepage() -> KeyIndicatorsOutput: + return build_key_indicators(KEY_INDICATORS) From cdf44df1f677937c31eafd2b9bde8a3e0dc71351 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sat, 5 Sep 2026 00:10:32 +0200 Subject: [PATCH 17/55] refactor!: name every tool, parameter and schema after what it is Tool descriptions came from a separate `config/tool_metadata.py`, so they drifted from the code they described: five routing hints pointed at tools that never existed (`query_insee_rmes`, `search_chiffres_clefs_insee`, `get_MELODI_datasets`) and `search_insee_documents` documented a `chiffre_clef` parameter its signature had lost. Descriptions now live in the docstring beside the function, so a rename cannot leave the prose behind. Cross-tool routing moved to the server `instructions`, assembled from the enabled families -- a deployment with `ENABLE_RMES_TOOLS=false` no longer advertises RMES workflows. Per-tool guidance keeps its WHEN TO USE / WHEN NOT TO USE shape. Tool parameters are flat: `params: XxxInput` nested every argument one level deep for no gain. The Input models became reusable `Annotated` aliases, which also removed the duplication four `# Fixme:` markers asked about. Names now say what a value is, not what the caller does with it -- `ResultCount` became `NumberOfResults`, `PublicationYearFilter` became `YearOfReference`, and enums took the `Choice` suffix so the plain name was free for the alias. Every parameter description, default, bound, example and enum is byte-identical to before; only names and nesting changed. BREAKING CHANGE: three tools are renamed and several parameters with them. Connected clients calling the old names fail. RMES_list_graphs -> search_rmes_graphs contains / category / expand -> graph_uri_substring / graph_category / expand_graphs RMES_describe_resource -> describe_rmes_resource uri / graph -> resource_uri / graph_uri RMES_run_sparql -> run_rmes_sparql full_sparql_query / timeout -> sparql_query / timeout_seconds search_insee_* geo_niveau -> geo_level get_insee_document list_of_url -> document_urls include_sommaire -> include_table_of_contents get_melodi_observations list_of_year -> years dict_of_columns_and_values -> column_filters number_of_results -> number_of_observations search_melodi_datasets french_query -> query number_of_results -> number_of_datasets search_melodi_modalities columns_id -> column_ids french_query -> query number_of_results -> number_of_modalities --- src/mcpdiffusion/config/tool_metadata.py | 385 ------------------ src/mcpdiffusion/core/instructions.py | 186 +++++++++ src/mcpdiffusion/data/indicators.py | 1 + src/mcpdiffusion/models/feedback.py | 41 +- src/mcpdiffusion/models/insee.py | 272 +++++++------ src/mcpdiffusion/models/melodi.py | 175 ++++---- src/mcpdiffusion/models/rmes.py | 155 ++++--- src/mcpdiffusion/server.py | 8 + src/mcpdiffusion/services/feedback.py | 20 +- src/mcpdiffusion/services/insee_document.py | 23 +- src/mcpdiffusion/services/insee_search.py | 6 +- src/mcpdiffusion/services/melodi.py | 90 ++-- src/mcpdiffusion/services/rmes.py | 125 +++--- src/mcpdiffusion/tools/__init__.py | 18 +- .../tools/extras_send_feedback.py | 21 - src/mcpdiffusion/tools/feedback_send.py | 26 ++ src/mcpdiffusion/tools/insee_get_document.py | 37 +- src/mcpdiffusion/tools/insee_get_homepage.py | 16 +- .../tools/insee_search_chiffrecle.py | 46 ++- .../tools/insee_search_conjoncture.py | 43 +- .../tools/insee_search_documents.py | 49 ++- .../tools/melodi_get_observations.py | 42 +- .../tools/melodi_search_datasets.py | 40 +- .../tools/melodi_search_modalities.py | 40 +- .../tools/rmes_describe_resource.py | 32 +- src/mcpdiffusion/tools/rmes_list_graphs.py | 24 -- src/mcpdiffusion/tools/rmes_run_sparql.py | 93 +++-- src/mcpdiffusion/tools/rmes_search_graphs.py | 40 ++ 28 files changed, 1055 insertions(+), 999 deletions(-) delete mode 100644 src/mcpdiffusion/config/tool_metadata.py create mode 100644 src/mcpdiffusion/core/instructions.py delete mode 100644 src/mcpdiffusion/tools/extras_send_feedback.py create mode 100644 src/mcpdiffusion/tools/feedback_send.py delete mode 100644 src/mcpdiffusion/tools/rmes_list_graphs.py create mode 100644 src/mcpdiffusion/tools/rmes_search_graphs.py diff --git a/src/mcpdiffusion/config/tool_metadata.py b/src/mcpdiffusion/config/tool_metadata.py deleted file mode 100644 index a48a052..0000000 --- a/src/mcpdiffusion/config/tool_metadata.py +++ /dev/null @@ -1,385 +0,0 @@ -"""Tool metadata (name, description, version). - -Design notes: -- Tool *names* are English snake_case; French is kept only where it is - actual data (enum literals that hit the ES index, user-supplied queries). - Fixme: the current is computed only once at import time, - if the server is running for 3 weeks, the old date will still be in effect... -- `CURRENT_DATE` is computed lazily so long-running servers always report - today's date, not the day the process started. -- Tool descriptions describe the *final* schemas; rewrite in lockstep - when schemas change. -""" - -from datetime import date - - -# Fixme: I'd put such a generic function in a separate module -# just a preference, not mandatory -def compute_current_date_iso() -> str: - """Return today's date as ISO-8601.""" - return date.today().isoformat() - - -# Fixme: I feel like having the tools metadata separate / not-colocated with the tools might be a mistake -# Fixme: I believe those metadata can live with the corresponding set of tool functions -# by leveraging FastMCP capabilities - descriptions could live in the functions' docstring -# Fixme: I would also create one file per tool as a preference, but I get the argument -# of having a clear overview of tools at the same place -# --- MELODI tools ----------------------------------------------------------- - -# Fixme: if keeping those metadata separate, prefer multiline strings that read better -# and avoid missing spaces issues -GET_DATASET = { - "tool_name": "get_melodi_observations", - "tool_description": ( - "Retrieve a filtered set of observations from a Melodi dataset. " - "The Melodi API holds official, high-granularity statistics " - "(prices, mortality, names, etc.).\n" - "\n" - "WHEN TO USE\n" - "- You already know the exact `dataset_id` (from `search_melodi_datasets`) " - "AND the modality codes you want to filter on " - "(from `search_melodi_modalities`).\n" - "\n" - "WHEN NOT TO USE\n" - "- You are still looking for the right dataset. Use `search_melodi_datasets` first.\n" - "- You need concept definitions or code-list vocabularies. Use `query_insee_rmes`.\n" - "\n" - "WORKFLOW (chain with companion tools)\n" - "1. `search_melodi_datasets` -> dataset_id + column ids\n" - "2. `search_melodi_modalities` -> exact modality codes for filtering\n" - "3. THIS TOOL (`get_melodi_observations`) -> final observations\n" - "\n" - "OUTPUT\n" - "A list of observations with dimensions, attributes and the numeric " - "measure (with unit). Returns an empty list when no rows match; " - "a structured error when the upstream API fails or inputs are invalid.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_DATASET = { - "tool_name": "search_melodi_datasets", - "tool_description": ( - "Search the INSEE Melodi dataset catalogue by French-language natural " - "language query. Each dataset has a unique `dataset_id`; the tool maps " - "the query to internal metadata to return the most relevant matches.\n" - "\n" - "WHEN TO USE\n" - "- The user asks for a specific statistic (price of a product, " - "mortality by region, frequency of a name, etc.) and you need to " - "locate the right dataset before fetching rows.\n" - "\n" - "WHEN NOT TO USE\n" - "- Generic, up-to-date indicator questions (use `get_insee_homepage`).\n" - "- Full-text analysis of a published report (use `search_insee_documents`).\n" - "- Definition/ontology lookups (use `query_insee_rmes`).\n" - "\n" - "TIPS\n" - "- Matching is lexical. Make `french_query` explicit and rich in French " - 'synonyms: e.g. `"indice des prix a la consommation"`, ' - '`"deces par departement"`, `"prenoms des nouveau-nes"`.\n' - "- Use `start_year` / `end_year` to narrow the temporal range. Leaving " - "both at default covers all years.\n" - "\n" - "NEXT STEP\n" - "Pass the returned `dataset_id` and column ids to " - "`search_melodi_modalities`, then feed the resolved codes into " - "`get_melodi_observations`.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_MODALITIES = { - "tool_name": "search_melodi_modalities", - "tool_description": ( - "Given a Melodi dataset and one or more column identifiers, rank the " - "most relevant modalities (codes/labels) for a free-text French query. " - "The result is what you need to filter rows in `get_melodi_observations`.\n" - "\n" - "WHEN TO USE\n" - "- You have a `dataset_id` (from `search_melodi_datasets`) and want " - "to find the exact modality code for a concept like `cote de boeuf`, " - "`Ile-de-France`, or `female Maria`.\n" - "\n" - "WHEN NOT TO USE\n" - "- You don't yet know the dataset. Run `search_melodi_datasets` first.\n" - "\n" - "INPUT\n" - "- `dataset_id` -- from a previous search result.\n" - '- `columns_id` -- which columns to search (e.g. `["PRICES", "GEO"]`).\n' - "- `french_query` -- natural-language query in French.\n" - "\n" - "OUTPUT\n" - "A list of matching columns, each containing its `code`, metadata text " - "and the top-scoring `matching_modalities` with `code`, `label_fr`, " - "`label_en` and `score`. Empty list when nothing matches.\n" - "\n" - "NEXT STEP\n" - "Use the modality `code` values as entries in " - "`get_melodi_observations.dict_of_columns_and_values`.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# --- INSEE.fr tools --------------------------------------------------------- - -GET_DOCUMENT = { - "tool_name": "get_insee_document", - "tool_description": ( - "Fetch and parse a single INSEE publication from a known URL and " - "return its full text in markdown. Use ONLY when you already have one " - "or more explicit URLs, from `search_insee_documents`, " - "`search_insee_conjoncture` or `search_insee_chiffrecle`.\n" - "\n" - "WHEN TO USE\n" - "- You have a concrete URL of the form `/fr/statistiques/` or " - "`/fr/statistiques/?sommaire=`.\n" - "\n" - "WHEN NOT TO USE\n" - "- You are still looking for the right publication. Use " - "`search_insee_documents` first.\n" - "- You need a quick, up-to-date indicator. Use `get_insee_homepage`.\n" - "\n" - "INPUT\n" - "- `list_of_url` -- list of relative URLs to fetch (e.g. " - '`["/fr/statistiques/4277658?sommaire=4318291"]`).\n' - "- `include_sommaire` -- also parse the page's table-of-contents " - "section. Use once to discover the structure of a multi-section " - "publication, then turn it off for subsequent requests on the same page.\n" - "- `truncate_content` -- when True (default), long markdown bodies are " - "clipped to keep the response compact for the model; set to False only " - "when you genuinely need the full text.\n" - "\n" - "OUTPUT\n" - "A uniform envelope: `{ status, results: [ { id, status, " - "markdown_content, sommaire, error, truncated } ], count }`. Each " - "per-URL entry has the same keys whether it succeeded or failed, so " - "downstream code can iterate without type-sniffing.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# Fixme: Those docstrings are computed once at import time, therefore, the current date is never re-computed -SEARCH_DOCUMENTS = { - "tool_name": "search_insee_documents", - "tool_description": ( - "Search the INSEE catalogue of official statistical publications " - "(Insee Premiere, Insee Analyses, Dossiers, References, Focus, ...). " - "Returns structured publication records; pass the URL of a record to " - "`get_insee_document` to fetch the full text.\n" - "\n" - "ROUTING PRIORITY\n" - "- Simple statistics (population, inflation, chomage, PIB, salaires) " - "by region/department? -> Use `search_chiffres_clefs_insee` FIRST.\n" - "- Granular product data (e.g., beef rib price 2000)? -> Use " - "`search_melodi_datasets` FIRST.\n" - "- This tool is for ANALYSIS, CONTEXT, and COMPLEX NARRATIVES.\n" - "\n" - "WHEN TO USE THIS TOOL\n" - "- Impact analyses (e.g., 'covid effects on tourism').\n" - "- Historical evolution and trends (e.g., 'unemployment 1990-2026').\n" - "- Detailed methodological or definitional content.\n" - "- Regional/departmental profiles with socioeconomic context.\n" - "- Specific thematic deep-dives (demography, labour market, inequalities, " - "environment, housing, ...). \n" - "- Comparative studies or cross-cutting analyses.\n" - "\n" - "WHEN NOT TO USE THIS TOOL\n" - "- Simple factual questions ('What is X region's population?') -> " - "`search_chiffres_clefs_insee`.\n" - "- Quick, up-to-date headline indicators -> `get_insee_homepage`.\n" - "- Latest monthly/quarterly rapid releases -> `search_insee_conjoncture`.\n" - "- Vocabulary / code definitions / classifications -> `query_insee_rmes`.\n" - "- Granular historical time series (product prices, individual wages) -> " - "`search_melodi_datasets`.\n" - "\n" - "HOW TO SEARCH WELL\n" - "- `query` -- rich natural-language query with synonyms, context, " - "and target year/geography if relevant.\n" - "- `chiffre_clef=False` (default) -- general publications. Set to True " - "ONLY for 'essentials sur...' publications (essentiel sur l'inflation, " - "etc.), but prefer `search_chiffres_clefs_insee` for those instead.\n" - "- `geo_niveau` + `geo_keyword` -- territorial filtering " - "(COM/DEP/REG/INTER/COMPRD/FRANCE).\n" - "- `theme` -- restrict to top-level theme (Demographie, " - "Marche du travail, Economie, etc.). Default ALL.\n" - "- `year_of_reference` -- hard filter on publication year; null = all years.\n" - "\n" - "OUTPUT\n" - "List of publications: `{ id, score, titre, soustitre, chapo, " - "anneediffusion, zone, theme, url }`. Feed `url` to `get_insee_document`.\n" - "\n" - f"Current date is {compute_current_date_iso()}.\n" - ), - "tool_metadata": {"version": "6.0", "author": "mirlon"}, -} - -SEARCH_CHIFFRECLEF = { - "tool_name": "search_insee_chiffrecle", - "tool_description": "Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, \n" - "comparaisons regionales/departementales et statistiques factuelles simples.\n" - "A utiliser EN PRIORITE pour : population, inflation, chomage, PIB, salaires, \n" - "prix par categorie, comparaisons geographiques (region, departement, commune).\n" - "A utiliser POUR LES CAS SIMPLES : 'Quelle est la population de X ?', " - "'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?'\n" - "A NE PAS utiliser pour : analyses detaillees, impacts/contexte, tendances \n" - "complexes, donnees produit granulaires historiques (-> utiliser search_melodi_datasets \n" - "ou search_insee_documents selon le contexte).\n" - "Retourne directement les tableaux synthetiques prets a l'emploi.\n", - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEARCH_CONJONCTURE = { - "tool_name": "search_insee_conjoncture", - "tool_description": ( - "Search INSEE Rapid Releases (Informations rapides): short, recurring " - "publications reporting the latest monthly/quarterly/annual results for " - "major economic and social indicators (prices, employment, production, " - "housing, wages, national accounts, ...).\n" - "\n" - "WHEN TO USE\n" - "- The user asks for the *latest* monthly/quarterly release of a " - "named indicator (e.g. last month's consumer confidence, " - "last quarter's GDP estimate). Prefer the most recent edition.\n" - "\n" - "WHEN NOT TO USE\n" - "- Generic up-to-date indicator on the homepage: `get_insee_homepage`.\n" - "- Deep, peer-reviewed analysis: `search_insee_documents`.\n" - "\n" - "HOW TO SEARCH WELL\n" - "- `query` -- provide several synonyms and related notions; the " - "search is lexical and rewards keyword breadth.\n" - "- `theme_conjoncture` -- optional broad category (Industrial " - "production and activity, Inflation and producer prices, " - "Employment, unemployment and labour market, ...). Leave null to " - "search across all categories.\n" - "- `year_of_reference` -- hard filter on publication year; leave null " - "to search all years.\n" - "\n" - "OUTPUT\n" - "A list of publications: `{ id, score, titre, soustitre, chapo, " - "anneediffusion, zone, theme, url }`.\n" - "\n" - f"Current date is {compute_current_date_iso()}.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# Business rule: this description calls the figures "latest" and makes the tool the preferred FIRST step, -# but they are frozen literals (see data/indicators.py). Whether the wording softens or the data becomes -# live is the same decision. Left as-is deliberately. -GET_HOMEPAGE = { - "tool_name": "get_insee_homepage", - "tool_description": ( - "Retrieve the INSEE home page with the latest key indicators at national level" - "published by the institute (population, inflation, unemployment, " - "GDP growth, ...).\n" - "\n" - "WHEN TO USE -- preferred FIRST step for any generic, up-to-date " - "statistical question. It gives the most recent official figure " - "instantly, without searching individual documents.\n" - "\n" - "WHEN NOT TO USE\n" - "- User asks for a previous year's figure. Use `search_insee_documents` " - "or `search_insee_conjoncture` with `year_of_reference`.\n" - "\n" - "OUTPUT\n" - "- `indicators` -- each with `key` (indicator name), `alias` (alternative name, often empty) " - "and `value`, a full sentence in French stating the figure and the period it covers.\n" - "- `count` -- number of indicators returned.\n" - "\n" - "WORKFLOW\n" - "1. Call this tool.\n" - "2. Present the indicator value, quoting the period it states.\n" - "3. Follow up with `search_insee_documents` or `search_insee_conjoncture` " - "if the user needs deeper tables, historic series, or a source document.\n" - "\n" - f"Current date is {compute_current_date_iso()}.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -# --- RMES (SPARQL) ---------------------------------------------------------- - -RMES_LIST_GRAPHS = { - "tool_name": "RMES_list_graphs", - "tool_description": ( - "Liste les graphes nommes disponibles dans la base RDF de l'INSEE (RMES). " - "Utilise ce tool EN PREMIER pour decouvrir quels graphes existent avant " - "d'ecrire une requete SPARQL avec RMES_run_sparql -- il y a plus de 700 graphes.\n" - "\n" - "Par defaut (`category=ALL`), le resultat est une vue CONDENSEE par categorie, " - "avec un compteur et quelques URIs d'exemple par categorie -- pas la liste plate " - "des 700+ graphes. Choisis une categorie precise dans le parametre `category` " - "pour cibler une famille, ou utilise `contains` pour une recherche libre par " - 'sous-chaine. Une categorie "autre" recueille tout graphe ne correspondant a ' - "aucune famille connue." - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -RMES_DESCRIBE_RESOURCE = { - "tool_name": "RMES_describe_resource", - "tool_description": ( - "Recupere toutes les proprietes connues (predicat -> valeur) d'une ressource RDF " - "identifiee par son URI complete. Combine automatiquement les proprietes ou la " - "ressource est sujet ET celles ou elle est objet (utile pour remonter des relations " - "skos:broader par exemple). Restreins avec `graph` si tu sais deja ou chercher -- " - "sinon la recherche se fait sur tous les graphes, ce qui est plus lent." - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -RMES_RUN_SPARQL = { - "tool_name": "RMES_run_sparql", - "tool_description": ( - "Execute une requete SPARQL libre sur RMES, la base de metadonnees, nomenclatures " - "et definitions de l'INSEE (elle ne contient PAS les chiffres/donnees, voir " - "get_MELODI_datasets pour ca).\n" - "\n" - "AVANT d'ecrire une requete complexe : appelle RMES_list_graphs pour connaitre les " - "categories de graphes disponibles.\n" - "\n" - "Bonnes pratiques :\n" - "- Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou " - " VALUES ?g { } plutot que de scanner tous les graphes.\n" - '- Toujours ajouter FILTER(lang(?label) = "fr") sur les litteraux SKOS pour eviter ' - " les doublons multilingues.\n" - "- Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee " - " automatiquement (indique dans la reponse via `limit_added`/`hint`).\n" - "- Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures " - " statistiques : ClassificationLevel, ExplanatoryNote), dcterms (metadonnees), " - " rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE).\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} - -SEND_FEEDBACK = { - "tool_name": "send_feedback", - "tool_description": ( - "Submit structured feedback about the MCP tools, server behavior, or user experience. " - "This tool appends a timestamped Markdown entry to the feedback log for administrator review.\n" - "\n" - "WHEN TO USE\n" - "- The user reports a bug, error, or unexpected behavior in any tool.\n" - "- The user suggests an improvement, new feature, or enhancement.\n" - "- The assistant encounters an issue during tool execution that should be logged.\n" - "- After completing a complex workflow where feedback on tool quality would be valuable.\n" - "\n" - "WHEN NOT TO USE\n" - "- For transient debugging or one-off troubleshooting (use terminal/logs instead).\n" - "- For questions about tool usage (ask the user or consult documentation).\n" - "\n" - "INPUT\n" - "- `username` -- identifier for the feedback author (e.g., user name, role, or session ID).\n" - "- `feedback` -- clear, actionable Markdown describing the issue or suggestion. " - "Include context (which tool, what happened), expected vs actual behavior, and " - "proposed solutions if applicable. Write as if filing a GitHub issue.\n" - "\n" - "OUTPUT\n" - "Confirmation message with the timestamp and path where feedback was recorded.\n" - ), - "tool_metadata": {"version": "5.0", "author": "mirlon"}, -} diff --git a/src/mcpdiffusion/core/instructions.py b/src/mcpdiffusion/core/instructions.py new file mode 100644 index 0000000..fbf00de --- /dev/null +++ b/src/mcpdiffusion/core/instructions.py @@ -0,0 +1,186 @@ +"""Server-level guidance, sent to every client during the MCP handshake. + +Assembled from the enabled tool families, so a deployment never advertises a workflow whose tools are not registered. +""" + +from textwrap import dedent + +# language=Markdown +OVERVIEW = """ + ## OVERVIEW + + This server exposes INSEE (French national statistics) data through three sources: + + - insee.fr -- publications, rapid releases and headline indicators + - MELODI -- the dataset catalogue and the observations themselves + - RMES -- statistical metadata: definitions and nomenclatures. It holds no figures. +""" + +# language=Markdown +GLOBAL_RULES = """ + ## RULES THAT APPLY TO EVERY TOOL + + - Never guess a dataset id, a modality code, a document URL or a graph URI. Each is opaque and must come from a + discovery call first. + - The data is French. Search with French keywords and rich synonyms. + - An empty result is a valid answer, not a failure. It usually means the filters were too narrow. +""" + +# language=Markdown +INSEE_SECTION = """ + ## insee.fr TOOLS + + ROUTING PRIORITY + - Simple statistics (population, inflation, chomage, PIB, salaires) by region/department? + -> Use `search_insee_chiffrecle` FIRST. + - Granular product data (e.g., beef rib price 2000)? -> Use `search_melodi_datasets` FIRST. + - `search_insee_documents` is for ANALYSIS, CONTEXT, and COMPLEX NARRATIVES. + + ### `search_insee_chiffrecle` + + WHEN TO USE + - Population, inflation, chomage, PIB, salaires, prix par categorie, comparaisons geographiques (region, + departement, commune). + - Cas simples : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?' + + WHEN NOT TO USE + - Analyses detaillees, impacts/contexte, tendances complexes, donnees produit granulaires historiques + -> `search_melodi_datasets` ou `search_insee_documents` selon le contexte. + + ### `search_insee_documents` + + WHEN TO USE + - Impact analyses (e.g., 'covid effects on tourism'). + - Historical evolution and trends (e.g., 'unemployment 1990-2026'). + - Detailed methodological or definitional content. + - Regional/departmental profiles with socioeconomic context. + - Specific thematic deep-dives (demography, labour market, inequalities, environment, housing, ...). + - Comparative studies or cross-cutting analyses. + + WHEN NOT TO USE + - Simple factual questions ('What is X region's population?') -> `search_insee_chiffrecle`. + - Quick, up-to-date headline indicators -> `get_insee_homepage`. + - Latest monthly/quarterly rapid releases -> `search_insee_conjoncture`. + - Vocabulary / code definitions / classifications -> `run_rmes_sparql`. + - Granular historical time series (product prices, individual wages) -> `search_melodi_datasets`. + + ### `search_insee_conjoncture` + + WHEN TO USE + - The user asks for the *latest* monthly/quarterly release of a named indicator (e.g. last month's consumer + confidence, last quarter's GDP estimate). Prefer the most recent edition. + + WHEN NOT TO USE + - Generic up-to-date indicator on the homepage: `get_insee_homepage`. + - Deep, peer-reviewed analysis: `search_insee_documents`. + + ### `get_insee_homepage` + + WHEN TO USE + - Preferred FIRST step for any generic, up-to-date statistical question. It gives the most recent official + figure instantly, without searching individual documents. + + WHEN NOT TO USE + - User asks for a previous year's figure. Use `search_insee_documents` or `search_insee_conjoncture` with + `year_of_reference`. + + WORKFLOW + 1. Call this tool. + 2. Present the indicator value, quoting the period it states. + 3. Follow up with `search_insee_documents` or `search_insee_conjoncture` if the user needs deeper tables, + historic series, or a source document. + + ### `get_insee_document` + + WHEN TO USE + - You have a concrete URL of the form `/fr/statistiques/` or `/fr/statistiques/?sommaire=`, + returned by one of the searches above. + + WHEN NOT TO USE + - You are still looking for the right publication. Use `search_insee_documents` first. + - You need a quick, up-to-date indicator. Use `get_insee_homepage`. +""" + +# language=Markdown +MELODI_SECTION = """ + ## MELODI TOOLS + + WORKFLOW (chain these three, in order) + 1. `search_melodi_datasets` -> dataset_id + column ids + 2. `search_melodi_modalities` -> exact modality codes for filtering + 3. `get_melodi_observations` -> final observations + + ### `search_melodi_datasets` + + WHEN TO USE + - The user asks for a specific statistic (price of a product, mortality by region, frequency of a name, etc.) + and you need to locate the right dataset before fetching rows. + + WHEN NOT TO USE + - Generic, up-to-date indicator questions (use `get_insee_homepage`). + - Full-text analysis of a published report (use `search_insee_documents`). + - Definition/ontology lookups (use `run_rmes_sparql`). + + ### `search_melodi_modalities` + + WHEN TO USE + - You have a `dataset_id` (from `search_melodi_datasets`) and want to find the exact modality code for a + concept like `cote de boeuf`, `Ile-de-France`, or `female Maria`. + + WHEN NOT TO USE + - You don't yet know the dataset. Run `search_melodi_datasets` first. + + ### `get_melodi_observations` + + WHEN TO USE + - You already know the exact `dataset_id` (from `search_melodi_datasets`) AND the modality codes you want to + filter on (from `search_melodi_modalities`). + + WHEN NOT TO USE + - You are still looking for the right dataset. Use `search_melodi_datasets` first. + - You need concept definitions or code-list vocabularies. Use `run_rmes_sparql`. +""" + +# language=Markdown +RMES_SECTION = """ + ## RMES TOOLS + + RMES holds metadata, definitions and nomenclatures. It holds no figures -- for actual data points use the + MELODI workflow. + + ### `search_rmes_graphs` + + WHEN TO USE + - FIRST, to discover which graphs exist before writing a SPARQL query with `run_rmes_sparql` -- there are more + than 700 graphs. + + ### `describe_rmes_resource` + + WHEN TO USE + - You already know a resource URI and want every property attached to it. + + ### `run_rmes_sparql` + + WHEN TO USE + - Vocabulary, code definitions and classifications, once you know which graphs to target. + + WHEN NOT TO USE + - You have not called `search_rmes_graphs` yet. Call it first to learn the available graph categories. +""" + + +def build_instructions( + *, + enable_inseefr_tools: bool, + enable_melodi_tools: bool, + enable_rmes_tools: bool, +) -> str: + """Assemble the guidance for the tool families this deployment actually registers.""" + sections = [OVERVIEW, GLOBAL_RULES] + if enable_inseefr_tools: + sections.append(INSEE_SECTION) + if enable_melodi_tools: + sections.append(MELODI_SECTION) + if enable_rmes_tools: + sections.append(RMES_SECTION) + return "\n\n".join(dedent(section).strip() for section in sections) diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/indicators.py index 5435b73..2da8d4c 100644 --- a/src/mcpdiffusion/data/indicators.py +++ b/src/mcpdiffusion/data/indicators.py @@ -8,6 +8,7 @@ # The tool description used to promise `mainIndicators` with a per-indicator link to pass to # `get_insee_document`, plus `lastArticles` and `keyGraphics`. None of that was ever produced. It is # recorded here because it says what the tool was meant to be, and is worth raising in that decision. +# Fixme: i feel this list can be typed, or at least the objects within KEY_INDICATORS = [ { "cle": "estimation de population France", diff --git a/src/mcpdiffusion/models/feedback.py b/src/mcpdiffusion/models/feedback.py index 1ad83e8..440f90c 100644 --- a/src/mcpdiffusion/models/feedback.py +++ b/src/mcpdiffusion/models/feedback.py @@ -2,15 +2,30 @@ from __future__ import annotations +from datetime import datetime +from typing import Annotated, Literal + from pydantic import BaseModel, Field +# ---------------------------------------------------------------------------------------------------------------------- +# Tool parameters ------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- -class SendFeedbackInput(BaseModel): - username: str = Field( +Author = Annotated[ + str, + Field( description="Identifier for the feedback author (e.g., user name, role, or session ID).", - examples=["alice", "data_analyst", "session_abc123"], - ) - feedback: str = Field( + examples=[ + "alice", + "data_analyst", + "session_abc123", + ], + ), +] + +Feedback = Annotated[ + str, + Field( description=( "Clear, actionable Markdown describing the issue or suggestion. Include context " "(which tool, what happened), expected vs actual behavior, and proposed solutions " @@ -22,12 +37,16 @@ class SendFeedbackInput(BaseModel): "at least one matching dataset.\n\n**Proposed fix:** Check if the Elasticsearch index " "includes this dataset.", ], - ) + ), +] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Result models -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- -class SendFeedbackOutput(BaseModel): - # Fixme: prefer a Literal - status: str = "success" +class FeedbackOutput(BaseModel): + status: Literal["success"] = "success" message: str - # Fixme: why not use a datetime object? - timestamp: str + timestamp: datetime diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index f65a45b..65c137b 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -3,15 +3,27 @@ from __future__ import annotations from enum import StrEnum +from typing import Annotated from pydantic import BaseModel, Field +# ---------------------------------------------------------------------------------------------------------------------- +# Constants ------------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- + +# Fixme: a lot of values in here belongs in settings +DEFAULT_RESULT_COUNT = 10 +MAX_RESULT_COUNT = 20 + + +# ---------------------------------------------------------------------------------------------------------------------- +# Enumerations --------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + -# Fixme: Pydantic BaseModel inheriting can be leverage to avoid duplication through model composition -# (FastAPI provides great examples on that) # Fixme: a lot of static data from this file seems derived from the one in the 'data' package # this could be merged / refactored / better exploited -class INSEETheme(StrEnum): +class ThemeChoice(StrEnum): ALL = "ALL" METHODES = "Methodes" DEMOGRAPHIE = "Demographie" @@ -25,7 +37,7 @@ class INSEETheme(StrEnum): TERRITOIRES = "Territoires, villes et quartiers" -class INSEEGeo(StrEnum): +class GeoLevelChoice(StrEnum): COM = "COM" DEP = "DEP" REG = "REG" @@ -34,7 +46,7 @@ class INSEEGeo(StrEnum): FRANCE = "FRANCE" -class ThemeConjoncture(StrEnum): +class ThemeConjonctureChoice(StrEnum): INDUSTRY = "Industrial production and activity" BUILDING = "Construction and building sector" HOUSING = "Housing and real estate" @@ -50,172 +62,168 @@ class ThemeConjoncture(StrEnum): FINANCE = "Business financing" -# --- Shared output model --- - - -class DocumentHit(BaseModel): - """Whitelisted publication record returned by INSEE.fr search tools.""" - - id: str = Field(description="Elasticsearch document id.") - score: float = Field(description="Relevance score from Elasticsearch.") - titre: str | None = None - soustitre: str | None = None - chapo: str | None = None - anneediffusion: str | None = Field(default=None, description="Publication year as indexed.") - zone: str | None = Field(default=None, description="Geographic zone (e.g. 'France', 'Bretagne').") - theme: str | None = None - collection_libelle: str | None = Field( - default=None, - description="Collection the publication belongs to (e.g. 'Insee Premiere', 'Informations rapides').", - ) - idproduit: str | None = Field( - default=None, - description="INSEE product identifier (often equal to the ES id).", - ) - url: str = Field(description="Relative URL ready to feed into `get_insee_document`.") +# ---------------------------------------------------------------------------------------------------------------------- +# Tool parameters ------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- +# --- shared by the INSEE.fr search tools --- -# --- search_insee_documents --- -# Fixme: use model composition to avoid duplication -class SearchInseeDocumentsInput(BaseModel): - query: str = Field( +Query = Annotated[ + str, + Field( description="Natural-language search query describing the statistics to retrieve.", - examples=["population de Lyon", "taux de chomage 2024", "PIB France"], - ) - theme: INSEETheme = Field( - default=INSEETheme.ALL, - description="Optional top-level INSEE theme used to restrict the search. Default: ALL.", - ) - year_of_reference: int | None = Field( - default=None, - description=("Hard filter on publication year (e.g. 2024). Leave null to search all years."), - ) - geo_niveau: INSEEGeo = Field( - default=INSEEGeo.FRANCE, - description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", - ) - geo_keyword: str | None = Field( - default=None, + examples=[ + "population de Lyon", + "taux de chomage 2024", + "PIB France", + ], + ), +] + +YearOfReference = Annotated[ + int | None, + Field(description="Hard filter on publication year (e.g. 2024). Leave null to search all years."), +] + +Theme = Annotated[ + ThemeChoice, + Field(description="Optional top-level INSEE theme used to restrict the search. Default: ALL."), +] + +GeoLevel = Annotated[ + GeoLevelChoice, + Field(description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE."), +] + +GeoKeyword = Annotated[ + str | None, + Field( description=( "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " "'Bouches-du-Rhone'). Leave null to skip geographic filtering." ), - ) - # Fixme: this field annotation is used multiple times and can be put in a variable to avoid duplication - # Fixme: magic values should be avoided - number_of_results: int = Field( - default=10, - description="Maximum number of results to return.", - ge=1, - le=20, - ) - - -# Fixme: the same model shape is used 3 times -class SearchInseeDocumentsOutput(BaseModel): - results: list[DocumentHit] - count: int - - -# --- search_insee_chiffrecle --- + ), +] - -class SearchInseeChiffrecleInput(BaseModel): - query: str = Field( - description="Natural-language search query describing the statistics to retrieve.", - examples=["population de Lyon", "taux de chomage 2024", "PIB France"], - ) - year_of_reference: int | None = Field( - default=None, - description=("Hard filter on publication year (e.g. 2024). Leave null to search all years."), - ) - geo_niveau: INSEEGeo = Field( - default=INSEEGeo.FRANCE, - description="Geographic level to search. Codes: COM / DEP / REG / INTER / COMPRD / FRANCE.", - ) - geo_keyword: str | None = Field( - default=None, - description=( - "Geographic name to filter on (e.g. 'Paris', 'Occitanie', " - "'Bouches-du-Rhone'). Leave null to skip geographic filtering." - ), - ) - number_of_results: int = Field( - default=10, +NumberOfResults = Annotated[ + int, + Field( description="Maximum number of results to return.", ge=1, - le=20, - ) - - -class SearchInseeChiffrecleOutput(BaseModel): - results: list[DocumentHit] - count: int - + le=MAX_RESULT_COUNT, + ), +] # --- search_insee_conjoncture --- - -class SearchInseeConjonctureInput(BaseModel): - query: str = Field( +ConjonctureQuery = Annotated[ + str, + Field( description=( "Natural-language query. The search is lexical and rewards " "keyword breadth -- provide several synonyms and related notions." ), - examples=["consommation", "hotel", "PIB"], - ) - theme_conjoncture: ThemeConjoncture | None = Field( - default=None, + examples=[ + "consommation", + "hotel", + "PIB", + ], + ), +] + +ThemeConjoncture = Annotated[ + ThemeConjonctureChoice | None, + Field( description=( "Optional broad category to restrict the search. Each category " "contains multiple sub-themes. Leave null to search across all." ), - ) - year_of_reference: int | None = Field( - default=None, + ), +] + +ConjonctureYearOfReference = Annotated[ + int | None, + Field( description=( "Hard filter on publication year (e.g. 2024). Leave null to " "search all years; for 'latest release' use cases, prefer " "leaving null so the freshest match wins by score." ), - ) - number_of_results: int = Field( - default=10, - description="Maximum number of results to return.", - ge=1, - le=20, - ) - - -class SearchInseeConjonctureOutput(BaseModel): - results: list[DocumentHit] - count: int - + ), +] # --- get_insee_document --- - -class GetInseeDocumentInput(BaseModel): - list_of_url: list[str] = Field( +DocumentUrls = Annotated[ + list[str], + Field( description=("List of relative URLs to retrieve (e.g. '/fr/statistiques/4277658?sommaire=4318291')."), - examples=[["/fr/statistiques/4277658?sommaire=4318291"]], - ) - include_sommaire: bool = Field( - default=True, + examples=[ + ["/fr/statistiques/4277658?sommaire=4318291"], + ], + ), +] + +IncludeTableOfContents = Annotated[ + bool, + Field( description=( "If True, parse the page's table-of-contents section alongside " "the main content. Use once to discover structure, then False " "for subsequent requests on the same page." ), - ) - truncate_content: bool = Field( - default=True, + ), +] + +TruncateContent = Annotated[ + bool, + Field( description=( "If True (default), long markdown bodies are clipped to keep the " "response compact for the model. Set to False only when the full " "text is required." ), + ), +] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Result models -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# --- shared by the INSEE.fr search tools --- + + +class DocumentHit(BaseModel): + """Whitelisted publication record returned by INSEE.fr search tools.""" + + id: str = Field(description="Elasticsearch document id.") + score: float = Field(description="Relevance score from Elasticsearch.") + titre: str | None = None + soustitre: str | None = None + chapo: str | None = None + anneediffusion: str | None = Field(default=None, description="Publication year as indexed.") + zone: str | None = Field(default=None, description="Geographic zone (e.g. 'France', 'Bretagne').") + theme: str | None = None + collection_libelle: str | None = Field( + default=None, + description="Collection the publication belongs to (e.g. 'Insee Premiere', 'Informations rapides').", ) + idproduit: str | None = Field( + default=None, + description="INSEE product identifier (often equal to the ES id).", + ) + url: str = Field(description="Relative URL ready to feed into `get_insee_document`.") + + +class DocumentSearchOutput(BaseModel): + """Result envelope shared by every INSEE.fr search tool.""" + + results: list[DocumentHit] + count: int + + +# --- get_insee_document --- class DocumentResult(BaseModel): @@ -226,7 +234,7 @@ class DocumentResult(BaseModel): default=None, description=( "Parsed table of contents as " - "{category: {title: url}}. None when include_sommaire=False " + "{category: {title: url}}. None when include_table_of_contents=False " "or when the page has no sommaire." ), ) @@ -240,7 +248,7 @@ class DocumentResult(BaseModel): ) -class GetInseeDocumentOutput(BaseModel): +class DocumentContentOutput(BaseModel): results: list[DocumentResult] count: int diff --git a/src/mcpdiffusion/models/melodi.py b/src/mcpdiffusion/models/melodi.py index a5cdc33..d38cf6d 100644 --- a/src/mcpdiffusion/models/melodi.py +++ b/src/mcpdiffusion/models/melodi.py @@ -2,28 +2,48 @@ from __future__ import annotations -from typing import Any +from typing import Annotated, Any from pydantic import BaseModel, Field -# --- get_melodi_observations --- +# ---------------------------------------------------------------------------------------------------------------------- +# Tool parameters ------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- +# --- shared by every Melodi tool --- -class GetMelodiObservationsInput(BaseModel): - dataset_id: str = Field( +DatasetId = Annotated[ + str, + Field( description="Identifier of the Melodi dataset (from search_melodi_datasets).", - examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], - ) - list_of_year: list[int] = Field( + examples=[ + "DS_DECES_MORTALITE_SERIES", + "DD_CNA_BRANCHES", + ], + ), +] + +# --- get_melodi_observations --- + +Years = Annotated[ + list[int], + Field( default_factory=list, description=( "Years to keep in the result set. Leave empty (the default) to " "return all available years. Pass e.g. [2020, 2021, 2022] to keep " "only those years." ), - examples=[[], [2020, 2021, 2022]], - ) - dict_of_columns_and_values: dict[str, str] = Field( + examples=[ + [], + [2020, 2021, 2022], + ], + ), +] + +ColumnFilters = Annotated[ + dict[str, str], + Field( default_factory=dict, description=( "Filters based on modality codes of columns. Leave empty to " @@ -35,26 +55,19 @@ class GetMelodiObservationsInput(BaseModel): {"PRICES": "D"}, {"PCS": "6", "GEO": "2025-FRANCE-FM"}, ], - ) - number_of_results: int = Field( - default=100, - description="Maximum number of observations to return.", - ge=1, - le=1000, - ) - - -class GetMelodiObservationsOutput(BaseModel): - dataset_id: str - observations: list[dict[str, Any]] - count: int + ), +] +NumberOfObservations = Annotated[ + int, + Field(description="Maximum number of observations to return.", ge=1, le=1000), +] # --- search_melodi_datasets --- - -class SearchMelodiDatasetsInput(BaseModel): - french_query: str = Field( +DatasetQuery = Annotated[ + str, + Field( description=( "Explicit French description of the statistical dataset to search. " "Mention the phenomenon (inflation, births, unemployment), " @@ -68,21 +81,71 @@ class SearchMelodiDatasetsInput(BaseModel): "population communale", "salaires des enseignants", ], - ) - start_year: int = Field( - default=1900, - description="Dataset must contain data from at least this year.", - ) - end_year: int = Field( - default=2100, - description="Dataset must contain data up to at least this year.", - ) - number_of_results: int = Field( - default=5, - description="Maximum number of datasets to return, ordered by relevance.", - ge=1, - le=20, - ) + ), +] + +StartYear = Annotated[ + int, + Field(description="Dataset must contain data from at least this year."), +] + +EndYear = Annotated[ + int, + Field(description="Dataset must contain data up to at least this year."), +] + +NumberOfDatasets = Annotated[ + int, + Field(description="Maximum number of datasets to return, ordered by relevance.", ge=1, le=20), +] + +# --- search_melodi_modalities --- + +ColumnIds = Annotated[ + list[str], + Field( + description="Identifiers of the columns within the dataset to search.", + examples=[ + ["PRICES"], + ["PRICES", "GEO"], + ], + ), +] + +ModalityQuery = Annotated[ + str, + Field( + description=( + "Natural-language French query describing the modalities to " + "retrieve (e.g. 'cote de boeuf', 'Ile-de-France', 'female Maria')." + ), + examples=[ + "prix", + "boissons non alcoolisees", + ], + ), +] + +NumberOfModalities = Annotated[ + int, + Field(description="Maximum number of modalities to return per column.", ge=1, le=50), +] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Result models -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# --- get_melodi_observations --- + + +class ObservationsOutput(BaseModel): + dataset_id: str + observations: list[dict[str, Any]] + count: int + + +# --- search_melodi_datasets --- class DatasetDescription(BaseModel): @@ -99,37 +162,13 @@ class DatasetSearchResult(BaseModel): dataset_score: float -class SearchMelodiDatasetsOutput(BaseModel): +class DatasetsOutput(BaseModel): results: list[DatasetSearchResult] # --- search_melodi_modalities --- -class SearchMelodiModalitiesInput(BaseModel): - dataset_id: str = Field( - description="Identifier of the Melodi dataset (from search_melodi_datasets).", - examples=["DS_DECES_MORTALITE_SERIES", "DD_CNA_BRANCHES"], - ) - columns_id: list[str] = Field( - description="Identifiers of the columns within the dataset to search.", - examples=[["PRICES"], ["PRICES", "GEO"]], - ) - french_query: str = Field( - description=( - "Natural-language French query describing the modalities to " - "retrieve (e.g. 'cote de boeuf', 'Ile-de-France', 'female Maria')." - ), - examples=["prix", "boissons non alcoolisees"], - ) - number_of_results: int = Field( - default=10, - description="Maximum number of modalities to return per column.", - ge=1, - le=50, - ) - - class Modality(BaseModel): code: str label_en: str @@ -139,9 +178,9 @@ class Modality(BaseModel): class ColumnResult(BaseModel): column_code: str - metadata_columns: str + column_metadata: str matching_modalities: list[Modality] -class SearchMelodiModalitiesOutput(BaseModel): +class ModalitiesOutput(BaseModel): results: list[ColumnResult] diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index ee2bc4c..808971a 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -3,11 +3,13 @@ from __future__ import annotations from enum import StrEnum -from typing import Any, Literal +from typing import Annotated, Any, Literal from pydantic import BaseModel, Field -# --- Shared RMES constants exposed to tools --- +# ---------------------------------------------------------------------------------------------------------------------- +# Constants ------------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- # Fixme: a lot of values in here belongs in settings DEFAULT_QUERY_TIMEOUT_SECONDS = 20.0 @@ -18,7 +20,9 @@ GRAPH_BASE = "http://rdf.insee.fr/graphes/" -# --- Graph taxonomy --- +# ---------------------------------------------------------------------------------------------------------------------- +# Enumerations --------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- class GraphCategoryChoice(StrEnum): @@ -38,36 +42,101 @@ class GraphCategoryChoice(StrEnum): AUTRE = "autre" -class GraphRow(BaseModel): - graph: str - triples: int +# ---------------------------------------------------------------------------------------------------------------------- +# Tool parameters ------------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- +# --- search_rmes_graphs --- -# --- RMES_list_graphs --- - - -class ListGraphsInput(BaseModel): - contains: str | None = Field( - default=None, +GraphUriSubstring = Annotated[ + str | None, + Field( description=( "Filtre les graphes dont l'URI contient cette sous-chaine (insensible a la " "casse), ex. 'naf' ou 'qualite/rapport'. Active automatiquement le detail " "complet (`graphs`) dans les categories retenues." ), - examples=["naf", "qualite/rapport", "geo"], - ) - category: GraphCategoryChoice = Field( - default=GraphCategoryChoice.ALL, - description="Categorie de graphes a cibler.", - ) - expand: bool = Field( - default=False, + examples=[ + "naf", + "qualite/rapport", + "geo", + ], + ), +] + +GraphCategory = Annotated[ + GraphCategoryChoice, + Field(description="Categorie de graphes a cibler."), +] + +ExpandGraphs = Annotated[ + bool, + Field( description=( "Si True, inclut la liste complete des graphes (URI + nb de triplets) pour " "chaque categorie retenue, au lieu de seulement quelques exemples. Se " - "declenche automatiquement si `contains` est fourni ou `category != ALL`." + "declenche automatiquement si `graph_uri_substring` est fourni ou `graph_category != ALL`." + ), + ), +] + +# --- describe_rmes_resource --- + +ResourceUri = Annotated[ + str, + Field( + description="URI complete de la ressource RDF a decrire.", + examples=[ + "http://id.insee.fr/codes/naf2025/section/A", + ], + ), +] + +GraphUri = Annotated[ + str | None, + Field( + description=( + "URI d'un graphe nomme pour restreindre la recherche. Sans cette valeur (None par defaut), " + "la recherche se fait sur tous les graphes (plus lent)." ), - ) + ), +] + +# --- run_rmes_sparql --- + +SparqlQuery = Annotated[ + str, + Field(description="Requete SPARQL complete (SELECT / ASK / CONSTRUCT / DESCRIBE)."), +] + +TimeoutSeconds = Annotated[ + float, + Field( + description=f"Timeout en secondes (plafonne a {MAX_QUERY_TIMEOUT_SECONDS}s).", + gt=0, + ), +] + +MaxRows = Annotated[ + int, + Field( + description=f"Limite de lignes ajoutee si absente de la requete (plafonnee a {MAX_ROW_LIMIT}).", + ge=1, + le=MAX_ROW_LIMIT, + ), +] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Result models -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# --- search_rmes_graphs --- + + +class GraphRow(BaseModel): + graph: str + triples: int class CategoryBucket(BaseModel): @@ -80,27 +149,12 @@ class CategoryBucket(BaseModel): graphs: list[GraphRow] | None = None -class ListGraphsOutput(BaseModel): +class GraphsOutput(BaseModel): total_graphs_matched: int categories: list[CategoryBucket] -# --- RMES_describe_resource --- - - -class DescribeResourceInput(BaseModel): - uri: str = Field( - description="URI complete de la ressource RDF a decrire.", - examples=["http://id.insee.fr/codes/naf2025/section/A"], - ) - # Fixme: either use Optional or the modern pipe syntax, but avoid mixing - graph: str | None = Field( - default=None, - description=( - "URI d'un graphe nomme pour restreindre la recherche. Sans cette valeur (None par defaut), " - "la recherche se fait sur tous les graphes (plus lent)." - ), - ) +# --- describe_rmes_resource --- class ResourceProperty(BaseModel): @@ -112,33 +166,16 @@ class ResourceProperty(BaseModel): lang: str | None = None -class DescribeResourceOutput(BaseModel): +class ResourceOutput(BaseModel): uri: str properties: list[ResourceProperty] count: int -# --- RMES_run_sparql --- - - -class RunSparqlInput(BaseModel): - full_sparql_query: str = Field( - description="Requete SPARQL complete (SELECT / ASK / CONSTRUCT / DESCRIBE).", - ) - timeout: float = Field( - default=DEFAULT_QUERY_TIMEOUT_SECONDS, - description=f"Timeout en secondes (plafonne a {MAX_QUERY_TIMEOUT_SECONDS}s).", - gt=0, - ) - max_rows: int = Field( - default=DEFAULT_ROW_LIMIT, - description=f"Limite de lignes ajoutee si absente de la requete (plafonnee a {MAX_ROW_LIMIT}).", - ge=1, - le=MAX_ROW_LIMIT, - ) +# --- run_rmes_sparql --- -class RunSparqlOutput(BaseModel): +class SparqlOutput(BaseModel): format: Literal["json", "turtle"] = "json" limit_added: int | None = None hint: str | None = None diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 58eccc1..fa1501c 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -10,6 +10,7 @@ from fastmcp.server.middleware.timing import TimingMiddleware from .config.settings import load_settings +from .core.instructions import build_instructions from .core.logging import build_logging_config, configure_logging from .core.rate_limiting import resolve_client_host from .infra.lifespan import build_lifespan @@ -21,6 +22,13 @@ mcp = FastMCP( "INSEE-mcp-diffusion", + # Routing guidance, delivered in the handshake so it reaches the caller without relying on a + # separate file being loaded. Built from the enabled families so it never names a missing tool. + instructions=build_instructions( + enable_inseefr_tools=settings.enable_inseefr_tools, + enable_melodi_tools=settings.enable_melodi_tools, + enable_rmes_tools=settings.enable_rmes_tools, + ), # Only AppToolError messages reach the caller; anything else is a bug and is replaced # by a generic message. mask_error_details=True, diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py index ccc6a8e..c65f6c6 100644 --- a/src/mcpdiffusion/services/feedback.py +++ b/src/mcpdiffusion/services/feedback.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path -from ..models.feedback import SendFeedbackInput, SendFeedbackOutput +from ..models.feedback import FeedbackOutput # Fixme: this is extremely hacky, and feedback gets tied to the running instance _FEEDBACK_DIR = Path(__file__).resolve().parent.parent / "feedback" @@ -28,14 +28,18 @@ def _ensure_feedback_file() -> Path: return _FEEDBACK_FILE -# Fixme: I am wondering where the username come from, because if sent by the client, this can be messed up -# Fixme: Also wondering if there is a cap on the username size of the feedback content, +# Fixme: I am wondering where the author come from, because if sent by the client, this can be messed up +# Fixme: Also wondering if there is a cap on the author size of the feedback content, # it can make the container write uncontrolled amount of data -async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: +async def send_feedback_service( + *, + author: str, + feedback: str, +) -> FeedbackOutput: feedback_path = _ensure_feedback_file() # Fixme: there is no timezone here, while at some in the code we consider timezones # it seems a bit inconsistent - timestamp = datetime.now().isoformat(timespec="seconds") + recorded_at = datetime.now() # Fixme: prefer more readable multiline strings entry = ( @@ -43,7 +47,7 @@ async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: # just hope this is not ultimately fed to an LLM # Fixme: a big flaw is that feedback.md is versioned, so user feedback might be fed into git # Fixme: beware the data is lost on each restart - f"## {timestamp} — {params.username}\n\n{params.feedback}\n\n---\n\n" + f"## {recorded_at.isoformat(timespec='seconds')} — {author}\n\n{feedback}\n\n---\n\n" ) # Fixme: this call is blocking the event loop @@ -51,7 +55,7 @@ async def send_feedback(params: SendFeedbackInput) -> SendFeedbackOutput: with feedback_path.open("a", encoding="utf-8") as f: f.write(entry) - return SendFeedbackOutput( + return FeedbackOutput( message="Feedback recorded successfully.", - timestamp=timestamp, + timestamp=recorded_at, ) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index 82342ff..b978cf8 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -13,9 +13,8 @@ from ..core.errors import AppToolError from ..models.insee import ( + DocumentContentOutput, DocumentResult, - GetInseeDocumentInput, - GetInseeDocumentOutput, ) logger = logging.getLogger(__name__) @@ -147,31 +146,33 @@ def _build_failed_document(url: object, message: str) -> DocumentResult: ) -async def get_insee_document( - params: GetInseeDocumentInput, +async def get_insee_document_service( *, + document_urls: list[str], + include_table_of_contents: bool, + truncate_content: bool, http_client: httpx.AsyncClient, -) -> GetInseeDocumentOutput: - if not params.list_of_url: +) -> DocumentContentOutput: + if not document_urls: raise AppToolError( "INVALID_INPUT", - "list_of_url must contain at least one URL. Use `search_insee_documents` to find URLs first.", + "document_urls must contain at least one URL. Use `search_insee_documents` to find URLs first.", ) results: list[DocumentResult] = [] # Fixme: there should be a cap in the number of URLs provided to avoid overloading the server # Fixme: on top of that, the fetching is done sequentially, impacting the event loop - for url in params.list_of_url: + for url in document_urls: try: html = await _fetch_html(str(url), http_client) markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" - if params.truncate_content: + if truncate_content: markdown, truncated = _truncate(markdown) else: truncated = False sommaire: dict[str, dict[str, str]] | None = None - if params.include_sommaire: + if include_table_of_contents: flat = _parse_sommaire(html, str(http_client.base_url)) sommaire = _format_sommaire(flat) if flat else None @@ -193,4 +194,4 @@ async def get_insee_document( logger.exception("Unexpected failure fetching %s", url) results.append(_build_failed_document(url, "[UNKNOWN] Could not fetch this document.")) - return GetInseeDocumentOutput(results=results, count=len(results)) + return DocumentContentOutput(results=results, count=len(results)) diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py index cfe42af..bb11c49 100644 --- a/src/mcpdiffusion/services/insee_search.py +++ b/src/mcpdiffusion/services/insee_search.py @@ -85,7 +85,7 @@ def apply_collection_filters( must_only_rapides: bool, chiffre_clef: bool = False, theme: str | None = None, - geo_niveau: str | None = None, + geo_level: str | None = None, geo_keyword: str | None = None, ) -> tuple[list, list]: """Return new (filters, should) lists. The caller's `filters` is left untouched.""" @@ -108,8 +108,8 @@ def apply_collection_filters( if chiffre_clef: filters.append(Q("term", categorie_libelle="Chiffres-clés")) - if geo_niveau: - key_geo = DICT_GEO.get(geo_niveau) + if geo_level: + key_geo = DICT_GEO.get(geo_level) if key_geo: # Business rule: same as the theme filter above — an unrecognised geo_niveau is dropped # silently and broadens the search. diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py index 22d844c..ef50c0c 100644 --- a/src/mcpdiffusion/services/melodi.py +++ b/src/mcpdiffusion/services/melodi.py @@ -12,27 +12,27 @@ from ..models.melodi import ( ColumnResult, DatasetSearchResult, - GetMelodiObservationsInput, - GetMelodiObservationsOutput, + DatasetsOutput, + ModalitiesOutput, Modality, - SearchMelodiDatasetsInput, - SearchMelodiDatasetsOutput, - SearchMelodiModalitiesInput, - SearchMelodiModalitiesOutput, + ObservationsOutput, ) -async def get_melodi_observations( - params: GetMelodiObservationsInput, +async def get_melodi_observations_service( + dataset_id: str, + years: list[int], + column_filters: dict[str, str], + number_of_observations: int, *, http_client: httpx.AsyncClient, -) -> GetMelodiObservationsOutput: +) -> ObservationsOutput: # Resolved against the client's base_url. - url = f"/{params.dataset_id}" + url = f"/{dataset_id}" try: response = await http_client.get( url, - params=params.dict_of_columns_and_values or None, + params=column_filters or None, ) response.raise_for_status() # Fixme: the following problematic error handling pattern has already been adressed @@ -49,14 +49,14 @@ async def get_melodi_observations( raise AppToolError( "INVALID_INPUT", f"Melodi API rejected the query (HTTP 400). " - f"Columns/values passed: {params.dict_of_columns_and_values}. " + f"Columns/values passed: {column_filters}. " f"Upstream detail: {body_excerpt}. " "Verify modality codes with `search_melodi_modalities`.", ) elif status == 404: raise AppToolError( "NOT_FOUND", - f"Melodi dataset {params.dataset_id!r} not found (HTTP 404). " + f"Melodi dataset {dataset_id!r} not found (HTTP 404). " "Check the dataset_id with `search_melodi_datasets`.", ) else: @@ -87,8 +87,8 @@ async def get_melodi_observations( "Melodi API response did not contain an 'observations' list.", ) - if params.list_of_year: - years_str = {str(y) for y in params.list_of_year} + if years: + years_str = {str(y) for y in years} # Fixme: it seems we retrieve all the observations data and filter next # I wonder whether the API supports filtering observations = [ @@ -98,9 +98,9 @@ async def get_melodi_observations( if (obs.get("dimensions", {}).get("TIME_PERIOD", "").split("-")[0]) in years_str ] - sliced = observations[: params.number_of_results] - return GetMelodiObservationsOutput( - dataset_id=params.dataset_id, + sliced = observations[:number_of_observations] + return ObservationsOutput( + dataset_id=dataset_id, observations=sliced, count=len(sliced), ) @@ -110,20 +110,23 @@ async def get_melodi_observations( # the code might benefit having a repository layer to encapsulate data access -async def search_melodi_datasets( - params: SearchMelodiDatasetsInput, +async def search_melodi_datasets_service( + query: str, + start_year: int, + end_year: int, + number_of_datasets: int, *, es: AsyncElasticsearch, index: str, -) -> SearchMelodiDatasetsOutput: +) -> DatasetsOutput: filters: list[dict[str, Any]] = [] - if params.start_year: - filters.append({"range": {"metadata.temporal.endPeriod": {"gte": f"{params.start_year}-01-01"}}}) - if params.end_year: - filters.append({"range": {"metadata.temporal.startPeriod": {"lte": f"{params.end_year}-12-31"}}}) + if start_year: + filters.append({"range": {"metadata.temporal.endPeriod": {"gte": f"{start_year}-01-01"}}}) + if end_year: + filters.append({"range": {"metadata.temporal.startPeriod": {"lte": f"{end_year}-12-31"}}}) body = { - "size": params.number_of_results, + "size": number_of_datasets, "query": { "bool": { "should": [ @@ -133,7 +136,7 @@ async def search_melodi_datasets( "query": { "match": { "metadata.title.content": { - "query": params.french_query, + "query": query, "boost": 10, } } @@ -146,7 +149,7 @@ async def search_melodi_datasets( "query": { "match": { "metadata.abstract.content": { - "query": params.french_query, + "query": query, "boost": 6, } } @@ -159,7 +162,7 @@ async def search_melodi_datasets( "query": { "match": { "metadata.description.content": { - "query": params.french_query, + "query": query, "boost": 3, } } @@ -169,7 +172,7 @@ async def search_melodi_datasets( { "match": { "variables_text": { - "query": params.french_query, + "query": query, "boost": 5, } } @@ -208,18 +211,21 @@ async def search_melodi_datasets( dataset_score=float(hit.get("_score") or 0.0), ) ) - return SearchMelodiDatasetsOutput(results=results) + return DatasetsOutput(results=results) -async def search_melodi_modalities( - params: SearchMelodiModalitiesInput, +async def search_melodi_modalities_service( + dataset_id: str, + column_ids: list[str], + query: str, + number_of_modalities: int, *, es: AsyncElasticsearch, index: str, -) -> SearchMelodiModalitiesOutput: - filters: list[dict[str, Any]] = [{"term": {"dataset_id": params.dataset_id}}] - if params.columns_id: - filters.append({"terms": {"code": params.columns_id}}) +) -> ModalitiesOutput: + filters: list[dict[str, Any]] = [{"term": {"dataset_id": dataset_id}}] + if column_ids: + filters.append({"terms": {"code": column_ids}}) try: ds_column = await es.search( @@ -232,7 +238,7 @@ async def search_melodi_modalities( { "match": { "text": { - "query": params.french_query, + "query": query, "boost": 2, } } @@ -243,7 +249,7 @@ async def search_melodi_modalities( "score_mode": "max", "query": { "multi_match": { - "query": params.french_query, + "query": query, "fields": [ "modalities.code^5", "modalities.label.en^3", @@ -253,7 +259,7 @@ async def search_melodi_modalities( } }, "inner_hits": { - "size": params.number_of_results, + "size": number_of_modalities, "sort": [{"_score": "desc"}], }, } @@ -287,9 +293,9 @@ async def search_melodi_modalities( results.append( ColumnResult( column_code=str(hit.get("_source", {}).get("code", "")), - metadata_columns=str(hit.get("_source", {}).get("text", "")), + column_metadata=str(hit.get("_source", {}).get("text", "")), matching_modalities=modalities, ) ) - return SearchMelodiModalitiesOutput(results=results) + return ModalitiesOutput(results=results) diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 5f3d71f..539c18b 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -20,15 +20,12 @@ MAX_QUERY_TIMEOUT_SECONDS, MAX_ROW_LIMIT, CategoryBucket, - DescribeResourceInput, - DescribeResourceOutput, GraphCategoryChoice, GraphRow, - ListGraphsInput, - ListGraphsOutput, + GraphsOutput, + ResourceOutput, ResourceProperty, - RunSparqlInput, - RunSparqlOutput, + SparqlOutput, ) # Fixme: follow a clear convention for logger names @@ -43,27 +40,9 @@ _GRAPH_CACHE_TTL = 3600.0 # 1h -# --- Known vocabularies note (injected in run_sparql description) --- - -KNOWN_VOCABULARIES_NOTE = """ -Vocabulaires principaux rencontres dans cette base (au-dela de skos/xkos/dcterms) : -- sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualite. Un sdmx-mm:MetadataReport - a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des - sdmx-mm:ReportedAttribute rattaches via sdmx-mm:metadataReport. -- rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, StatisticalOperationSeries, - StatisticalOperationFamily (graphe "operations"), StatisticalIndicator (graphe "produits"), - StatutDiffusion... -- org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes - "organisations" et "organisations/insee"). -- dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). -Utilise RMES_list_graphs pour voir les grandes categories de graphes avant de creuser -avec ce tool. -""".strip() - - -# --------------------------------------------------------------------------- -# Graph taxonomy -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- +# Graph taxonomy ------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- # Fixme: this is too broad of a type CategoryMatcher = Any # Callable[[str], bool] @@ -222,9 +201,9 @@ def _categorize(graph_uri: str) -> _CategoryRule: return _CATEGORY_AUTRE -# --------------------------------------------------------------------------- -# SPARQL query helpers -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- +# SPARQL query helpers ------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- _STRIP_PREFIX_RE = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) _QUERY_FORM_RE = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") @@ -253,14 +232,14 @@ def _accept_header(query_form: str) -> str: return "text/turtle" -# --------------------------------------------------------------------------- -# Low-level SPARQL execution -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- +# Low-level SPARQL execution ------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- async def _execute_sparql( query: str, - timeout: float, + timeout_seconds: float, max_rows: int, *, sparql_client: httpx.AsyncClient, @@ -284,14 +263,14 @@ async def _execute_sparql( endpoint, data={"query": effective_query}, headers={"Accept": accept}, - timeout=min(timeout, MAX_QUERY_TIMEOUT_SECONDS), + timeout=min(timeout_seconds, MAX_QUERY_TIMEOUT_SECONDS), ) response.raise_for_status() except httpx.TimeoutException: raise AppToolError( "BACKEND_UNAVAILABLE", - f"Le endpoint RMES n'a pas repondu en moins de {timeout}s. " + f"Le endpoint RMES n'a pas repondu en moins de {timeout_seconds}s. " "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " "evite les scans sans filtre sur tous les graphes).", retryable=True, @@ -354,7 +333,7 @@ async def _get_raw_graph_rows( # otherwise each waiter just re-runs the same expensive query result = await _execute_sparql( query, - timeout=GRAPH_LISTING_TIMEOUT_SECONDS, + timeout_seconds=GRAPH_LISTING_TIMEOUT_SECONDS, max_rows=GRAPH_LISTING_MAX_ROWS, sparql_client=sparql_client, endpoint=endpoint, @@ -368,9 +347,9 @@ async def _get_raw_graph_rows( return _GRAPH_CACHE["data"] -# --------------------------------------------------------------------------- -# High-level tool operations -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- +# High-level tool operations ------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: @@ -397,30 +376,31 @@ def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: return [buckets[k] for k in ordered_keys if k in buckets] -async def list_graphs( - params: ListGraphsInput, +async def search_rmes_graphs_service( + graph_uri_substring: str | None, + graph_category: GraphCategoryChoice, + expand_graphs: bool, *, sparql_client: httpx.AsyncClient, endpoint: str, -) -> ListGraphsOutput: +) -> GraphsOutput: rows = await _get_raw_graph_rows( sparql_client=sparql_client, endpoint=endpoint, ) - expand = params.expand - if params.contains: - needle = params.contains.lower() + if graph_uri_substring: + needle = graph_uri_substring.lower() rows = [r for r in rows if needle in r["graph"].lower()] - expand = True + expand_graphs = True - if params.category != GraphCategoryChoice.ALL: - rows = [r for r in rows if _categorize(r["graph"]).key == params.category.value] - expand = True + if graph_category != GraphCategoryChoice.ALL: + rows = [r for r in rows if _categorize(r["graph"]).key == graph_category.value] + expand_graphs = True summary = _build_category_summary(rows) - if expand: + if expand_graphs: rows_by_graph = {r["graph"]: r["triples"] for r in rows} for bucket in summary: bucket_rows = [ @@ -432,7 +412,7 @@ async def list_graphs( bucket_rows.sort(key=lambda r: r.triples, reverse=True) bucket.graphs = bucket_rows - return ListGraphsOutput(total_graphs_matched=len(rows), categories=summary) + return GraphsOutput(total_graphs_matched=len(rows), categories=summary) def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[ResourceProperty]: @@ -451,68 +431,69 @@ def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[Resour return props -async def describe_resource( - params: DescribeResourceInput, +async def describe_rmes_resource_service( + resource_uri: str, + graph_uri: str | None, *, sparql_client: httpx.AsyncClient, endpoint: str, -) -> DescribeResourceOutput: - graph_clause = f"<{params.graph}>" if params.graph else "?g" - graph_values = f"VALUES ?g {{ <{params.graph}> }}" if params.graph else "" +) -> ResourceOutput: + graph_clause = f"<{graph_uri}>" if graph_uri else "?g" + graph_values = f"VALUES ?g {{ <{graph_uri}> }}" if graph_uri else "" # Fixme: the query is built using string interpolation # just check whether injection can cause problems here query = f""" SELECT ?g ?direction ?p ?o WHERE {{ {graph_values} {{ - GRAPH {graph_clause} {{ <{params.uri}> ?p ?o }} + GRAPH {graph_clause} {{ <{resource_uri}> ?p ?o }} BIND("outgoing" AS ?direction) }} UNION {{ - GRAPH {graph_clause} {{ ?o ?p <{params.uri}> }} + GRAPH {graph_clause} {{ ?o ?p <{resource_uri}> }} BIND("incoming" AS ?direction) }} }} LIMIT {MAX_ROW_LIMIT} """ result = await _execute_sparql( query, - timeout=DEFAULT_QUERY_TIMEOUT_SECONDS, + timeout_seconds=DEFAULT_QUERY_TIMEOUT_SECONDS, max_rows=MAX_ROW_LIMIT, sparql_client=sparql_client, endpoint=endpoint, ) properties = _parse_bindings_to_properties(result["results"]["bindings"]) - return DescribeResourceOutput(uri=params.uri, properties=properties, count=len(properties)) + return ResourceOutput(uri=resource_uri, properties=properties, count=len(properties)) -async def run_sparql( - params: RunSparqlInput, +async def run_rmes_sparql_service( + sparql_query: str, + timeout_seconds: float, + max_rows: int, *, sparql_client: httpx.AsyncClient, endpoint: str, -) -> RunSparqlOutput: - if not params.full_sparql_query or not params.full_sparql_query.strip(): +) -> SparqlOutput: + if not sparql_query or not sparql_query.strip(): raise AppToolError( "INVALID_INPUT", "La requete SPARQL est vide. Fournis une requete SELECT, ASK, CONSTRUCT ou DESCRIBE.", ) - max_rows = max(1, min(params.max_rows, MAX_ROW_LIMIT)) + max_rows = max(1, min(max_rows, MAX_ROW_LIMIT)) result = await _execute_sparql( - params.full_sparql_query, - timeout=params.timeout, + sparql_query, + timeout_seconds=timeout_seconds, max_rows=max_rows, sparql_client=sparql_client, endpoint=endpoint, ) if result.get("format") == "turtle": - return RunSparqlOutput( - format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"] - ) + return SparqlOutput(format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"]) meta = result.get("_meta", {}) - return RunSparqlOutput( + return SparqlOutput( format="json", limit_added=meta.get("limit_added"), hint=meta.get("hint"), diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 6d0d42e..f42d0cf 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -13,18 +13,18 @@ # Imported but never registered: send_feedback is not exposed. Decide whether to wire it up or # drop it, then remove this import or the noqa. -from .extras_send_feedback import register_extras_send_feedback # noqa: F401 +from .feedback_send import register_send_feedback # noqa: F401 from .insee_get_document import register_get_insee_document from .insee_get_homepage import register_get_insee_homepage -from .insee_search_chiffrecle import register_search_insee_chiffreclef +from .insee_search_chiffrecle import register_search_insee_chiffrecle from .insee_search_conjoncture import register_search_insee_conjoncture from .insee_search_documents import register_search_insee_documents from .melodi_get_observations import register_get_melodi_observations from .melodi_search_datasets import register_search_melodi_datasets from .melodi_search_modalities import register_search_melodi_modalities -from .rmes_describe_resource import register_rmes_describe_resource -from .rmes_list_graphs import register_rmes_list_graphs -from .rmes_run_sparql import register_rmes_run_sparql +from .rmes_describe_resource import register_describe_rmes_resource +from .rmes_run_sparql import register_run_rmes_sparql +from .rmes_search_graphs import register_search_rmes_graphs # Fixme: there might be better pattern instead of iterating with if statements on tool groups @@ -35,7 +35,7 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: register_get_insee_homepage(mcp) register_get_insee_document(mcp) register_search_insee_conjoncture(mcp, index=settings.es_index_produits) - register_search_insee_chiffreclef(mcp, index=settings.es_index_produits) + register_search_insee_chiffrecle(mcp, index=settings.es_index_produits) if settings.enable_melodi_tools: register_search_melodi_datasets(mcp, index=settings.es_index_melodi_datasets) @@ -43,6 +43,6 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: register_get_melodi_observations(mcp) if settings.enable_rmes_tools: - register_rmes_list_graphs(mcp, endpoint=settings.rmes_endpoint) - register_rmes_describe_resource(mcp, endpoint=settings.rmes_endpoint) - register_rmes_run_sparql(mcp, endpoint=settings.rmes_endpoint) + register_search_rmes_graphs(mcp, endpoint=settings.rmes_endpoint) + register_describe_rmes_resource(mcp, endpoint=settings.rmes_endpoint) + register_run_rmes_sparql(mcp, endpoint=settings.rmes_endpoint) diff --git a/src/mcpdiffusion/tools/extras_send_feedback.py b/src/mcpdiffusion/tools/extras_send_feedback.py deleted file mode 100644 index 503292f..0000000 --- a/src/mcpdiffusion/tools/extras_send_feedback.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Tool: send_feedback -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import FastMCP - -from ..config.tool_metadata import SEND_FEEDBACK -from ..models.feedback import SendFeedbackInput, SendFeedbackOutput -from ..services.feedback import send_feedback - - -# Fixme: I advocated for co-location schema + tools using docstrings if possible -# Fixme: I already stated clients can send anything as username and feedback -def register_extras_send_feedback(mcp: FastMCP) -> None: - @mcp.tool( - name=SEND_FEEDBACK["tool_name"], - description=SEND_FEEDBACK["tool_description"], - meta=SEND_FEEDBACK["tool_metadata"], - ) - async def send_feedback_tool(params: SendFeedbackInput) -> SendFeedbackOutput: - return await send_feedback(params) diff --git a/src/mcpdiffusion/tools/feedback_send.py b/src/mcpdiffusion/tools/feedback_send.py new file mode 100644 index 0000000..24cffe3 --- /dev/null +++ b/src/mcpdiffusion/tools/feedback_send.py @@ -0,0 +1,26 @@ +"""Tool: send_feedback -- thin registration layer.""" + +from __future__ import annotations + +from fastmcp import FastMCP + +from ..models.feedback import Author, Feedback, FeedbackOutput +from ..services.feedback import send_feedback_service + + +# Fixme: I already stated clients can send anything as author and feedback +def register_send_feedback(mcp: FastMCP) -> None: + @mcp.tool + async def send_feedback( + author: Author, + feedback: Feedback, + ) -> FeedbackOutput: + """Submit structured feedback about the MCP tools, server behavior, or user experience. This + tool appends a timestamped Markdown entry to the feedback log for administrator review. + + Returns a confirmation carrying the timestamp under which the feedback was recorded. + """ + return await send_feedback_service( + author=author, + feedback=feedback, + ) diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 1da3c09..0673a6b 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -4,21 +4,32 @@ from fastmcp import Context, FastMCP -from ..config.tool_metadata import GET_DOCUMENT from ..infra.http import get_insee_http_client -from ..models.insee import GetInseeDocumentInput, GetInseeDocumentOutput -from ..services.insee_document import get_insee_document +from ..models.insee import ( + DocumentContentOutput, + DocumentUrls, + IncludeTableOfContents, + TruncateContent, +) +from ..services.insee_document import get_insee_document_service def register_get_insee_document(mcp: FastMCP) -> None: - @mcp.tool( - name=GET_DOCUMENT["tool_name"], - description=GET_DOCUMENT["tool_description"], - meta=GET_DOCUMENT["tool_metadata"], - ) - async def get_insee_documents( - params: GetInseeDocumentInput, + @mcp.tool + async def get_insee_document( ctx: Context, - ) -> GetInseeDocumentOutput: - # Fixme: use singular or plural and stick to it - return await get_insee_document(params, http_client=get_insee_http_client(ctx)) + document_urls: DocumentUrls, + include_table_of_contents: IncludeTableOfContents = True, + truncate_content: TruncateContent = True, + ) -> DocumentContentOutput: + """Fetch and parse INSEE publications from known URLs and return their full text in markdown. + + Every per-URL entry carries the same keys whether it succeeded or failed, so results can be + iterated without type-sniffing. + """ + return await get_insee_document_service( + document_urls=document_urls, + include_table_of_contents=include_table_of_contents, + truncate_content=truncate_content, + http_client=get_insee_http_client(ctx), + ) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py index fd1b74f..2af6dde 100644 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ b/src/mcpdiffusion/tools/insee_get_homepage.py @@ -2,17 +2,21 @@ from fastmcp import FastMCP -from ..config.tool_metadata import GET_HOMEPAGE from ..data.indicators import KEY_INDICATORS from ..models.insee import KeyIndicatorsOutput from ..services.insee_indicators import build_key_indicators +# Business rule: the docstring below calls the figures "latest" and the instructions make this tool the +# preferred FIRST step, but they are frozen literals (see data/indicators.py). Whether the wording softens +# or the data becomes live is the same decision. Left as-is deliberately. def register_get_insee_homepage(mcp: FastMCP) -> None: - @mcp.tool( - name=GET_HOMEPAGE["tool_name"], - description=GET_HOMEPAGE["tool_description"], - meta=GET_HOMEPAGE["tool_metadata"], - ) + @mcp.tool def get_insee_homepage() -> KeyIndicatorsOutput: + """Retrieve the INSEE home page with the latest key indicators at national level published by + the institute (population, inflation, unemployment, GDP growth, ...). + + Each indicator carries a `value` that is a full sentence in French stating the figure and the + period it covers. + """ return build_key_indicators(KEY_INDICATORS) diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index efc3c83..a28a546 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -6,12 +6,17 @@ from elasticsearch import TransportError from fastmcp import Context, FastMCP -from ..config.tool_metadata import SEARCH_CHIFFRECLEF from ..core.errors import AppToolError from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( - SearchInseeChiffrecleInput, - SearchInseeChiffrecleOutput, + DEFAULT_RESULT_COUNT, + DocumentSearchOutput, + GeoKeyword, + GeoLevel, + GeoLevelChoice, + NumberOfResults, + Query, + YearOfReference, ) from ..services.insee_search import ( apply_collection_filters, @@ -22,30 +27,33 @@ # Fixme: the orchestration present in that function belongs in a service # Indeed, the approach from one tool to another is inconsistent -def register_search_insee_chiffreclef(mcp: FastMCP, *, index: str) -> None: - @mcp.tool( - name=SEARCH_CHIFFRECLEF["tool_name"], - description=SEARCH_CHIFFRECLEF["tool_description"], - meta=SEARCH_CHIFFRECLEF["tool_metadata"], - ) +def register_search_insee_chiffrecle(mcp: FastMCP, *, index: str) -> None: + @mcp.tool async def search_insee_chiffrecle( - params: SearchInseeChiffrecleInput, ctx: Context, - ) -> SearchInseeChiffrecleOutput: + query: Query, + year_of_reference: YearOfReference = None, + geo_level: GeoLevel = GeoLevelChoice.FRANCE, + geo_keyword: GeoKeyword = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + ) -> DocumentSearchOutput: + """Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, comparaisons + regionales/departementales et statistiques factuelles simples. + + Retourne directement les tableaux synthetiques prets a l'emploi. + """ must, filters, should = build_text_clauses( - query=params.query, - year_of_reference=params.year_of_reference, + query=query, + year_of_reference=year_of_reference, ) - - # Fixme: should is overridden here filters, collection_should = apply_collection_filters( filters, must_not_rapides=True, must_only_rapides=False, chiffre_clef=True, theme=None, - geo_niveau=params.geo_niveau, - geo_keyword=params.geo_keyword, + geo_level=geo_level, + geo_keyword=geo_keyword, ) try: hits = await execute_search( @@ -53,7 +61,7 @@ async def search_insee_chiffrecle( filters=filters, should=should + collection_should, minimum_should_match=1 if collection_should else 0, - number_of_results=params.number_of_results, + number_of_results=number_of_results, es=get_elasticsearch_client(ctx), index=index, ) @@ -63,4 +71,4 @@ async def search_insee_chiffrecle( f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) - return SearchInseeChiffrecleOutput(results=hits, count=len(hits)) + return DocumentSearchOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index deca8e1..32b7a97 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -7,13 +7,16 @@ from elasticsearch.dsl import Q from fastmcp import Context, FastMCP -from ..config.tool_metadata import SEARCH_CONJONCTURE from ..core.errors import AppToolError from ..data.themes import DICT_THEME_CONJ from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( - SearchInseeConjonctureInput, - SearchInseeConjonctureOutput, + DEFAULT_RESULT_COUNT, + ConjonctureQuery, + ConjonctureYearOfReference, + DocumentSearchOutput, + NumberOfResults, + ThemeConjoncture, ) from ..services.insee_search import ( apply_collection_filters, @@ -24,18 +27,24 @@ # Fixme: again, a lot of code in that tool that should belong in the service def register_search_insee_conjoncture(mcp: FastMCP, *, index: str) -> None: - @mcp.tool( - name=SEARCH_CONJONCTURE["tool_name"], - description=SEARCH_CONJONCTURE["tool_description"], - meta=SEARCH_CONJONCTURE["tool_metadata"], - ) + @mcp.tool async def search_insee_conjoncture( - params: SearchInseeConjonctureInput, ctx: Context, - ) -> SearchInseeConjonctureOutput: + query: ConjonctureQuery, + theme_conjoncture: ThemeConjoncture = None, + year_of_reference: ConjonctureYearOfReference = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + ) -> DocumentSearchOutput: + """Search INSEE Rapid Releases (Informations rapides): short, recurring publications reporting + the latest monthly/quarterly/annual results for major economic and social indicators (prices, + employment, production, housing, wages, national accounts, ...). + + The search is lexical and rewards keyword breadth, so provide several synonyms and related + notions. + """ must, filters, should = build_text_clauses( - query=params.query, - year_of_reference=params.year_of_reference, + query=query, + year_of_reference=year_of_reference, ) filters, collection_should = apply_collection_filters( filters, @@ -43,10 +52,10 @@ async def search_insee_conjoncture( must_not_rapides=False, must_only_rapides=True, ) - if params.theme_conjoncture: - subthemes = DICT_THEME_CONJ.get(params.theme_conjoncture) + if theme_conjoncture: + subthemes = DICT_THEME_CONJ.get(theme_conjoncture) # Business rule: an unrecognised subtheme drops the filter silently and returns everything, - # the same shape as the theme and geo_niveau filters. + # the same shape as the theme and geo_level filters. if subthemes: filters.append(Q("terms", conjoncture_libelle=subthemes)) @@ -56,7 +65,7 @@ async def search_insee_conjoncture( filters=filters, should=should + collection_should, minimum_should_match=1 if collection_should else 0, - number_of_results=params.number_of_results, + number_of_results=number_of_results, es=get_elasticsearch_client(ctx), index=index, ) @@ -66,4 +75,4 @@ async def search_insee_conjoncture( f"INSEE conjoncture search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) - return SearchInseeConjonctureOutput(results=hits, count=len(hits)) + return DocumentSearchOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index 6d6cbf9..b1b325b 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -6,12 +6,19 @@ from elasticsearch import TransportError from fastmcp import Context, FastMCP -from ..config.tool_metadata import SEARCH_DOCUMENTS from ..core.errors import AppToolError from ..infra.elasticsearch import get_elasticsearch_client from ..models.insee import ( - SearchInseeDocumentsInput, - SearchInseeDocumentsOutput, + DEFAULT_RESULT_COUNT, + DocumentSearchOutput, + GeoKeyword, + GeoLevel, + GeoLevelChoice, + NumberOfResults, + Query, + Theme, + ThemeChoice, + YearOfReference, ) from ..services.insee_search import ( apply_collection_filters, @@ -23,27 +30,35 @@ # Fixme: state clear conventions between what goes to a tool and what do not # most of the code might belong in the service def register_search_insee_documents(mcp: FastMCP, *, index: str) -> None: - @mcp.tool( - name=SEARCH_DOCUMENTS["tool_name"], - description=SEARCH_DOCUMENTS["tool_description"], - meta=SEARCH_DOCUMENTS["tool_metadata"], - ) + @mcp.tool async def search_insee_documents( - params: SearchInseeDocumentsInput, ctx: Context, - ) -> SearchInseeDocumentsOutput: + query: Query, + theme: Theme = ThemeChoice.ALL, + year_of_reference: YearOfReference = None, + geo_level: GeoLevel = GeoLevelChoice.FRANCE, + geo_keyword: GeoKeyword = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + ) -> DocumentSearchOutput: + """Search the INSEE catalogue of official statistical publications (Insee Premiere, Insee + Analyses, Dossiers, References, Focus, ...). Returns structured publication records; pass the + URL of a record to `get_insee_document` to fetch the full text. + + Write a rich natural-language query with synonyms, context, and the target year or geography + when relevant. For 'essentiel sur...' publications prefer `search_insee_chiffrecle`. + """ must, filters, should = build_text_clauses( - query=params.query, - year_of_reference=params.year_of_reference, + query=query, + year_of_reference=year_of_reference, ) filters, collection_should = apply_collection_filters( filters, must_not_rapides=True, must_only_rapides=False, chiffre_clef=False, - theme=params.theme, - geo_niveau=params.geo_niveau, - geo_keyword=params.geo_keyword, + theme=theme, + geo_level=geo_level, + geo_keyword=geo_keyword, ) try: hits = await execute_search( @@ -51,7 +66,7 @@ async def search_insee_documents( filters=filters, should=should + collection_should, minimum_should_match=1 if collection_should else 0, - number_of_results=params.number_of_results, + number_of_results=number_of_results, es=get_elasticsearch_client(ctx), index=index, ) @@ -61,4 +76,4 @@ async def search_insee_documents( f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", retryable=True, ) - return SearchInseeDocumentsOutput(results=hits, count=len(hits)) + return DocumentSearchOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py index def3b5e..dd71136 100644 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ b/src/mcpdiffusion/tools/melodi_get_observations.py @@ -4,21 +4,37 @@ from fastmcp import Context, FastMCP -from ..config.tool_metadata import GET_DATASET from ..infra.http import get_melodi_http_client -from ..models.melodi import GetMelodiObservationsInput, GetMelodiObservationsOutput -from ..services.melodi import get_melodi_observations +from ..models.melodi import ( + ColumnFilters, + DatasetId, + NumberOfObservations, + ObservationsOutput, + Years, +) +from ..services.melodi import get_melodi_observations_service def register_get_melodi_observations(mcp: FastMCP) -> None: - @mcp.tool( - # Fixme: 'GET_DATASET' as variable name is too broad, thus misleading - name=GET_DATASET["tool_name"], - description=GET_DATASET["tool_description"], - meta=GET_DATASET["tool_metadata"], - ) - async def get_melodi_observations_tool( - params: GetMelodiObservationsInput, + @mcp.tool + async def get_melodi_observations( ctx: Context, - ) -> GetMelodiObservationsOutput: - return await get_melodi_observations(params, http_client=get_melodi_http_client(ctx)) + dataset_id: DatasetId, + years: Years, + column_filters: ColumnFilters, + number_of_observations: NumberOfObservations = 100, + ) -> ObservationsOutput: + """Retrieve a filtered set of observations from a Melodi dataset. The Melodi API holds official, + high-granularity statistics (prices, mortality, names, etc.). + + Observations carry dimensions, attributes and the numeric measure with its unit. An empty + list means no rows matched; a structured error means the upstream API failed or the inputs + were invalid. + """ + return await get_melodi_observations_service( + dataset_id=dataset_id, + years=years, + column_filters=column_filters, + number_of_observations=number_of_observations, + http_client=get_melodi_http_client(ctx), + ) diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py index 5942ddd..fe6f0aa 100644 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ b/src/mcpdiffusion/tools/melodi_search_datasets.py @@ -4,24 +4,38 @@ from fastmcp import Context, FastMCP -from ..config.tool_metadata import SEARCH_DATASET from ..infra.elasticsearch import get_elasticsearch_client -from ..models.melodi import SearchMelodiDatasetsInput, SearchMelodiDatasetsOutput -from ..services.melodi import search_melodi_datasets +from ..models.melodi import ( + DatasetQuery, + DatasetsOutput, + EndYear, + NumberOfDatasets, + StartYear, +) +from ..services.melodi import search_melodi_datasets_service def register_search_melodi_datasets(mcp: FastMCP, *, index: str) -> None: - @mcp.tool( - name=SEARCH_DATASET["tool_name"], - description=SEARCH_DATASET["tool_description"], - meta=SEARCH_DATASET["tool_metadata"], - ) - async def search_melodi_datasets_tool( - params: SearchMelodiDatasetsInput, + @mcp.tool + async def search_melodi_datasets( ctx: Context, - ) -> SearchMelodiDatasetsOutput: - return await search_melodi_datasets( - params, + query: DatasetQuery, + start_year: StartYear = 1900, + end_year: EndYear = 2100, + number_of_datasets: NumberOfDatasets = 5, + ) -> DatasetsOutput: + """Search the INSEE Melodi dataset catalogue by French-language natural language query. Each + dataset has a unique `dataset_id`; the tool maps the query to internal metadata to return the + most relevant matches. + + Matching is lexical, so make the query explicit and rich in French synonyms, e.g. + `"indice des prix a la consommation"`, `"deces par departement"`, `"prenoms des nouveau-nes"`. + """ + return await search_melodi_datasets_service( + query=query, + start_year=start_year, + end_year=end_year, + number_of_datasets=number_of_datasets, es=get_elasticsearch_client(ctx), index=index, ) diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py index 5ef2643..f88977b 100644 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ b/src/mcpdiffusion/tools/melodi_search_modalities.py @@ -4,24 +4,38 @@ from fastmcp import Context, FastMCP -from ..config.tool_metadata import SEARCH_MODALITIES from ..infra.elasticsearch import get_elasticsearch_client -from ..models.melodi import SearchMelodiModalitiesInput, SearchMelodiModalitiesOutput -from ..services.melodi import search_melodi_modalities +from ..models.melodi import ( + ColumnIds, + DatasetId, + ModalitiesOutput, + ModalityQuery, + NumberOfModalities, +) +from ..services.melodi import search_melodi_modalities_service def register_search_melodi_modalities(mcp: FastMCP, *, index: str) -> None: - @mcp.tool( - name=SEARCH_MODALITIES["tool_name"], - description=SEARCH_MODALITIES["tool_description"], - meta=SEARCH_MODALITIES["tool_metadata"], - ) - async def search_melodi_modalities_tool( - params: SearchMelodiModalitiesInput, + @mcp.tool + async def search_melodi_modalities( ctx: Context, - ) -> SearchMelodiModalitiesOutput: - return await search_melodi_modalities( - params, + dataset_id: DatasetId, + column_ids: ColumnIds, + query: ModalityQuery, + number_of_modalities: NumberOfModalities = 10, + ) -> ModalitiesOutput: + """Given a Melodi dataset and one or more column identifiers, rank the most relevant modalities + (codes/labels) for a free-text French query. The result is what you need to filter rows in + `get_melodi_observations`. + + Each matching column carries its `code`, its metadata text and the top-scoring + `matching_modalities`. An empty list means nothing matched. + """ + return await search_melodi_modalities_service( + dataset_id=dataset_id, + column_ids=column_ids, + query=query, + number_of_modalities=number_of_modalities, es=get_elasticsearch_client(ctx), index=index, ) diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index 7cc2c0a..d30ca01 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -1,24 +1,30 @@ -"""Tool: RMES_describe_resource -- thin registration layer.""" +"""Tool: describe_rmes_resource -- thin registration layer.""" from __future__ import annotations from fastmcp import Context, FastMCP -from ..config.tool_metadata import RMES_DESCRIBE_RESOURCE from ..infra.sparql import get_sparql_http_client -from ..models.rmes import DescribeResourceInput, DescribeResourceOutput -from ..services.rmes import describe_resource +from ..models.rmes import GraphUri, ResourceOutput, ResourceUri +from ..services.rmes import describe_rmes_resource_service -def register_rmes_describe_resource(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool( - name=RMES_DESCRIBE_RESOURCE["tool_name"], - description=RMES_DESCRIBE_RESOURCE["tool_description"], - meta=RMES_DESCRIBE_RESOURCE["tool_metadata"], - ) - async def describe_resource_tool(params: DescribeResourceInput, ctx: Context) -> DescribeResourceOutput: - return await describe_resource( - params, +def register_describe_rmes_resource(mcp: FastMCP, *, endpoint: str) -> None: + @mcp.tool + async def describe_rmes_resource( + ctx: Context, + resource_uri: ResourceUri, + graph_uri: GraphUri = None, + ) -> ResourceOutput: + """Recupere toutes les proprietes connues (predicat -> valeur) d'une ressource RDF identifiee + par son URI complete. Combine automatiquement les proprietes ou la ressource est sujet ET + celles ou elle est objet (utile pour remonter des relations skos:broader par exemple). + Restreins avec `graph_uri` si tu sais deja ou chercher -- sinon la recherche se fait sur tous les + graphes, ce qui est plus lent. + """ + return await describe_rmes_resource_service( + resource_uri=resource_uri, + graph_uri=graph_uri, sparql_client=get_sparql_http_client(ctx), endpoint=endpoint, ) diff --git a/src/mcpdiffusion/tools/rmes_list_graphs.py b/src/mcpdiffusion/tools/rmes_list_graphs.py deleted file mode 100644 index b293f59..0000000 --- a/src/mcpdiffusion/tools/rmes_list_graphs.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Tool: RMES_list_graphs -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..config.tool_metadata import RMES_LIST_GRAPHS -from ..infra.sparql import get_sparql_http_client -from ..models.rmes import ListGraphsInput, ListGraphsOutput -from ..services.rmes import list_graphs - - -def register_rmes_list_graphs(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool( - name=RMES_LIST_GRAPHS["tool_name"], - description=RMES_LIST_GRAPHS["tool_description"], - meta=RMES_LIST_GRAPHS["tool_metadata"], - ) - async def list_graphs_tool(params: ListGraphsInput, ctx: Context) -> ListGraphsOutput: - return await list_graphs( - params, - sparql_client=get_sparql_http_client(ctx), - endpoint=endpoint, - ) diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index e5918d6..99ac74a 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -1,39 +1,72 @@ -"""Tool: RMES_run_sparql -- thin registration layer.""" +"""Tool: run_rmes_sparql -- thin registration layer.""" from __future__ import annotations from fastmcp import Context, FastMCP -from ..config.tool_metadata import RMES_RUN_SPARQL from ..infra.sparql import get_sparql_http_client -from ..models.rmes import RunSparqlInput, RunSparqlOutput -from ..services.rmes import KNOWN_VOCABULARIES_NOTE, run_sparql - - -def register_rmes_run_sparql(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool( - name=RMES_RUN_SPARQL["tool_name"], - # Fixme: this description combinaison involves too many sources of data, the metadata file, - # the service plus a hardoded description - # Fixme: this is also not the right place for a query - description=RMES_RUN_SPARQL["tool_description"] + "\n" + KNOWN_VOCABULARIES_NOTE + "\n\n" - 'Exemple -- recherche de codes NAF contenant "extraction" :\n' - "PREFIX skos: \n" - "SELECT ?s ?label WHERE {\n" - " GRAPH {\n" - " ?s skos:prefLabel ?label .\n" - ' FILTER(lang(?label) = "fr")\n' - ' FILTER(CONTAINS(LCASE(STR(?label)), "extraction"))\n' - " }\n" - "} LIMIT 10\n" - "\n" - 'Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format="turtle"`, champ `turtle`) ' - 'plutot que des lignes (`format="json"`, champs `variables`/`bindings`).', - meta=RMES_RUN_SPARQL["tool_metadata"], - ) - async def run_sparql_tool(params: RunSparqlInput, ctx: Context) -> RunSparqlOutput: - return await run_sparql( - params, +from ..models.rmes import ( + DEFAULT_QUERY_TIMEOUT_SECONDS, + DEFAULT_ROW_LIMIT, + MaxRows, + SparqlOutput, + SparqlQuery, + TimeoutSeconds, +) +from ..services.rmes import run_rmes_sparql_service + + +def register_run_rmes_sparql(mcp: FastMCP, *, endpoint: str) -> None: + @mcp.tool + async def run_rmes_sparql( + ctx: Context, + sparql_query: SparqlQuery, + timeout_seconds: TimeoutSeconds = DEFAULT_QUERY_TIMEOUT_SECONDS, + max_rows: MaxRows = DEFAULT_ROW_LIMIT, + ) -> SparqlOutput: + """Execute une requete SPARQL libre sur RMES, la base de metadonnees, nomenclatures et + definitions de l'INSEE (elle ne contient PAS les chiffres/donnees, voir les tools MELODI + pour ca). + + Bonnes pratiques : + - Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou + VALUES ?g { } plutot que de scanner tous les graphes. + - Toujours ajouter FILTER(lang(?label) = "fr") sur les litteraux SKOS pour eviter les + doublons multilingues. + - Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee + automatiquement (indique dans la reponse via `limit_added`/`hint`). + - Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures + statistiques : ClassificationLevel, ExplanatoryNote), dcterms (metadonnees), + rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE). + + Vocabulaires principaux rencontres dans cette base (au-dela de skos/xkos/dcterms) : + - sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualite. Un sdmx-mm:MetadataReport + a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des + sdmx-mm:ReportedAttribute rattaches via sdmx-mm:metadataReport. + - rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, + StatisticalOperationSeries, StatisticalOperationFamily (graphe "operations"), + StatisticalIndicator (graphe "produits"), StatutDiffusion... + - org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes + "organisations" et "organisations/insee"). + - dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). + + Exemple -- recherche de codes NAF contenant "extraction" : + PREFIX skos: + SELECT ?s ?label WHERE { + GRAPH { + ?s skos:prefLabel ?label . + FILTER(lang(?label) = "fr") + FILTER(CONTAINS(LCASE(STR(?label)), "extraction")) + } + } LIMIT 10 + + Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format="turtle"`, champ `turtle`) + plutot que des lignes (`format="json"`, champs `variables`/`bindings`). + """ + return await run_rmes_sparql_service( + sparql_query=sparql_query, + timeout_seconds=timeout_seconds, + max_rows=max_rows, sparql_client=get_sparql_http_client(ctx), endpoint=endpoint, ) diff --git a/src/mcpdiffusion/tools/rmes_search_graphs.py b/src/mcpdiffusion/tools/rmes_search_graphs.py new file mode 100644 index 0000000..a3a77ec --- /dev/null +++ b/src/mcpdiffusion/tools/rmes_search_graphs.py @@ -0,0 +1,40 @@ +"""Tool: search_rmes_graphs -- thin registration layer.""" + +from __future__ import annotations + +from fastmcp import Context, FastMCP + +from ..infra.sparql import get_sparql_http_client +from ..models.rmes import ( + ExpandGraphs, + GraphCategory, + GraphCategoryChoice, + GraphsOutput, + GraphUriSubstring, +) +from ..services.rmes import search_rmes_graphs_service + + +def register_search_rmes_graphs(mcp: FastMCP, *, endpoint: str) -> None: + @mcp.tool + async def search_rmes_graphs( + ctx: Context, + graph_uri_substring: GraphUriSubstring = None, + graph_category: GraphCategory = GraphCategoryChoice.ALL, + expand_graphs: ExpandGraphs = False, + ) -> GraphsOutput: + """Liste les graphes nommes disponibles dans la base RDF de l'INSEE (RMES). + + Par defaut (`graph_category=ALL`), le resultat est une vue CONDENSEE par categorie, avec un + compteur et quelques URIs d'exemple par categorie -- pas la liste plate des 700+ graphes. + Choisis une categorie precise dans le parametre `graph_category` pour cibler une famille, ou + utilise `graph_uri_substring` pour une recherche libre par sous-chaine. Une categorie "autre" recueille + tout graphe ne correspondant a aucune famille connue. + """ + return await search_rmes_graphs_service( + graph_uri_substring=graph_uri_substring, + graph_category=graph_category, + expand_graphs=expand_graphs, + sparql_client=get_sparql_http_client(ctx), + endpoint=endpoint, + ) From 52ca3967de32e88d2d3d65a3e5e1c2f18e5b8f02 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 07:49:16 +0200 Subject: [PATCH 18/55] build: exempt the FastMCP dependency markers from B008 `Depends()` and `CurrentContext()` appear as argument defaults, which B008 flags because a shared mutable default would leak between calls. These build a dependency marker rather than a value: FastMCP resolves them per request, so the default is never what the function receives. Configured once here rather than repeating a `noqa` at every injection site. --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b71f74a..afb6ccd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,15 @@ select = [ "UP", # pyupgrade - enforces modern syntax, including `str | None` over `Optional[str]` ] +[tool.ruff.lint.flake8-bugbear] +# B008 forbids calls in argument defaults, because a shared mutable would leak between calls. +# These two build a dependency marker, not a value: FastMCP resolves them per request, so the +# default is never what the function receives. +extend-immutable-calls = [ + "fastmcp.dependencies.Depends", + "fastmcp.dependencies.CurrentContext", +] + [tool.ruff.lint.per-file-ignores] # Long literal statistics, pending replacement by a live source. "src/mcpdiffusion/data/indicators.py" = ["E501"] From efa39aaccd8bd2665d299876049202707d76a5c4 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 07:49:35 +0200 Subject: [PATCH 19/55] refactor(melodi): inject Melodi's backend services into the tools Melodi's data access lived in one module that the tools wired by hand: each tool pulled a raw client out of the lifespan context, passed an index name down on every call, and translated backend failures itself. Each backend now sits behind a service, built once at startup with its client and index already bound: services/melodi/index_service.py MelodiIndexService -- Elasticsearch services/melodi/api_service.py MelodiApiService -- Melodi REST API services/elasticsearch_failures.py one failure boundary for both searches Tools declare the service they need and FastMCP injects it, so no Melodi tool imports `elasticsearch` or `httpx` any more, and none carries an index name. Query construction moves onto the `elasticsearch.dsl` builders insee already uses; every generated body is identical to the raw dict it replaces, except that the DSL omits an empty `filter` clause, which is a no-op in Elasticsearch. The tool contract -- names, input schemas, descriptions, output schemas -- is byte-identical to the previous commit. Also corrects four failures the caller could not act on: - Elasticsearch `ApiError` escaped the boundary entirely, because it does not subclass `TransportError`. A missing index surfaced as a generic error. It is now mapped by status and marked non-retryable, so the model reports it instead of rephrasing the query. - The Elasticsearch host and port travelled to the client inside the exception message. Only the exception type is sent now; the full cause stays in the server log. - Melodi answers HTTP 400 for an unknown dataset, column and modality alike, yet the message only suggested checking modality codes. It now names every remedy and quotes the upstream detail that distinguishes them. - Values were forced through `str()`, so a field arriving as a list became the literal "['GEO', 'SEX']". Malformed data now fails validation rather than reaching the caller as text. --- src/mcpdiffusion/infra/dependencies.py | 21 ++ src/mcpdiffusion/infra/lifespan.py | 16 + src/mcpdiffusion/server.py | 2 + .../services/elasticsearch_failures.py | 67 ++++ src/mcpdiffusion/services/melodi.py | 301 ------------------ src/mcpdiffusion/services/melodi/__init__.py | 0 .../services/melodi/api_service.py | 84 +++++ .../services/melodi/index_service.py | 244 ++++++++++++++ src/mcpdiffusion/tools/__init__.py | 22 +- src/mcpdiffusion/tools/melodi/__init__.py | 0 .../tools/melodi/get_observations_tool.py | 64 ++++ .../tools/melodi/search_datasets_tool.py | 38 +++ .../tools/melodi/search_modalities_tool.py | 38 +++ .../tools/melodi_get_observations.py | 40 --- .../tools/melodi_search_datasets.py | 41 --- .../tools/melodi_search_modalities.py | 41 --- 16 files changed, 586 insertions(+), 433 deletions(-) create mode 100644 src/mcpdiffusion/infra/dependencies.py create mode 100644 src/mcpdiffusion/services/elasticsearch_failures.py delete mode 100644 src/mcpdiffusion/services/melodi.py create mode 100644 src/mcpdiffusion/services/melodi/__init__.py create mode 100644 src/mcpdiffusion/services/melodi/api_service.py create mode 100644 src/mcpdiffusion/services/melodi/index_service.py create mode 100644 src/mcpdiffusion/tools/melodi/__init__.py create mode 100644 src/mcpdiffusion/tools/melodi/get_observations_tool.py create mode 100644 src/mcpdiffusion/tools/melodi/search_datasets_tool.py create mode 100644 src/mcpdiffusion/tools/melodi/search_modalities_tool.py delete mode 100644 src/mcpdiffusion/tools/melodi_get_observations.py delete mode 100644 src/mcpdiffusion/tools/melodi_search_datasets.py delete mode 100644 src/mcpdiffusion/tools/melodi_search_modalities.py diff --git a/src/mcpdiffusion/infra/dependencies.py b/src/mcpdiffusion/infra/dependencies.py new file mode 100644 index 0000000..be35651 --- /dev/null +++ b/src/mcpdiffusion/infra/dependencies.py @@ -0,0 +1,21 @@ +"""Typed access to the objects the lifespan built, declared as FastMCP dependencies. + +A tool asks for what it needs in its signature; FastMCP resolves it per request and hides the +parameter from the tool schema, so the LLM never sees it. +""" + +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from ..services.melodi.api_service import MelodiApiService +from ..services.melodi.index_service import MelodiIndexService + + +def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: + """Return the Melodi Elasticsearch service built at startup.""" + return ctx.lifespan_context["melodi_index_service"] + + +def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: + """Return the Melodi REST API service built at startup.""" + return ctx.lifespan_context["melodi_api_service"] diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/infra/lifespan.py index 48774f7..52476cf 100644 --- a/src/mcpdiffusion/infra/lifespan.py +++ b/src/mcpdiffusion/infra/lifespan.py @@ -8,6 +8,9 @@ from fastmcp.server.lifespan import lifespan from httpx import AsyncClient, Timeout +from ..services.melodi.api_service import MelodiApiService +from ..services.melodi.index_service import MelodiIndexService + logger = logging.getLogger(__name__) # insee.fr serves different markup to unknown agents, so the scraper has to look like a browser. @@ -30,6 +33,8 @@ def build_lifespan( melodi_data_base_url: str, melodi_request_timeout_seconds: int, melodi_connect_timeout_seconds: int, + melodi_datasets_index: str, + melodi_columns_index: str, ) -> Callable[..., Any]: @lifespan @@ -69,12 +74,23 @@ async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: ) logger.info("SPARQL client initialized") + # Services bind a client to its index or base URL once, so nothing downstream has to + # carry an index name around. They hold no request state, so one instance serves every call. + melodi_index_service = MelodiIndexService( + elasticsearch_client=elasticsearch_client, + datasets_index=melodi_datasets_index, + columns_index=melodi_columns_index, + ) + melodi_api_service = MelodiApiService(http_client=melodi_http_client) + try: yield { "elasticsearch_client": elasticsearch_client, "insee_http_client": insee_http_client, "melodi_http_client": melodi_http_client, "sparql_http_client": sparql_http_client, + "melodi_index_service": melodi_index_service, + "melodi_api_service": melodi_api_service, } finally: await elasticsearch_client.close() diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index fa1501c..128e6be 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -42,6 +42,8 @@ melodi_data_base_url=settings.melodi_data_base_url, melodi_request_timeout_seconds=settings.melodi_request_timeout_seconds, melodi_connect_timeout_seconds=settings.melodi_connect_timeout_seconds, + melodi_datasets_index=settings.es_index_melodi_datasets, + melodi_columns_index=settings.es_index_melodi_columns, ), ) diff --git a/src/mcpdiffusion/services/elasticsearch_failures.py b/src/mcpdiffusion/services/elasticsearch_failures.py new file mode 100644 index 0000000..c2a7ae1 --- /dev/null +++ b/src/mcpdiffusion/services/elasticsearch_failures.py @@ -0,0 +1,67 @@ +"""The single place Elasticsearch failures become errors the caller can act on. + +Two disjoint exception families reach here, and both have to be named: + +- `elastic_transport.TransportError` -- the request never got a usable answer (refused, timed + out, TLS, unparseable). `ConnectionError` and `ConnectionTimeout` are subclasses. +- `elasticsearch.ApiError` -- Elasticsearch answered, with an error status. `NotFoundError` + (a missing index) and `BadRequestError` are subclasses. + +`ApiError` does not inherit from `TransportError`, so catching one never catches the other. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from http import HTTPStatus + +from elasticsearch import ApiError, TransportError + +from ..core.errors import AppToolError + + +@asynccontextmanager +async def elasticsearch_failures_as_tool_errors(backend_label: str) -> AsyncIterator[None]: + """Translate a failed Elasticsearch search into an `AppToolError` naming the backend. + + Wraps the `await` rather than performing it, so it fits both the DSL and the raw client. + Only the short error type is quoted: the full body belongs in the server log, not in a + message sent to the caller. + """ + try: + yield + except TransportError as exc: + # Only the exception type, never its message: with retries enabled the message carries + # the host and port, and that must not leave the process. The full cause, host included, + # is in the server log via ErrorHandlingMiddleware. + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"{backend_label} search backend unreachable ({type(exc).__name__}). Verify ES_HOST and try again.", + retryable=True, + ) + except ApiError as exc: + status = exc.status_code + if status == HTTPStatus.NOT_FOUND: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"The {backend_label} index is missing from Elasticsearch ({exc.error}). " + "The index is not loaded on the server, so no query against it can succeed. " + "Rephrasing will not help -- report this instead of retrying.", + ) + if status == HTTPStatus.BAD_REQUEST: + raise AppToolError( + "INVALID_QUERY", + f"Elasticsearch rejected the {backend_label} search as malformed " + f"({exc.error}). This is a defect in the server's query, not in the arguments " + "you passed. Report it instead of retrying.", + ) + if status in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN): + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Elasticsearch refused the {backend_label} search ({exc.error}). The server's " + "credentials are missing or insufficient. Report it instead of retrying.", + ) + raise AppToolError( + "UPSTREAM_ERROR", + f"Elasticsearch returned HTTP {status} for the {backend_label} search ({exc.error}).", + retryable=status >= HTTPStatus.INTERNAL_SERVER_ERROR, + ) diff --git a/src/mcpdiffusion/services/melodi.py b/src/mcpdiffusion/services/melodi.py deleted file mode 100644 index ef50c0c..0000000 --- a/src/mcpdiffusion/services/melodi.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Business logic for Melodi tools (observations, datasets, modalities).""" - -from __future__ import annotations - -from typing import Any - -import httpx -from elasticsearch import AsyncElasticsearch, TransportError -from elasticsearch import ConnectionError as ESConnectionError - -from ..core.errors import AppToolError -from ..models.melodi import ( - ColumnResult, - DatasetSearchResult, - DatasetsOutput, - ModalitiesOutput, - Modality, - ObservationsOutput, -) - - -async def get_melodi_observations_service( - dataset_id: str, - years: list[int], - column_filters: dict[str, str], - number_of_observations: int, - *, - http_client: httpx.AsyncClient, -) -> ObservationsOutput: - # Resolved against the client's base_url. - url = f"/{dataset_id}" - try: - response = await http_client.get( - url, - params=column_filters or None, - ) - response.raise_for_status() - # Fixme: the following problematic error handling pattern has already been adressed - except httpx.TimeoutException as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Melodi API timed out calling {url}: {exc}. Try again or narrow the query.", - retryable=True, - ) - except httpx.HTTPStatusError as exc: - status = exc.response.status_code - body_excerpt = (exc.response.text or "")[:500] - if status == 400: - raise AppToolError( - "INVALID_INPUT", - f"Melodi API rejected the query (HTTP 400). " - f"Columns/values passed: {column_filters}. " - f"Upstream detail: {body_excerpt}. " - "Verify modality codes with `search_melodi_modalities`.", - ) - elif status == 404: - raise AppToolError( - "NOT_FOUND", - f"Melodi dataset {dataset_id!r} not found (HTTP 404). " - "Check the dataset_id with `search_melodi_datasets`.", - ) - else: - raise AppToolError( - "UPSTREAM_ERROR", - f"Melodi API returned HTTP {status}: {body_excerpt}", - retryable=(500 <= status < 600), - ) - except httpx.HTTPError as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Could not reach Melodi API at {url}: {exc}", - retryable=True, - ) - - try: - payload = response.json() - except ValueError as exc: - raise AppToolError( - "PARSE_ERROR", - f"Melodi API returned non-JSON response: {exc}", - ) - - observations = payload.get("observations") if isinstance(payload, dict) else None - if not isinstance(observations, list): - raise AppToolError( - "PARSE_ERROR", - "Melodi API response did not contain an 'observations' list.", - ) - - if years: - years_str = {str(y) for y in years} - # Fixme: it seems we retrieve all the observations data and filter next - # I wonder whether the API supports filtering - observations = [ - obs - for obs in observations - # Fixme: 'TIME_PERIOD' could be sanitized - if (obs.get("dimensions", {}).get("TIME_PERIOD", "").split("-")[0]) in years_str - ] - - sliced = observations[:number_of_observations] - return ObservationsOutput( - dataset_id=dataset_id, - observations=sliced, - count=len(sliced), - ) - - -# Fixme: I believe this is not the correct place (inside the service) to place a raw complex query -# the code might benefit having a repository layer to encapsulate data access - - -async def search_melodi_datasets_service( - query: str, - start_year: int, - end_year: int, - number_of_datasets: int, - *, - es: AsyncElasticsearch, - index: str, -) -> DatasetsOutput: - filters: list[dict[str, Any]] = [] - if start_year: - filters.append({"range": {"metadata.temporal.endPeriod": {"gte": f"{start_year}-01-01"}}}) - if end_year: - filters.append({"range": {"metadata.temporal.startPeriod": {"lte": f"{end_year}-12-31"}}}) - - body = { - "size": number_of_datasets, - "query": { - "bool": { - "should": [ - { - "nested": { - "path": "metadata.title", - "query": { - "match": { - "metadata.title.content": { - "query": query, - "boost": 10, - } - } - }, - } - }, - { - "nested": { - "path": "metadata.abstract", - "query": { - "match": { - "metadata.abstract.content": { - "query": query, - "boost": 6, - } - } - }, - } - }, - { - "nested": { - "path": "metadata.description", - "query": { - "match": { - "metadata.description.content": { - "query": query, - "boost": 3, - } - } - }, - } - }, - { - "match": { - "variables_text": { - "query": query, - "boost": 5, - } - } - }, - ], - "filter": filters, - } - }, - } - - try: - ds_res = await es.search( - index=index, - body=body, - ) - except (ESConnectionError, TransportError) as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Melodi datasets search backend unreachable: {exc}. Verify ES_HOST and try again.", - retryable=True, - ) - - results: list[DatasetSearchResult] = [] - for hit in ds_res.get("hits", {}).get("hits", []): - source = hit.get("_source", {}) - description = source.get("metadata", {}).get("description") - if isinstance(description, list) and description: - description = description[0] - elif not isinstance(description, dict): - description = {"content": "", "lang": "fr"} - results.append( - DatasetSearchResult( - dataset_id=hit.get("_id", ""), - dataset_columns=source.get("columns", ""), - dataset_description=description, - dataset_score=float(hit.get("_score") or 0.0), - ) - ) - return DatasetsOutput(results=results) - - -async def search_melodi_modalities_service( - dataset_id: str, - column_ids: list[str], - query: str, - number_of_modalities: int, - *, - es: AsyncElasticsearch, - index: str, -) -> ModalitiesOutput: - filters: list[dict[str, Any]] = [{"term": {"dataset_id": dataset_id}}] - if column_ids: - filters.append({"terms": {"code": column_ids}}) - - try: - ds_column = await es.search( - index=index, - size=20, - query={ - "bool": { - "filter": filters, - "should": [ - { - "match": { - "text": { - "query": query, - "boost": 2, - } - } - }, - { - "nested": { - "path": "modalities", - "score_mode": "max", - "query": { - "multi_match": { - "query": query, - "fields": [ - "modalities.code^5", - "modalities.label.en^3", - "modalities.label.fr^3", - ], - "fuzziness": "AUTO", - } - }, - "inner_hits": { - "size": number_of_modalities, - "sort": [{"_score": "desc"}], - }, - } - }, - ], - } - }, - ) - except (ESConnectionError, TransportError) as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Melodi columns search backend unreachable: {exc}. Verify ES_HOST and try again.", - retryable=True, - ) - - results: list[ColumnResult] = [] - for hit in ds_column.get("hits", {}).get("hits", []): - modalities: list[Modality] = [] - inner_hits = hit.get("inner_hits", {}).get("modalities", {}).get("hits", {}).get("hits", []) - for m in inner_hits: - src = m.get("_source", {}) - label = src.get("label", {}) or {} - modalities.append( - Modality( - code=str(src.get("code", "")), - label_en=str(label.get("en", "")), - label_fr=str(label.get("fr", "")), - score=float(m.get("_score") or 0.0), - ) - ) - results.append( - ColumnResult( - column_code=str(hit.get("_source", {}).get("code", "")), - column_metadata=str(hit.get("_source", {}).get("text", "")), - matching_modalities=modalities, - ) - ) - - return ModalitiesOutput(results=results) diff --git a/src/mcpdiffusion/services/melodi/__init__.py b/src/mcpdiffusion/services/melodi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/services/melodi/api_service.py b/src/mcpdiffusion/services/melodi/api_service.py new file mode 100644 index 0000000..9681747 --- /dev/null +++ b/src/mcpdiffusion/services/melodi/api_service.py @@ -0,0 +1,84 @@ +"""Melodi's REST API access: the observations endpoint.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from ...core.errors import AppToolError + + +class MelodiApiService: + """Fetches observations from the Melodi REST API.""" + + def __init__(self, http_client: httpx.AsyncClient) -> None: + self._http_client = http_client + + async def fetch_observations( + self, + dataset_id: str, + column_filters: dict[str, str], + ) -> list[dict[str, Any]]: + """Return every observation the API holds for the dataset, before any year filtering.""" + # Resolved against the client's base_url. + url = f"/{dataset_id}" + try: + response = await self._http_client.get( + url, + params=column_filters or None, + ) + response.raise_for_status() + except httpx.TimeoutException as exc: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Melodi API timed out calling {url}: {exc}. Try again or narrow the query.", + retryable=True, + ) + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + body_excerpt = exc.response.text[:500].strip() + # Melodi answers 400 for an unknown dataset, an unknown column and an unknown + # modality alike, in French plain text. The prose is the only signal, and matching + # on it would break the moment it is reworded -- so name every remedy instead. + if status == httpx.codes.BAD_REQUEST: + raise AppToolError( + "INVALID_INPUT", + f'Melodi API rejected the query (HTTP 400). Upstream detail: "{body_excerpt}" ' + f"Columns/values passed: {column_filters}. " + "Confirm the dataset_id with `search_melodi_datasets`, and the column ids " + "and modality codes with `search_melodi_modalities`.", + ) + if status == httpx.codes.NOT_FOUND: + raise AppToolError( + "NOT_FOUND", + f"Melodi dataset {dataset_id!r} not found (HTTP 404). " + "Check the dataset_id with `search_melodi_datasets`.", + ) + raise AppToolError( + "UPSTREAM_ERROR", + f'Melodi API returned HTTP {status}: "{body_excerpt}"', + retryable=exc.response.is_server_error, + ) + except httpx.HTTPError as exc: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Could not reach Melodi API at {url}: {exc}", + retryable=True, + ) + + try: + payload = response.json() + except ValueError as exc: + raise AppToolError( + "PARSE_ERROR", + f"Melodi API returned non-JSON response: {exc}", + ) + + observations = payload.get("observations") if isinstance(payload, dict) else None + if not isinstance(observations, list): + raise AppToolError( + "PARSE_ERROR", + "Melodi API response did not contain an 'observations' list.", + ) + return observations diff --git a/src/mcpdiffusion/services/melodi/index_service.py b/src/mcpdiffusion/services/melodi/index_service.py new file mode 100644 index 0000000..9859d2e --- /dev/null +++ b/src/mcpdiffusion/services/melodi/index_service.py @@ -0,0 +1,244 @@ +"""Melodi's Elasticsearch access: query construction, execution and parsing. + +Elasticsearch vocabulary stops here. Tools never see a `Hit`, a `_source` or an `inner_hits` +envelope, so a change in the index shape is contained to this file. +""" + +from __future__ import annotations + +from elasticsearch import AsyncElasticsearch +from elasticsearch.dsl import AsyncSearch, Q +from elasticsearch.dsl.query import Query +from elasticsearch.dsl.response import Hit +from elasticsearch.dsl.utils import AttrList + +from ...models.melodi import ( + ColumnResult, + DatasetDescription, + DatasetSearchResult, + Modality, +) +from ..elasticsearch_failures import elasticsearch_failures_as_tool_errors + +# The column query asks for a fixed page of columns and narrows within them via inner_hits. +COLUMN_SEARCH_SIZE = 20 + + +# ---------------------------------------------------------------------------------------------------------------------- +# Query builders ------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# Pure: no client, no index, no I/O. A client-less `AsyncSearch` is valid on its own, so the body +# these produce is assertable without an Elasticsearch instance. The service binds `.using()` and +# `.index()` before executing. + + +def build_dataset_search( + query: str, + start_year: int, + end_year: int, + number_of_datasets: int, +) -> AsyncSearch: + """Rank datasets on title, abstract, description and variable text, title weighing most. + + A year of 0 means "unbounded", so it contributes no range filter. + """ + filters: list[Query] = [] + if start_year: + filters.append(Q("range", **{"metadata.temporal.endPeriod": {"gte": f"{start_year}-01-01"}})) + if end_year: + filters.append(Q("range", **{"metadata.temporal.startPeriod": {"lte": f"{end_year}-12-31"}})) + + def match_nested_content(path: str, boost: int) -> Query: + return Q( + "nested", + path=path, + query=Q("match", **{f"{path}.content": {"query": query, "boost": boost}}), + ) + + return AsyncSearch().query( + Q( + "bool", + should=[ + match_nested_content("metadata.title", 10), + match_nested_content("metadata.abstract", 6), + match_nested_content("metadata.description", 3), + Q("match", variables_text={"query": query, "boost": 5}), + ], + filter=filters, + ) + )[:number_of_datasets] + + +def build_column_search( + dataset_id: str, + column_ids: list[str], + query: str, + number_of_modalities: int, +) -> AsyncSearch: + """Find the dataset's columns whose text or modality labels match, keeping the best modalities. + + `number_of_modalities` caps the inner hits, not the columns: the page of columns is fixed. + `inner_hits` is what makes Elasticsearch report *which* nested modalities matched, and with + what score -- a plain match would only say the column matched. The typed `InnerHits` object + serialises `sort` to a string in elasticsearch 9.5.0, so the clause stays a plain dict. + """ + filters: list[Query] = [Q("term", dataset_id=dataset_id)] + if column_ids: + filters.append(Q("terms", code=column_ids)) + + return AsyncSearch().query( + Q( + "bool", + filter=filters, + should=[ + Q("match", text={"query": query, "boost": 2}), + Q( + "nested", + path="modalities", + score_mode="max", + query=Q( + "multi_match", + query=query, + fields=[ + "modalities.code^5", + "modalities.label.en^3", + "modalities.label.fr^3", + ], + fuzziness="AUTO", + ), + inner_hits={ + "size": number_of_modalities, + "sort": [ + {"_score": "desc"}, + ], + }, + ), + ], + ) + )[:COLUMN_SEARCH_SIZE] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Hit helpers ---------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def parse_first_description(dataset_hit: Hit) -> DatasetDescription: + """Return a dataset's first description, or an empty French one when it has none. + + Descriptions arrive as an array, as a single object, or not at all. The DSL wraps a JSON array + in `AttrList`, which is not a `list`, so both types have to be named or every description + parses as empty. + """ + metadata = getattr(dataset_hit, "metadata", None) + description = getattr(metadata, "description", None) if metadata is not None else None + if isinstance(description, list | AttrList): + description = description[0] if len(description) else None + if description is None: + return DatasetDescription(content="", lang="fr") + return DatasetDescription( + content=getattr(description, "content", ""), + lang=getattr(description, "lang", "fr"), + ) + + +def parse_modality(modality_hit: Hit) -> Modality: + """Map one matched nested modality, with the score that ranked it.""" + label = getattr(modality_hit, "label", None) + return Modality( + code=getattr(modality_hit, "code", ""), + label_fr=getattr(label, "fr", ""), + label_en=getattr(label, "en", ""), + # A non-scoring query reports a null score, which is not a float. + score=getattr(modality_hit.meta, "score", 0.0) or 0.0, + ) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Service -------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +class MelodiIndexService: + """Searches the two Melodi Elasticsearch indices. + + Holds the client and the index names, so nothing above has to carry an index around. + """ + + def __init__( + self, + elasticsearch_client: AsyncElasticsearch, + datasets_index: str, + columns_index: str, + ) -> None: + self._elasticsearch_client = elasticsearch_client + self._datasets_index = datasets_index + self._columns_index = columns_index + + async def search_datasets( + self, + query: str, + start_year: int, + end_year: int, + number_of_datasets: int, + ) -> list[DatasetSearchResult]: + """Return the datasets matching the query, most relevant first.""" + search = ( + build_dataset_search( + query=query, + start_year=start_year, + end_year=end_year, + number_of_datasets=number_of_datasets, + ) + .using(self._elasticsearch_client) + .index(self._datasets_index) + ) + async with elasticsearch_failures_as_tool_errors("Melodi datasets"): + response = await search.execute() + + results: list[DatasetSearchResult] = [] + for dataset_hit in response: + results.append( + DatasetSearchResult( + dataset_id=dataset_hit.meta.id, + dataset_columns=getattr(dataset_hit, "columns", ""), + dataset_description=parse_first_description(dataset_hit), + dataset_score=getattr(dataset_hit.meta, "score", 0.0) or 0.0, + ) + ) + return results + + async def search_columns( + self, + dataset_id: str, + column_ids: list[str], + query: str, + number_of_modalities: int, + ) -> list[ColumnResult]: + """Return the dataset's matching columns, each with its top-scoring modalities.""" + search = ( + build_column_search( + dataset_id=dataset_id, + column_ids=column_ids, + query=query, + number_of_modalities=number_of_modalities, + ) + .using(self._elasticsearch_client) + .index(self._columns_index) + ) + async with elasticsearch_failures_as_tool_errors("Melodi columns"): + response = await search.execute() + + results: list[ColumnResult] = [] + for column_hit in response: + inner_hits = getattr(column_hit.meta, "inner_hits", None) + modality_hits = getattr(inner_hits, "modalities", []) if inner_hits is not None else [] + results.append( + ColumnResult( + column_code=getattr(column_hit, "code", ""), + column_metadata=getattr(column_hit, "text", ""), + matching_modalities=[parse_modality(modality_hit) for modality_hit in modality_hits], + ) + ) + return results diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index f42d0cf..727ee76 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -1,8 +1,11 @@ """Tool registration entrypoint. -Each tool module exposes a `register_xxx(mcp: FastMCP)` function. This file -wires all of them in one place; to disable a tool, comment out its import -and the corresponding call below. +The only place that knows about the MCP server. A family is registered only when its flag is on: +an unregistered tool is the one kind of "disabled" the protocol guarantees, unlike tag or +visibility filtering, which a later call can undo. + +Melodi tools are plain functions taking their service through `Depends`, so they carry no +registration wrapper. The other families still bind settings through a `register_xxx` closure. """ from __future__ import annotations @@ -19,15 +22,14 @@ from .insee_search_chiffrecle import register_search_insee_chiffrecle from .insee_search_conjoncture import register_search_insee_conjoncture from .insee_search_documents import register_search_insee_documents -from .melodi_get_observations import register_get_melodi_observations -from .melodi_search_datasets import register_search_melodi_datasets -from .melodi_search_modalities import register_search_melodi_modalities +from .melodi.get_observations_tool import get_melodi_observations +from .melodi.search_datasets_tool import search_melodi_datasets +from .melodi.search_modalities_tool import search_melodi_modalities from .rmes_describe_resource import register_describe_rmes_resource from .rmes_run_sparql import register_run_rmes_sparql from .rmes_search_graphs import register_search_rmes_graphs -# Fixme: there might be better pattern instead of iterating with if statements on tool groups def register_tools(mcp: FastMCP, settings: Settings) -> None: """Register the enabled tools, handing each the settings it needs.""" if settings.enable_inseefr_tools: @@ -38,9 +40,9 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: register_search_insee_chiffrecle(mcp, index=settings.es_index_produits) if settings.enable_melodi_tools: - register_search_melodi_datasets(mcp, index=settings.es_index_melodi_datasets) - register_search_melodi_modalities(mcp, index=settings.es_index_melodi_columns) - register_get_melodi_observations(mcp) + mcp.add_tool(search_melodi_datasets) + mcp.add_tool(search_melodi_modalities) + mcp.add_tool(get_melodi_observations) if settings.enable_rmes_tools: register_search_rmes_graphs(mcp, endpoint=settings.rmes_endpoint) diff --git a/src/mcpdiffusion/tools/melodi/__init__.py b/src/mcpdiffusion/tools/melodi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/tools/melodi/get_observations_tool.py b/src/mcpdiffusion/tools/melodi/get_observations_tool.py new file mode 100644 index 0000000..a358faf --- /dev/null +++ b/src/mcpdiffusion/tools/melodi/get_observations_tool.py @@ -0,0 +1,64 @@ +"""Tool: get_melodi_observations.""" + +from __future__ import annotations + +from typing import Any + +from fastmcp.dependencies import Depends + +from ...infra.dependencies import get_melodi_api_service +from ...models.melodi import ( + ColumnFilters, + DatasetId, + NumberOfObservations, + ObservationsOutput, + Years, +) +from ...services.melodi.api_service import MelodiApiService + + +def keep_requested_years( + observations: list[dict[str, Any]], + years: list[int], +) -> list[dict[str, Any]]: + """Drop observations whose TIME_PERIOD does not start with one of the requested years. + + An empty year list means every year is kept. + """ + if not years: + return observations + requested_years = {str(year) for year in years} + return [ + observation + for observation in observations + # Fixme: 'TIME_PERIOD' could be sanitized + if (observation.get("dimensions", {}).get("TIME_PERIOD", "").split("-")[0]) in requested_years + ] + + +async def get_melodi_observations( + dataset_id: DatasetId, + years: Years, + column_filters: ColumnFilters, + number_of_observations: NumberOfObservations = 100, + melodi_api_service: MelodiApiService = Depends(get_melodi_api_service), +) -> ObservationsOutput: + """Retrieve a filtered set of observations from a Melodi dataset. The Melodi API holds official, + high-granularity statistics (prices, mortality, names, etc.). + + Observations carry dimensions, attributes and the numeric measure with its unit. An empty + list means no rows matched; a structured error means the upstream API failed or the inputs + were invalid. + """ + # Fixme: it seems we retrieve all the observations data and filter next + # I wonder whether the API supports filtering + observations = await melodi_api_service.fetch_observations( + dataset_id=dataset_id, + column_filters=column_filters, + ) + selected_observations = keep_requested_years(observations, years)[:number_of_observations] + return ObservationsOutput( + dataset_id=dataset_id, + observations=selected_observations, + count=len(selected_observations), + ) diff --git a/src/mcpdiffusion/tools/melodi/search_datasets_tool.py b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py new file mode 100644 index 0000000..5af801b --- /dev/null +++ b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py @@ -0,0 +1,38 @@ +"""Tool: search_melodi_datasets.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...infra.dependencies import get_melodi_index_service +from ...models.melodi import ( + DatasetQuery, + DatasetsOutput, + EndYear, + NumberOfDatasets, + StartYear, +) +from ...services.melodi.index_service import MelodiIndexService + + +async def search_melodi_datasets( + query: DatasetQuery, + start_year: StartYear = 1900, + end_year: EndYear = 2100, + number_of_datasets: NumberOfDatasets = 5, + melodi_index_service: MelodiIndexService = Depends(get_melodi_index_service), +) -> DatasetsOutput: + """Search the INSEE Melodi dataset catalogue by French-language natural language query. Each + dataset has a unique `dataset_id`; the tool maps the query to internal metadata to return the + most relevant matches. + + Matching is lexical, so make the query explicit and rich in French synonyms, e.g. + `"indice des prix a la consommation"`, `"deces par departement"`, `"prenoms des nouveau-nes"`. + """ + results = await melodi_index_service.search_datasets( + query=query, + start_year=start_year, + end_year=end_year, + number_of_datasets=number_of_datasets, + ) + return DatasetsOutput(results=results) diff --git a/src/mcpdiffusion/tools/melodi/search_modalities_tool.py b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py new file mode 100644 index 0000000..b1b733b --- /dev/null +++ b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py @@ -0,0 +1,38 @@ +"""Tool: search_melodi_modalities.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...infra.dependencies import get_melodi_index_service +from ...models.melodi import ( + ColumnIds, + DatasetId, + ModalitiesOutput, + ModalityQuery, + NumberOfModalities, +) +from ...services.melodi.index_service import MelodiIndexService + + +async def search_melodi_modalities( + dataset_id: DatasetId, + column_ids: ColumnIds, + query: ModalityQuery, + number_of_modalities: NumberOfModalities = 10, + melodi_index_service: MelodiIndexService = Depends(get_melodi_index_service), +) -> ModalitiesOutput: + """Given a Melodi dataset and one or more column identifiers, rank the most relevant modalities + (codes/labels) for a free-text French query. The result is what you need to filter rows in + `get_melodi_observations`. + + Each matching column carries its `code`, its metadata text and the top-scoring + `matching_modalities`. An empty list means nothing matched. + """ + results = await melodi_index_service.search_columns( + dataset_id=dataset_id, + column_ids=column_ids, + query=query, + number_of_modalities=number_of_modalities, + ) + return ModalitiesOutput(results=results) diff --git a/src/mcpdiffusion/tools/melodi_get_observations.py b/src/mcpdiffusion/tools/melodi_get_observations.py deleted file mode 100644 index dd71136..0000000 --- a/src/mcpdiffusion/tools/melodi_get_observations.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tool: get_melodi_observations -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..infra.http import get_melodi_http_client -from ..models.melodi import ( - ColumnFilters, - DatasetId, - NumberOfObservations, - ObservationsOutput, - Years, -) -from ..services.melodi import get_melodi_observations_service - - -def register_get_melodi_observations(mcp: FastMCP) -> None: - @mcp.tool - async def get_melodi_observations( - ctx: Context, - dataset_id: DatasetId, - years: Years, - column_filters: ColumnFilters, - number_of_observations: NumberOfObservations = 100, - ) -> ObservationsOutput: - """Retrieve a filtered set of observations from a Melodi dataset. The Melodi API holds official, - high-granularity statistics (prices, mortality, names, etc.). - - Observations carry dimensions, attributes and the numeric measure with its unit. An empty - list means no rows matched; a structured error means the upstream API failed or the inputs - were invalid. - """ - return await get_melodi_observations_service( - dataset_id=dataset_id, - years=years, - column_filters=column_filters, - number_of_observations=number_of_observations, - http_client=get_melodi_http_client(ctx), - ) diff --git a/src/mcpdiffusion/tools/melodi_search_datasets.py b/src/mcpdiffusion/tools/melodi_search_datasets.py deleted file mode 100644 index fe6f0aa..0000000 --- a/src/mcpdiffusion/tools/melodi_search_datasets.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tool: search_melodi_datasets -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..infra.elasticsearch import get_elasticsearch_client -from ..models.melodi import ( - DatasetQuery, - DatasetsOutput, - EndYear, - NumberOfDatasets, - StartYear, -) -from ..services.melodi import search_melodi_datasets_service - - -def register_search_melodi_datasets(mcp: FastMCP, *, index: str) -> None: - @mcp.tool - async def search_melodi_datasets( - ctx: Context, - query: DatasetQuery, - start_year: StartYear = 1900, - end_year: EndYear = 2100, - number_of_datasets: NumberOfDatasets = 5, - ) -> DatasetsOutput: - """Search the INSEE Melodi dataset catalogue by French-language natural language query. Each - dataset has a unique `dataset_id`; the tool maps the query to internal metadata to return the - most relevant matches. - - Matching is lexical, so make the query explicit and rich in French synonyms, e.g. - `"indice des prix a la consommation"`, `"deces par departement"`, `"prenoms des nouveau-nes"`. - """ - return await search_melodi_datasets_service( - query=query, - start_year=start_year, - end_year=end_year, - number_of_datasets=number_of_datasets, - es=get_elasticsearch_client(ctx), - index=index, - ) diff --git a/src/mcpdiffusion/tools/melodi_search_modalities.py b/src/mcpdiffusion/tools/melodi_search_modalities.py deleted file mode 100644 index f88977b..0000000 --- a/src/mcpdiffusion/tools/melodi_search_modalities.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Tool: search_melodi_modalities -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..infra.elasticsearch import get_elasticsearch_client -from ..models.melodi import ( - ColumnIds, - DatasetId, - ModalitiesOutput, - ModalityQuery, - NumberOfModalities, -) -from ..services.melodi import search_melodi_modalities_service - - -def register_search_melodi_modalities(mcp: FastMCP, *, index: str) -> None: - @mcp.tool - async def search_melodi_modalities( - ctx: Context, - dataset_id: DatasetId, - column_ids: ColumnIds, - query: ModalityQuery, - number_of_modalities: NumberOfModalities = 10, - ) -> ModalitiesOutput: - """Given a Melodi dataset and one or more column identifiers, rank the most relevant modalities - (codes/labels) for a free-text French query. The result is what you need to filter rows in - `get_melodi_observations`. - - Each matching column carries its `code`, its metadata text and the top-scoring - `matching_modalities`. An empty list means nothing matched. - """ - return await search_melodi_modalities_service( - dataset_id=dataset_id, - column_ids=column_ids, - query=query, - number_of_modalities=number_of_modalities, - es=get_elasticsearch_client(ctx), - index=index, - ) From 0c639b89fb0e1aaed09251fe4584e129cddea51c Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 08:11:00 +0200 Subject: [PATCH 20/55] refactor: flatten the package so every module says what it holds `core/` and `infra/` were junk drawers: neither name states a membership rule, so unrelated files accumulated. `core/` held an error type, a 186-line instructions document, logging setup and a rate-limit helper. `infra/` held the lifespan, three client accessors and the dependency providers -- and imported `services/`, inverting the hierarchy its own name claimed. Both are gone, along with `config/`, whose single module now names itself: settings.py errors.py instructions.py lifespan.py logging.py rate_limiting.py dependencies.py The three client accessors fold into `dependencies.py`, so one module holds everything a tool can inject. They keep their explicit-`ctx` signatures and sit under their own heading until insee and rmes move to `Depends`. `data/`, `models/`, `services/` and `tools/` stay: each one can state what belongs in it. Pure relocation. The tool contract -- names, input schemas, descriptions, output schemas -- is byte-identical, as are the generated Elasticsearch queries and every `# Fixme:` and `# Business rule:` marker. The Fixme about untyped dependency functions moves from `infra/__init__.py` into `dependencies.py`, where it now names the cause: the lifespan context is an untyped mapping, so the return annotations are asserted, never checked. --- src/mcpdiffusion/config/__init__.py | 1 - src/mcpdiffusion/core/__init__.py | 1 - src/mcpdiffusion/dependencies.py | 60 +++++++++++++++++++ src/mcpdiffusion/{core => }/errors.py | 0 src/mcpdiffusion/infra/__init__.py | 3 - src/mcpdiffusion/infra/dependencies.py | 21 ------- src/mcpdiffusion/infra/elasticsearch.py | 8 --- src/mcpdiffusion/infra/http.py | 12 ---- src/mcpdiffusion/infra/sparql.py | 8 --- src/mcpdiffusion/{core => }/instructions.py | 0 src/mcpdiffusion/{infra => }/lifespan.py | 4 +- src/mcpdiffusion/{core => }/logging.py | 0 src/mcpdiffusion/{core => }/rate_limiting.py | 0 src/mcpdiffusion/server.py | 10 ++-- .../services/elasticsearch_failures.py | 2 +- src/mcpdiffusion/services/insee_document.py | 2 +- .../services/melodi/api_service.py | 2 +- src/mcpdiffusion/services/rmes.py | 2 +- src/mcpdiffusion/{config => }/settings.py | 0 src/mcpdiffusion/tools/__init__.py | 2 +- src/mcpdiffusion/tools/insee_get_document.py | 2 +- .../tools/insee_search_chiffrecle.py | 4 +- .../tools/insee_search_conjoncture.py | 4 +- .../tools/insee_search_documents.py | 4 +- .../tools/melodi/get_observations_tool.py | 2 +- .../tools/melodi/search_datasets_tool.py | 2 +- .../tools/melodi/search_modalities_tool.py | 2 +- .../tools/rmes_describe_resource.py | 2 +- src/mcpdiffusion/tools/rmes_run_sparql.py | 2 +- src/mcpdiffusion/tools/rmes_search_graphs.py | 2 +- 30 files changed, 85 insertions(+), 79 deletions(-) delete mode 100644 src/mcpdiffusion/config/__init__.py delete mode 100644 src/mcpdiffusion/core/__init__.py create mode 100644 src/mcpdiffusion/dependencies.py rename src/mcpdiffusion/{core => }/errors.py (100%) delete mode 100644 src/mcpdiffusion/infra/__init__.py delete mode 100644 src/mcpdiffusion/infra/dependencies.py delete mode 100644 src/mcpdiffusion/infra/elasticsearch.py delete mode 100644 src/mcpdiffusion/infra/http.py delete mode 100644 src/mcpdiffusion/infra/sparql.py rename src/mcpdiffusion/{core => }/instructions.py (100%) rename src/mcpdiffusion/{infra => }/lifespan.py (96%) rename src/mcpdiffusion/{core => }/logging.py (100%) rename src/mcpdiffusion/{core => }/rate_limiting.py (100%) rename src/mcpdiffusion/{config => }/settings.py (100%) diff --git a/src/mcpdiffusion/config/__init__.py b/src/mcpdiffusion/config/__init__.py deleted file mode 100644 index ed78ad4..0000000 --- a/src/mcpdiffusion/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Centralized configuration.""" diff --git a/src/mcpdiffusion/core/__init__.py b/src/mcpdiffusion/core/__init__.py deleted file mode 100644 index f4552e7..0000000 --- a/src/mcpdiffusion/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Shared cross-cutting concerns (logging, errors, middleware).""" diff --git a/src/mcpdiffusion/dependencies.py b/src/mcpdiffusion/dependencies.py new file mode 100644 index 0000000..d7d26c9 --- /dev/null +++ b/src/mcpdiffusion/dependencies.py @@ -0,0 +1,60 @@ +"""Everything a tool can ask FastMCP to inject. + +A tool declares what it needs in its signature; FastMCP resolves it per request and hides the +parameter from the tool schema, so the LLM never sees it. + +Two styles coexist while the sources migrate. `Depends(...)` factories read the context +themselves and appear as a parameter default. The older accessors take an explicit `ctx` and are +called from inside the tool body; insee and rmes still use those. +""" + +from elasticsearch import AsyncElasticsearch +from fastmcp import Context +from fastmcp.dependencies import CurrentContext +from httpx import AsyncClient + +from .services.melodi.api_service import MelodiApiService +from .services.melodi.index_service import MelodiIndexService + +# Fixme: these dependency functions do not provide proper typing which is a pity -- the lifespan +# context is an untyped mapping, so every return annotation below is asserted, never checked. + + +# ---------------------------------------------------------------------------------------------------------------------- +# Injected dependencies ------------------------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------------------------------------------------- + + +def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: + """Return the Melodi Elasticsearch service built at startup.""" + return ctx.lifespan_context["melodi_index_service"] + + +def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: + """Return the Melodi REST API service built at startup.""" + return ctx.lifespan_context["melodi_api_service"] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Client accessors, pending migration to Depends ----------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def get_elasticsearch_client(ctx: Context) -> AsyncElasticsearch: + """Return the shared Elasticsearch client built at startup.""" + return ctx.lifespan_context["elasticsearch_client"] + + +def get_insee_http_client(ctx: Context) -> AsyncClient: + """Return the shared insee.fr scraping client built at startup.""" + return ctx.lifespan_context["insee_http_client"] + + +def get_melodi_http_client(ctx: Context) -> AsyncClient: + """Return the shared Melodi API client built at startup.""" + return ctx.lifespan_context["melodi_http_client"] + + +def get_sparql_http_client(ctx: Context) -> AsyncClient: + """Return the shared SPARQL client built at startup.""" + return ctx.lifespan_context["sparql_http_client"] diff --git a/src/mcpdiffusion/core/errors.py b/src/mcpdiffusion/errors.py similarity index 100% rename from src/mcpdiffusion/core/errors.py rename to src/mcpdiffusion/errors.py diff --git a/src/mcpdiffusion/infra/__init__.py b/src/mcpdiffusion/infra/__init__.py deleted file mode 100644 index a2a266c..0000000 --- a/src/mcpdiffusion/infra/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Infrastructure: external clients (ES, HTTP, SPARQL).""" - -# Fixme: each of the dependency functions in that folder does not provide proper typing which is a pity diff --git a/src/mcpdiffusion/infra/dependencies.py b/src/mcpdiffusion/infra/dependencies.py deleted file mode 100644 index be35651..0000000 --- a/src/mcpdiffusion/infra/dependencies.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Typed access to the objects the lifespan built, declared as FastMCP dependencies. - -A tool asks for what it needs in its signature; FastMCP resolves it per request and hides the -parameter from the tool schema, so the LLM never sees it. -""" - -from fastmcp import Context -from fastmcp.dependencies import CurrentContext - -from ..services.melodi.api_service import MelodiApiService -from ..services.melodi.index_service import MelodiIndexService - - -def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: - """Return the Melodi Elasticsearch service built at startup.""" - return ctx.lifespan_context["melodi_index_service"] - - -def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: - """Return the Melodi REST API service built at startup.""" - return ctx.lifespan_context["melodi_api_service"] diff --git a/src/mcpdiffusion/infra/elasticsearch.py b/src/mcpdiffusion/infra/elasticsearch.py deleted file mode 100644 index 653bf3d..0000000 --- a/src/mcpdiffusion/infra/elasticsearch.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Elasticsearch client accessor from the FastMCP lifespan context.""" - -from elasticsearch import AsyncElasticsearch -from fastmcp import Context - - -def get_elasticsearch_client(ctx: Context) -> AsyncElasticsearch: - return ctx.lifespan_context["elasticsearch_client"] diff --git a/src/mcpdiffusion/infra/http.py b/src/mcpdiffusion/infra/http.py deleted file mode 100644 index 7b723f1..0000000 --- a/src/mcpdiffusion/infra/http.py +++ /dev/null @@ -1,12 +0,0 @@ -"""HTTP client accessors from the FastMCP lifespan context.""" - -from fastmcp import Context -from httpx import AsyncClient - - -def get_insee_http_client(ctx: Context) -> AsyncClient: - return ctx.lifespan_context["insee_http_client"] - - -def get_melodi_http_client(ctx: Context) -> AsyncClient: - return ctx.lifespan_context["melodi_http_client"] diff --git a/src/mcpdiffusion/infra/sparql.py b/src/mcpdiffusion/infra/sparql.py deleted file mode 100644 index a0ff10d..0000000 --- a/src/mcpdiffusion/infra/sparql.py +++ /dev/null @@ -1,8 +0,0 @@ -"""SPARQL client accessor from the FastMCP lifespan context.""" - -from fastmcp import Context -from httpx import AsyncClient - - -def get_sparql_http_client(ctx: Context) -> AsyncClient: - return ctx.lifespan_context["sparql_http_client"] diff --git a/src/mcpdiffusion/core/instructions.py b/src/mcpdiffusion/instructions.py similarity index 100% rename from src/mcpdiffusion/core/instructions.py rename to src/mcpdiffusion/instructions.py diff --git a/src/mcpdiffusion/infra/lifespan.py b/src/mcpdiffusion/lifespan.py similarity index 96% rename from src/mcpdiffusion/infra/lifespan.py rename to src/mcpdiffusion/lifespan.py index 52476cf..f9e11a0 100644 --- a/src/mcpdiffusion/infra/lifespan.py +++ b/src/mcpdiffusion/lifespan.py @@ -8,8 +8,8 @@ from fastmcp.server.lifespan import lifespan from httpx import AsyncClient, Timeout -from ..services.melodi.api_service import MelodiApiService -from ..services.melodi.index_service import MelodiIndexService +from .services.melodi.api_service import MelodiApiService +from .services.melodi.index_service import MelodiIndexService logger = logging.getLogger(__name__) diff --git a/src/mcpdiffusion/core/logging.py b/src/mcpdiffusion/logging.py similarity index 100% rename from src/mcpdiffusion/core/logging.py rename to src/mcpdiffusion/logging.py diff --git a/src/mcpdiffusion/core/rate_limiting.py b/src/mcpdiffusion/rate_limiting.py similarity index 100% rename from src/mcpdiffusion/core/rate_limiting.py rename to src/mcpdiffusion/rate_limiting.py diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 128e6be..8ff5b64 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -9,11 +9,11 @@ from fastmcp.server.middleware.rate_limiting import SlidingWindowRateLimitingMiddleware from fastmcp.server.middleware.timing import TimingMiddleware -from .config.settings import load_settings -from .core.instructions import build_instructions -from .core.logging import build_logging_config, configure_logging -from .core.rate_limiting import resolve_client_host -from .infra.lifespan import build_lifespan +from .instructions import build_instructions +from .lifespan import build_lifespan +from .logging import build_logging_config, configure_logging +from .rate_limiting import resolve_client_host +from .settings import load_settings from .tools import register_tools settings = load_settings() diff --git a/src/mcpdiffusion/services/elasticsearch_failures.py b/src/mcpdiffusion/services/elasticsearch_failures.py index c2a7ae1..14fc440 100644 --- a/src/mcpdiffusion/services/elasticsearch_failures.py +++ b/src/mcpdiffusion/services/elasticsearch_failures.py @@ -16,7 +16,7 @@ from elasticsearch import ApiError, TransportError -from ..core.errors import AppToolError +from ..errors import AppToolError @asynccontextmanager diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py index b978cf8..ee4b3fa 100644 --- a/src/mcpdiffusion/services/insee_document.py +++ b/src/mcpdiffusion/services/insee_document.py @@ -11,7 +11,7 @@ from trafilatura import extract from trafilatura.settings import Extractor -from ..core.errors import AppToolError +from ..errors import AppToolError from ..models.insee import ( DocumentContentOutput, DocumentResult, diff --git a/src/mcpdiffusion/services/melodi/api_service.py b/src/mcpdiffusion/services/melodi/api_service.py index 9681747..1d62182 100644 --- a/src/mcpdiffusion/services/melodi/api_service.py +++ b/src/mcpdiffusion/services/melodi/api_service.py @@ -6,7 +6,7 @@ import httpx -from ...core.errors import AppToolError +from ...errors import AppToolError class MelodiApiService: diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py index 539c18b..0a8ee63 100644 --- a/src/mcpdiffusion/services/rmes.py +++ b/src/mcpdiffusion/services/rmes.py @@ -13,7 +13,7 @@ import httpx -from ..core.errors import AppToolError +from ..errors import AppToolError from ..models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, GRAPH_BASE, diff --git a/src/mcpdiffusion/config/settings.py b/src/mcpdiffusion/settings.py similarity index 100% rename from src/mcpdiffusion/config/settings.py rename to src/mcpdiffusion/settings.py diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 727ee76..43ee91c 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -12,7 +12,7 @@ from fastmcp import FastMCP -from ..config.settings import Settings +from ..settings import Settings # Imported but never registered: send_feedback is not exposed. Decide whether to wire it up or # drop it, then remove this import or the noqa. diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py index 0673a6b..53e0e63 100644 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ b/src/mcpdiffusion/tools/insee_get_document.py @@ -4,7 +4,7 @@ from fastmcp import Context, FastMCP -from ..infra.http import get_insee_http_client +from ..dependencies import get_insee_http_client from ..models.insee import ( DocumentContentOutput, DocumentUrls, diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py index a28a546..f3602ef 100644 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ b/src/mcpdiffusion/tools/insee_search_chiffrecle.py @@ -6,8 +6,8 @@ from elasticsearch import TransportError from fastmcp import Context, FastMCP -from ..core.errors import AppToolError -from ..infra.elasticsearch import get_elasticsearch_client +from ..dependencies import get_elasticsearch_client +from ..errors import AppToolError from ..models.insee import ( DEFAULT_RESULT_COUNT, DocumentSearchOutput, diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py index 32b7a97..9ad7d3c 100644 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ b/src/mcpdiffusion/tools/insee_search_conjoncture.py @@ -7,9 +7,9 @@ from elasticsearch.dsl import Q from fastmcp import Context, FastMCP -from ..core.errors import AppToolError from ..data.themes import DICT_THEME_CONJ -from ..infra.elasticsearch import get_elasticsearch_client +from ..dependencies import get_elasticsearch_client +from ..errors import AppToolError from ..models.insee import ( DEFAULT_RESULT_COUNT, ConjonctureQuery, diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py index b1b325b..a3ae330 100644 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ b/src/mcpdiffusion/tools/insee_search_documents.py @@ -6,8 +6,8 @@ from elasticsearch import TransportError from fastmcp import Context, FastMCP -from ..core.errors import AppToolError -from ..infra.elasticsearch import get_elasticsearch_client +from ..dependencies import get_elasticsearch_client +from ..errors import AppToolError from ..models.insee import ( DEFAULT_RESULT_COUNT, DocumentSearchOutput, diff --git a/src/mcpdiffusion/tools/melodi/get_observations_tool.py b/src/mcpdiffusion/tools/melodi/get_observations_tool.py index a358faf..4d8f56e 100644 --- a/src/mcpdiffusion/tools/melodi/get_observations_tool.py +++ b/src/mcpdiffusion/tools/melodi/get_observations_tool.py @@ -6,7 +6,7 @@ from fastmcp.dependencies import Depends -from ...infra.dependencies import get_melodi_api_service +from ...dependencies import get_melodi_api_service from ...models.melodi import ( ColumnFilters, DatasetId, diff --git a/src/mcpdiffusion/tools/melodi/search_datasets_tool.py b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py index 5af801b..3602945 100644 --- a/src/mcpdiffusion/tools/melodi/search_datasets_tool.py +++ b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...infra.dependencies import get_melodi_index_service +from ...dependencies import get_melodi_index_service from ...models.melodi import ( DatasetQuery, DatasetsOutput, diff --git a/src/mcpdiffusion/tools/melodi/search_modalities_tool.py b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py index b1b733b..8e088de 100644 --- a/src/mcpdiffusion/tools/melodi/search_modalities_tool.py +++ b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...infra.dependencies import get_melodi_index_service +from ...dependencies import get_melodi_index_service from ...models.melodi import ( ColumnIds, DatasetId, diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py index d30ca01..19fdb7c 100644 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ b/src/mcpdiffusion/tools/rmes_describe_resource.py @@ -4,7 +4,7 @@ from fastmcp import Context, FastMCP -from ..infra.sparql import get_sparql_http_client +from ..dependencies import get_sparql_http_client from ..models.rmes import GraphUri, ResourceOutput, ResourceUri from ..services.rmes import describe_rmes_resource_service diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py index 99ac74a..7dda085 100644 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ b/src/mcpdiffusion/tools/rmes_run_sparql.py @@ -4,7 +4,7 @@ from fastmcp import Context, FastMCP -from ..infra.sparql import get_sparql_http_client +from ..dependencies import get_sparql_http_client from ..models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, DEFAULT_ROW_LIMIT, diff --git a/src/mcpdiffusion/tools/rmes_search_graphs.py b/src/mcpdiffusion/tools/rmes_search_graphs.py index a3a77ec..52b61d3 100644 --- a/src/mcpdiffusion/tools/rmes_search_graphs.py +++ b/src/mcpdiffusion/tools/rmes_search_graphs.py @@ -4,7 +4,7 @@ from fastmcp import Context, FastMCP -from ..infra.sparql import get_sparql_http_client +from ..dependencies import get_sparql_http_client from ..models.rmes import ( ExpandGraphs, GraphCategory, From 66d88903c0d7aae249a47f1f1043634200aca1d0 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 08:11:09 +0200 Subject: [PATCH 21/55] chore: move the example env file to the repo root It sat inside the package at `src/mcpdiffusion/.env.example`, which shipped it in the wheel and put the documented configuration surface where nobody looks for it. Nothing referenced that path except README. `docker-compose-dev.yaml` reads `mcp-diffusion.env` and is unaffected, and `.gitignore` ignores `*.env` but not `*.env.example`, so the example stays tracked. --- src/mcpdiffusion/.env.example => .env.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/mcpdiffusion/.env.example => .env.example (100%) diff --git a/src/mcpdiffusion/.env.example b/.env.example similarity index 100% rename from src/mcpdiffusion/.env.example rename to .env.example From 8c71e837aafd4955bd011a57c69cf4e58c4bf962 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 20:56:54 +0200 Subject: [PATCH 22/55] refactor(insee): inject insee's backend services into the tools insee's data access lived in three modules the tools wired by hand: each search tool assembled its own query, threaded an index name down on every call, and translated Elasticsearch failures itself. Each backend now sits behind a service, built once at startup with its client and index already bound: services/insee/index_service.py InseeIndexService -- Elasticsearch services/insee/document_service.py InseeDocumentService -- insee.fr pages Tools declare the service they need and FastMCP injects it, so no insee tool imports `elasticsearch` or `httpx` any more, and none carries an index name. `build_key_indicators` moves into the homepage tool, its only caller, and `services/insee_indicators.py` goes with it. The one query builder driven by three booleans -- `must_not_rapides`, `must_only_rapides`, `chiffre_clef` -- becomes one builder per search. Those flags encoded which tool was calling and allowed combinations that mean nothing. The clause lists they passed around as a bare tuple are now a frozen `QueryClauses` with named `must`, `filter` and `should`. Every generated Elasticsearch body is identical to the one it replaces, checked through the registered tools rather than in isolation. Also corrects three failures the caller could not act on: - `search_insee_chiffrecle` reported "INSEE documents search backend unreachable", copy-pasted from the documents tool. Each search now names itself. - Elasticsearch `ApiError` escaped the boundary entirely, because it does not subclass `TransportError`. A missing index surfaced as a generic error. All three searches now share the boundary that maps it by status. - `document_urls` entries were typed `object` and coerced with `str()`, though the model has always declared `list[str]`. `es_index_produits` becomes `es_index_publications`, so the setting and the parameter reading it finally share one name. The value stays "produit": that is the real index. Deployments setting ES_INDEX_PRODUITS must rename it to ES_INDEX_PUBLICATIONS -- an unknown variable is ignored, not rejected. The only change to the tool contract is `DocumentResult.status`, now `Literal["success", "error"]` rather than `str`. That adds an enum to the output schema without changing any value a caller receives. --- .env.example | 2 +- src/mcpdiffusion/dependencies.py | 32 +- src/mcpdiffusion/lifespan.py | 10 + src/mcpdiffusion/models/insee.py | 13 +- src/mcpdiffusion/server.py | 2 + src/mcpdiffusion/services/insee/__init__.py | 0 .../services/insee/document_service.py | 238 +++++++++++ .../services/insee/index_service.py | 392 ++++++++++++++++++ src/mcpdiffusion/services/insee_document.py | 197 --------- src/mcpdiffusion/services/insee_indicators.py | 18 - src/mcpdiffusion/services/insee_search.py | 185 --------- src/mcpdiffusion/settings.py | 2 +- src/mcpdiffusion/tools/__init__.py | 24 +- src/mcpdiffusion/tools/insee/__init__.py | 0 .../tools/insee/get_document_tool.py | 36 ++ .../tools/insee/get_homepage_tool.py | 30 ++ .../tools/insee/search_chiffrecle_tool.py | 44 ++ .../tools/insee/search_conjoncture_tool.py | 42 ++ .../tools/insee/search_documents_tool.py | 50 +++ src/mcpdiffusion/tools/insee_get_document.py | 35 -- src/mcpdiffusion/tools/insee_get_homepage.py | 22 - .../tools/insee_search_chiffrecle.py | 74 ---- .../tools/insee_search_conjoncture.py | 78 ---- .../tools/insee_search_documents.py | 79 ---- 24 files changed, 882 insertions(+), 723 deletions(-) create mode 100644 src/mcpdiffusion/services/insee/__init__.py create mode 100644 src/mcpdiffusion/services/insee/document_service.py create mode 100644 src/mcpdiffusion/services/insee/index_service.py delete mode 100644 src/mcpdiffusion/services/insee_document.py delete mode 100644 src/mcpdiffusion/services/insee_indicators.py delete mode 100644 src/mcpdiffusion/services/insee_search.py create mode 100644 src/mcpdiffusion/tools/insee/__init__.py create mode 100644 src/mcpdiffusion/tools/insee/get_document_tool.py create mode 100644 src/mcpdiffusion/tools/insee/get_homepage_tool.py create mode 100644 src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py create mode 100644 src/mcpdiffusion/tools/insee/search_conjoncture_tool.py create mode 100644 src/mcpdiffusion/tools/insee/search_documents_tool.py delete mode 100644 src/mcpdiffusion/tools/insee_get_document.py delete mode 100644 src/mcpdiffusion/tools/insee_get_homepage.py delete mode 100644 src/mcpdiffusion/tools/insee_search_chiffrecle.py delete mode 100644 src/mcpdiffusion/tools/insee_search_conjoncture.py delete mode 100644 src/mcpdiffusion/tools/insee_search_documents.py diff --git a/.env.example b/.env.example index 9e10203..cda922c 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,7 @@ ENABLE_RMES_TOOLS=true # Elasticsearch -------------------------------------------------------------------------------------------------------- # Inside Docker use the service name; on the host use localhost. ES_HOST=http://localhost:9200 -ES_INDEX_PRODUITS=produit +ES_INDEX_PUBLICATIONS=produit ES_INDEX_MELODI_DATASETS=melodi_datasets ES_INDEX_MELODI_COLUMNS=melodi_columns # Only Elasticsearch is configurable here. diff --git a/src/mcpdiffusion/dependencies.py b/src/mcpdiffusion/dependencies.py index d7d26c9..d35b26e 100644 --- a/src/mcpdiffusion/dependencies.py +++ b/src/mcpdiffusion/dependencies.py @@ -5,14 +5,15 @@ Two styles coexist while the sources migrate. `Depends(...)` factories read the context themselves and appear as a parameter default. The older accessors take an explicit `ctx` and are -called from inside the tool body; insee and rmes still use those. +called from inside the tool body; rmes still uses one. """ -from elasticsearch import AsyncElasticsearch from fastmcp import Context from fastmcp.dependencies import CurrentContext from httpx import AsyncClient +from .services.insee.document_service import InseeDocumentService +from .services.insee.index_service import InseeIndexService from .services.melodi.api_service import MelodiApiService from .services.melodi.index_service import MelodiIndexService @@ -21,10 +22,20 @@ # ---------------------------------------------------------------------------------------------------------------------- -# Injected dependencies ------------------------------------------------------------------------------------------------ +# Injected services ---------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- +def get_insee_index_service(ctx: Context = CurrentContext()) -> InseeIndexService: + """Return the insee.fr Elasticsearch service built at startup.""" + return ctx.lifespan_context["insee_index_service"] + + +def get_insee_document_service(ctx: Context = CurrentContext()) -> InseeDocumentService: + """Return the insee.fr document scraping service built at startup.""" + return ctx.lifespan_context["insee_document_service"] + + def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: """Return the Melodi Elasticsearch service built at startup.""" return ctx.lifespan_context["melodi_index_service"] @@ -40,21 +51,6 @@ def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: # ---------------------------------------------------------------------------------------------------------------------- -def get_elasticsearch_client(ctx: Context) -> AsyncElasticsearch: - """Return the shared Elasticsearch client built at startup.""" - return ctx.lifespan_context["elasticsearch_client"] - - -def get_insee_http_client(ctx: Context) -> AsyncClient: - """Return the shared insee.fr scraping client built at startup.""" - return ctx.lifespan_context["insee_http_client"] - - -def get_melodi_http_client(ctx: Context) -> AsyncClient: - """Return the shared Melodi API client built at startup.""" - return ctx.lifespan_context["melodi_http_client"] - - def get_sparql_http_client(ctx: Context) -> AsyncClient: """Return the shared SPARQL client built at startup.""" return ctx.lifespan_context["sparql_http_client"] diff --git a/src/mcpdiffusion/lifespan.py b/src/mcpdiffusion/lifespan.py index f9e11a0..242d67c 100644 --- a/src/mcpdiffusion/lifespan.py +++ b/src/mcpdiffusion/lifespan.py @@ -8,6 +8,8 @@ from fastmcp.server.lifespan import lifespan from httpx import AsyncClient, Timeout +from .services.insee.document_service import InseeDocumentService +from .services.insee.index_service import InseeIndexService from .services.melodi.api_service import MelodiApiService from .services.melodi.index_service import MelodiIndexService @@ -35,6 +37,7 @@ def build_lifespan( melodi_connect_timeout_seconds: int, melodi_datasets_index: str, melodi_columns_index: str, + insee_publications_index: str, ) -> Callable[..., Any]: @lifespan @@ -82,6 +85,11 @@ async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: columns_index=melodi_columns_index, ) melodi_api_service = MelodiApiService(http_client=melodi_http_client) + insee_index_service = InseeIndexService( + elasticsearch_client=elasticsearch_client, + publications_index=insee_publications_index, + ) + insee_document_service = InseeDocumentService(http_client=insee_http_client) try: yield { @@ -91,6 +99,8 @@ async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: "sparql_http_client": sparql_http_client, "melodi_index_service": melodi_index_service, "melodi_api_service": melodi_api_service, + "insee_index_service": insee_index_service, + "insee_document_service": insee_document_service, } finally: await elasticsearch_client.close() diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 65c137b..5639fb8 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -3,7 +3,7 @@ from __future__ import annotations from enum import StrEnum -from typing import Annotated +from typing import Annotated, Literal from pydantic import BaseModel, Field @@ -225,12 +225,19 @@ class DocumentSearchOutput(BaseModel): # --- get_insee_document --- +# The entries of one category: publication title -> relative url. +CategoryFields = dict[str, str] +# Category name -> its entries. +TableOfContents = dict[str, CategoryFields] + class DocumentResult(BaseModel): id: str = Field(description="The input URL that produced this entry.") - status: str = Field(description="'success' or 'error'.") + status: Literal["success", "error"] = Field( + description="Whether this URL was fetched and parsed, or failed.", + ) markdown_content: str | None = None - sommaire: dict[str, dict[str, str]] | None = Field( + sommaire: TableOfContents | None = Field( default=None, description=( "Parsed table of contents as " diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 8ff5b64..6b58cf4 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -32,6 +32,7 @@ # Only AppToolError messages reach the caller; anything else is a bug and is replaced # by a generic message. mask_error_details=True, + # Fixme: it takes too many arguments, we can maybe pass settings here directly lifespan=build_lifespan( es_host=settings.es_host, es_tls_verify=settings.es_tls_verify, @@ -44,6 +45,7 @@ melodi_connect_timeout_seconds=settings.melodi_connect_timeout_seconds, melodi_datasets_index=settings.es_index_melodi_datasets, melodi_columns_index=settings.es_index_melodi_columns, + insee_publications_index=settings.es_index_publications, ), ) diff --git a/src/mcpdiffusion/services/insee/__init__.py b/src/mcpdiffusion/services/insee/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py new file mode 100644 index 0000000..87a44ab --- /dev/null +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -0,0 +1,238 @@ +"""insee.fr document access: fetching a publication page and turning it into markdown.""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from urllib.parse import urljoin, urlparse + +import httpx +from bs4 import BeautifulSoup +from trafilatura import extract +from trafilatura.settings import Extractor + +from ...errors import AppToolError +from ...models.insee import DocumentResult, TableOfContents + +logger = logging.getLogger(__name__) + +TRAFILATURA_OPTIONS = Extractor( + output_format="markdown", + links=True, + formatting=True, + # Fixme: this URL might belong in the settings + source="insee.fr", + with_metadata=True, +) + +MAX_MARKDOWN_CHARS = 30_000 + +TRUNCATION_MARKER = """ + + + +""" + +# One flat entry per link, before it is grouped: {"category": ..., "title": ..., "url": ...}. +TableOfContentsEntry = dict[str, str] +TableOfContentsEntries = list[TableOfContentsEntry] + +# The values are the CSS classes insee.fr serves, hence French. +TABLE_OF_CONTENTS_CLASS = "sommaire" +PRODUCT_LINK_CLASS = "lien-produit" + + +# ---------------------------------------------------------------------------------------------------------------------- +# Page parsing --------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def to_relative_url(url: str) -> str: + """Strip the scheme and host, keeping the path and query the caller can pass back in.""" + parsed = urlparse(url) + return f"{parsed.path}?{parsed.query}" if parsed.query else parsed.path + + +def parse_table_of_contents(html: str, base_url: str) -> TableOfContentsEntries: + """Extract the table of contents as flat (category, title, url) entries. + + A page either groups its links under `h2` headings or lists them flat; both shapes appear, + and an ungrouped link gets an empty category. + """ + soup = BeautifulSoup(html, "lxml") + entries: TableOfContentsEntries = [] + + section = soup.find(lambda tag: tag.has_attr("class") and any(TABLE_OF_CONTENTS_CLASS in c for c in tag["class"])) + if not section: + return [] + + outer_list = section.find("ul", class_=TABLE_OF_CONTENTS_CLASS) + if not outer_list: + return [] + + for top_item in outer_list.find_all("li", recursive=False): + heading = top_item.find("h2") + if heading: + category_name = heading.get_text(strip=True) + inner_list = top_item.find("ul", class_=TABLE_OF_CONTENTS_CLASS) + if not inner_list: + continue + for link_item in inner_list.find_all("li", class_=PRODUCT_LINK_CLASS): + anchor = link_item.find("a") + if not anchor: + continue + entries.append( + { + "category": category_name, + "title": anchor.get_text(strip=True), + "url": to_relative_url(urljoin(base_url, anchor.get("href", ""))), + } + ) + else: + anchor = top_item.find("a") + if not anchor: + continue + entries.append( + { + "category": "", + "title": anchor.get_text(strip=True), + "url": to_relative_url(urljoin(base_url, anchor.get("href", ""))), + } + ) + return entries + + +def group_table_of_contents(entries: TableOfContentsEntries) -> TableOfContents: + """Turn the flat entries into {category: {title: url}}.""" + by_category: TableOfContents = defaultdict(dict) + for entry in entries: + by_category[entry["category"]][entry["title"]] = entry["url"] + return dict(by_category) + + +def truncate_markdown(text: str, limit: int = MAX_MARKDOWN_CHARS) -> tuple[str, bool]: + """Keep the head and tail of an over-long document, marking where the middle was dropped.""" + if len(text) <= limit: + return text, False + budget = max(0, limit - len(TRUNCATION_MARKER)) + head_size = (budget * 2) // 3 + tail_size = budget - head_size + # text[-0:] returns the whole string, so an empty tail has to be spelled out. + tail = text[-tail_size:] if tail_size else "" + return text[:head_size] + TRUNCATION_MARKER + tail, True + + +def build_failed_document(url: str, message: str) -> DocumentResult: + """Report one URL's failure in the same shape as a success, so callers need no type check.""" + return DocumentResult( + id=url, + status="error", + markdown_content=None, + sommaire=None, + truncated=False, + error=message, + ) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Service -------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +class InseeDocumentService: + """Fetches insee.fr publication pages and renders them as markdown.""" + + def __init__(self, http_client: httpx.AsyncClient) -> None: + self._http_client = http_client + + async def fetch_html(self, url: str) -> str: + """Return the raw HTML of one publication page.""" + # A relative path resolves against the client's base_url; an absolute one overrides it. + target = self._http_client.base_url.join(url) + try: + response = await self._http_client.get(url, follow_redirects=True) + response.raise_for_status() + return response.text + except httpx.TimeoutException as exc: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"insee.fr timed out fetching {target}: {exc}", + retryable=True, + ) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == httpx.codes.NOT_FOUND: + raise AppToolError( + "NOT_FOUND", + f"INSEE document not found at {target} (HTTP 404). Verify the URL with `search_insee_documents`.", + ) + raise AppToolError( + "UPSTREAM_ERROR", + f"insee.fr returned HTTP {exc.response.status_code} for {target}.", + retryable=exc.response.is_server_error, + ) + except httpx.HTTPError as exc: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Network error fetching {target}: {exc}", + retryable=True, + ) + + async def fetch_documents( + self, + document_urls: list[str], + include_table_of_contents: bool, + truncate_content: bool, + ) -> list[DocumentResult]: + """Fetch and render each URL, reporting per-URL failures rather than aborting the batch.""" + if not document_urls: + raise AppToolError( + "INVALID_INPUT", + "document_urls must contain at least one URL. Use `search_insee_documents` to find URLs first.", + ) + + results: list[DocumentResult] = [] + # Fixme: there should be a cap in the number of URLs provided to avoid overloading the server + # Fixme: on top of that, the fetching is done sequentially, impacting the event loop + for url in document_urls: + try: + html = await self.fetch_html(url) + markdown = extract(html, options=TRAFILATURA_OPTIONS) or "" + markdown, truncated = truncate_markdown(markdown) if truncate_content else (markdown, False) + + table_of_contents: TableOfContents | None = None + if include_table_of_contents: + entries = parse_table_of_contents( + html=html, + base_url=str(self._http_client.base_url), + ) + table_of_contents = group_table_of_contents(entries) if entries else None + + results.append( + DocumentResult( + id=url, + status="success", + markdown_content=markdown, + sommaire=table_of_contents, + truncated=truncated, + error=None, + ) + ) + except AppToolError as exc: + # A typed failure is written for the caller, so it is safe to pass on. + results.append( + build_failed_document( + url=url, + message=str(exc), + ) + ) + except Exception: + # Anything else is a bug: log it here, tell the caller only that this URL failed. + logger.exception("Unexpected failure fetching %s", url) + results.append( + build_failed_document( + url=url, + message="[UNKNOWN] Could not fetch this document.", + ) + ) + + return results diff --git a/src/mcpdiffusion/services/insee/index_service.py b/src/mcpdiffusion/services/insee/index_service.py new file mode 100644 index 0000000..4281120 --- /dev/null +++ b/src/mcpdiffusion/services/insee/index_service.py @@ -0,0 +1,392 @@ +"""INSEE's Elasticsearch access: query construction, execution and parsing. + +Elasticsearch vocabulary stops here. Tools never see a `Hit` or a `_source` envelope, so a change +in the index shape is contained to this file. + +The three searches share one index and one text-matching rule, and differ only in the collection +they keep and the filters they add. Each gets its own builder rather than one builder driven by +boolean flags, so no caller can ask for a combination that makes no sense. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass, field + +from elasticsearch import AsyncElasticsearch +from elasticsearch.dsl import AsyncSearch, Q +from elasticsearch.dsl.query import Query +from elasticsearch.dsl.response import Response + +from ...data.geography import DICT_GEO +from ...data.themes import DICT_THEME_CONJ, KEYS_THEME_NIV1 +from ...models.insee import DocumentHit +from ..elasticsearch_failures import elasticsearch_failures_as_tool_errors + +RAPIDES_COLLECTION = "Informations rapides" +CHIFFRES_CLES_CATEGORY = "Chiffres-clés" + + +@dataclass(frozen=True) +class QueryClauses: + """The clause lists of an Elasticsearch `bool` query, named rather than positional. + + A builder that contributes nothing to one of them leaves it empty; the search builders then + concatenate the parts in the order Elasticsearch receives them. + """ + + must: list[Query] = field(default_factory=list) + filter: list[Query] = field(default_factory=list) + should: list[Query] = field(default_factory=list) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Shared clause builders ----------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def build_text_clauses( + query: str | None, + year_of_reference: int | None, + keywords: Iterable[str] = (), +) -> QueryClauses: + """Return the clauses matching a text query, optionally pinned to a publication year.""" + must: list[Query] = [] + filters: list[Query] = [] + should: list[Query] = [] + + if query: + must.append( + Q( + "multi_match", + query=query, + fields=[ + "titre^5", + "titre.ngram^3", + "soustitre^2", + "zone^5", + "chapo", + "theme", + ], + fuzziness="AUTO", + ) + ) + should.append(Q("match_phrase", titre={"query": query, "boost": 1})) + + if year_of_reference: + filters.append( + Q( + "multi_match", + query=str(year_of_reference), + fields=["titre^10", "soustitre^5", "chapo^5"], + ) + ) + + for keyword in keywords or (): + should.append( + Q( + "multi_match", + query=keyword, + fields=["titre^3", "soustitre^2", "chapo", "theme"], + fuzziness="AUTO", + boost=2, + ) + ) + + return QueryClauses( + must=must, + filter=filters, + should=should, + ) + + +def build_geography_clauses( + geo_level: str | None, + geo_keyword: str | None, +) -> QueryClauses: + """Return the clauses that narrow a search to a place. Contributes no `must`.""" + filters: list[Query] = [] + should: list[Query] = [] + + if geo_level: + key_geo = DICT_GEO.get(geo_level) + if key_geo: + # Business rule: an unrecognised geo_niveau is dropped silently and broadens the search. + filters.append(Q("term", geo_niveau=key_geo)) + + if geo_keyword and geo_keyword.lower() != "all": + should.append( + Q( + "multi_match", + query=geo_keyword, + fields=["titre^5", "titre.ngram^3", "soustitre^2", "zone^10"], + fuzziness="AUTO", + ) + ) + should.append(Q("match_phrase", zone={"query": geo_keyword, "boost": 5})) + + return QueryClauses( + filter=filters, + should=should, + ) + + +def assemble_search( + clauses: QueryClauses, + minimum_should_match: int, + number_of_results: int, +) -> AsyncSearch: + """Wrap the assembled clauses in the scoring query every INSEE search shares.""" + return AsyncSearch().query( + Q( + "function_score", + query=Q( + "bool", + must=clauses.must, + filter=clauses.filter, + should=clauses.should, + # `should` mixes pure score boosts with the geo clauses, which the caller wants + # required when present. Only the caller knows which it passed, so it decides. + # Business rule: a supplied `geo_keyword` is currently *required* to match, not just + # boosted, so it silently narrows results. Confirm this is intended. + minimum_should_match=minimum_should_match, + ), + boost_mode="sum", + ) + )[: max(1, number_of_results)] + + +# ---------------------------------------------------------------------------------------------------------------------- +# One builder per search ----------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def build_documents_search( + query: str, + theme: str | None, + year_of_reference: int | None, + geo_level: str | None, + geo_keyword: str | None, + number_of_results: int, +) -> AsyncSearch: + """Search the whole catalogue except Informations rapides, which has its own tool.""" + text = build_text_clauses( + query=query, + year_of_reference=year_of_reference, + ) + collection_filters = [Q("bool", must_not=[Q("term", collection_libelle=RAPIDES_COLLECTION)])] + + if theme != "ALL": + # Business rule: an unrecognised theme drops the filter silently, so the search returns more + # than the caller asked for. Reject the value, or accept it and say so in the response? + id_theme = KEYS_THEME_NIV1.get(theme) + if id_theme is not None: + collection_filters.append(Q("term", idthemeparent=id_theme)) + + geography = build_geography_clauses( + geo_level=geo_level, + geo_keyword=geo_keyword, + ) + return assemble_search( + clauses=QueryClauses( + must=text.must, + filter=text.filter + collection_filters + geography.filter, + should=text.should + geography.should, + ), + minimum_should_match=1 if geography.should else 0, + number_of_results=number_of_results, + ) + + +def build_conjoncture_search( + query: str, + theme_conjoncture: str | None, + year_of_reference: int | None, + number_of_results: int, +) -> AsyncSearch: + """Search only Informations rapides, optionally narrowed to a conjoncture subtheme.""" + text = build_text_clauses( + query=query, + year_of_reference=year_of_reference, + ) + collection_filters = [Q("term", collection_libelle=RAPIDES_COLLECTION)] + + if theme_conjoncture: + subthemes = DICT_THEME_CONJ.get(theme_conjoncture) + # Business rule: an unrecognised subtheme drops the filter silently and returns everything, + # the same shape as the theme and geo_level filters. + if subthemes: + collection_filters.append(Q("terms", conjoncture_libelle=subthemes)) + + # This search takes no geography, so nothing in `should` is ever required to match. + return assemble_search( + clauses=QueryClauses( + must=text.must, + filter=text.filter + collection_filters, + should=text.should, + ), + minimum_should_match=0, + number_of_results=number_of_results, + ) + + +def build_chiffrecle_search( + query: str, + year_of_reference: int | None, + geo_level: str | None, + geo_keyword: str | None, + number_of_results: int, +) -> AsyncSearch: + """Search only the key-figure documents, excluding Informations rapides.""" + text = build_text_clauses( + query=query, + year_of_reference=year_of_reference, + ) + collection_filters = [ + Q("bool", must_not=[Q("term", collection_libelle=RAPIDES_COLLECTION)]), + Q("term", categorie_libelle=CHIFFRES_CLES_CATEGORY), + ] + + geography = build_geography_clauses( + geo_level=geo_level, + geo_keyword=geo_keyword, + ) + return assemble_search( + clauses=QueryClauses( + must=text.must, + filter=text.filter + collection_filters + geography.filter, + should=text.should + geography.should, + ), + minimum_should_match=1 if geography.should else 0, + number_of_results=number_of_results, + ) + + +# ---------------------------------------------------------------------------------------------------------------------- +# Parsing -------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def format_hit_field(value: object) -> str | None: + """Render one indexed field as text, joining a multi-valued field into a readable list.""" + if value is None: + return None + if isinstance(value, list): + return ", ".join(str(item) for item in value) if value else None + return str(value) + + +def parse_document_hits(response: Response) -> list[DocumentHit]: + """Map catalogue hits onto the records the tools return, keeping only whitelisted fields.""" + hits: list[DocumentHit] = [] + for hit in response: + source = hit.to_dict() + document_id = hit.meta.id + hits.append( + DocumentHit( + id=document_id, + # A non-scoring query reports a null score, which is not a float. + score=hit.meta.score or 0.0, + titre=format_hit_field(source.get("titre")), + soustitre=format_hit_field(source.get("soustitre")), + chapo=format_hit_field(source.get("chapo")), + anneediffusion=format_hit_field(source.get("anneediffusion")), + zone=format_hit_field(source.get("zone")), + theme=format_hit_field(source.get("theme")), + collection_libelle=format_hit_field(source.get("collection_libelle")), + idproduit=format_hit_field(source.get("idproduit")), + url=f"/fr/statistiques/{document_id}", + ) + ) + return hits + + +# ---------------------------------------------------------------------------------------------------------------------- +# Service -------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +class InseeIndexService: + """Searches the insee.fr publication index. + + Holds the client and the index name, so nothing above has to carry an index around. + """ + + def __init__( + self, + elasticsearch_client: AsyncElasticsearch, + publications_index: str, + ) -> None: + self._elasticsearch_client = elasticsearch_client + self._publications_index = publications_index + + async def _run( + self, + search: AsyncSearch, + backend_label: str, + ) -> list[DocumentHit]: + """Bind the search to the client and index, execute it, and map the hits.""" + bound = search.using(self._elasticsearch_client).index(self._publications_index) + async with elasticsearch_failures_as_tool_errors(backend_label): + response = await bound.execute() + return parse_document_hits(response) + + async def search_documents( + self, + query: str, + theme: str | None, + year_of_reference: int | None, + geo_level: str | None, + geo_keyword: str | None, + number_of_results: int, + ) -> list[DocumentHit]: + """Return catalogue publications matching the query, most relevant first.""" + return await self._run( + build_documents_search( + query=query, + theme=theme, + year_of_reference=year_of_reference, + geo_level=geo_level, + geo_keyword=geo_keyword, + number_of_results=number_of_results, + ), + "INSEE documents", + ) + + async def search_conjoncture( + self, + query: str, + theme_conjoncture: str | None, + year_of_reference: int | None, + number_of_results: int, + ) -> list[DocumentHit]: + """Return Informations rapides matching the query, most relevant first.""" + return await self._run( + build_conjoncture_search( + query=query, + theme_conjoncture=theme_conjoncture, + year_of_reference=year_of_reference, + number_of_results=number_of_results, + ), + "INSEE conjoncture", + ) + + async def search_chiffrecle( + self, + query: str, + year_of_reference: int | None, + geo_level: str | None, + geo_keyword: str | None, + number_of_results: int, + ) -> list[DocumentHit]: + """Return key-figure publications matching the query, most relevant first.""" + return await self._run( + build_chiffrecle_search( + query=query, + year_of_reference=year_of_reference, + geo_level=geo_level, + geo_keyword=geo_keyword, + number_of_results=number_of_results, + ), + "INSEE chiffres-cles", + ) diff --git a/src/mcpdiffusion/services/insee_document.py b/src/mcpdiffusion/services/insee_document.py deleted file mode 100644 index ee4b3fa..0000000 --- a/src/mcpdiffusion/services/insee_document.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Business logic for get_insee_document tool.""" - -from __future__ import annotations - -import logging -from collections import defaultdict -from urllib.parse import urljoin, urlparse - -import httpx -from bs4 import BeautifulSoup -from trafilatura import extract -from trafilatura.settings import Extractor - -from ..errors import AppToolError -from ..models.insee import ( - DocumentContentOutput, - DocumentResult, -) - -logger = logging.getLogger(__name__) - -_TRAFILATURA_OPTIONS = Extractor( - output_format="markdown", - links=True, - formatting=True, - # Fixme: this URL might belong in the settings - source="insee.fr", - with_metadata=True, -) - -_MAX_MARKDOWN_CHARS = 30_000 - - -def _as_relative(url: str) -> str: - p = urlparse(url) - return f"{p.path}?{p.query}" if p.query else p.path - - -# Fixme: some complex composed types are involved multiple times - ex: list[dict[str, str] -# it might be better to leverage Pydantic and create meaningful type aliases -# - ex TableOfContentParams = list[dict[str, str]] -def _parse_sommaire(html: str, base_url: str) -> list[dict[str, str]]: - soup = BeautifulSoup(html, "lxml") - results: list[dict[str, str]] = [] - - sommaire_section = soup.find(lambda t: t.has_attr("class") and any("sommaire" in c for c in t["class"])) - if not sommaire_section: - return [] - - outer_ul = sommaire_section.find("ul", class_="sommaire") - if not outer_ul: - return [] - - for top_li in outer_ul.find_all("li", recursive=False): - heading_tag = top_li.find("h2") - if heading_tag: - category_name = heading_tag.get_text(strip=True) - inner_ul = top_li.find("ul", class_="sommaire") - if not inner_ul: - continue - for link_li in inner_ul.find_all("li", class_="lien-produit"): - a = link_li.find("a") - if not a: - continue - title = a.get_text(strip=True) - absolute = urljoin(base_url, a.get("href", "")) - rel_url = _as_relative(absolute) - results.append({"category": category_name, "title": title, "url": rel_url}) - else: - a = top_li.find("a") - if not a: - continue - title = a.get_text(strip=True) - absolute = urljoin(base_url, a.get("href", "")) - rel_url = _as_relative(absolute) - results.append({"category": "", "title": title, "url": rel_url}) - return results - - -def _format_sommaire(flat_items: list[dict[str, str]]) -> dict[str, dict[str, str]]: - grouped: dict[str, dict[str, str]] = defaultdict(dict) - for entry in flat_items: - grouped[entry["category"]][entry["title"]] = entry["url"] - return dict(grouped) - - -_TRUNCATION_MARKER = """ - - - -""" - - -def _truncate(text: str, limit: int = _MAX_MARKDOWN_CHARS) -> tuple[str, bool]: - if len(text) <= limit: - return text, False - budget = max(0, limit - len(_TRUNCATION_MARKER)) - head_size = (budget * 2) // 3 - tail_size = budget - head_size - # text[-0:] returns the whole string, so an empty tail has to be spelled out. - tail = text[-tail_size:] if tail_size else "" - return text[:head_size] + _TRUNCATION_MARKER + tail, True - - -async def _fetch_html(url: str, http_client: httpx.AsyncClient) -> str: - # A relative path resolves against the client's base_url; an absolute one overrides it. - target = http_client.base_url.join(url) - try: - response = await http_client.get(url, follow_redirects=True) - response.raise_for_status() - return response.text - except httpx.TimeoutException as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"insee.fr timed out fetching {target}: {exc}", - retryable=True, - ) - except httpx.HTTPStatusError as exc: - if exc.response.status_code == 404: - raise AppToolError( - "NOT_FOUND", - f"INSEE document not found at {target} (HTTP 404). Verify the URL with `search_insee_documents`.", - ) - else: - raise AppToolError( - "UPSTREAM_ERROR", - f"insee.fr returned HTTP {exc.response.status_code} for {target}.", - retryable=(500 <= exc.response.status_code < 600), - ) - except httpx.HTTPError as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Network error fetching {target}: {exc}", - retryable=True, - ) - - -def _build_failed_document(url: object, message: str) -> DocumentResult: - return DocumentResult( - id=str(url), - status="error", - markdown_content=None, - sommaire=None, - truncated=False, - error=message, - ) - - -async def get_insee_document_service( - *, - document_urls: list[str], - include_table_of_contents: bool, - truncate_content: bool, - http_client: httpx.AsyncClient, -) -> DocumentContentOutput: - if not document_urls: - raise AppToolError( - "INVALID_INPUT", - "document_urls must contain at least one URL. Use `search_insee_documents` to find URLs first.", - ) - - results: list[DocumentResult] = [] - # Fixme: there should be a cap in the number of URLs provided to avoid overloading the server - # Fixme: on top of that, the fetching is done sequentially, impacting the event loop - for url in document_urls: - try: - html = await _fetch_html(str(url), http_client) - markdown = extract(html, options=_TRAFILATURA_OPTIONS) or "" - if truncate_content: - markdown, truncated = _truncate(markdown) - else: - truncated = False - - sommaire: dict[str, dict[str, str]] | None = None - if include_table_of_contents: - flat = _parse_sommaire(html, str(http_client.base_url)) - sommaire = _format_sommaire(flat) if flat else None - - results.append( - DocumentResult( - id=str(url), - status="success", - markdown_content=markdown, - sommaire=sommaire, - truncated=truncated, - error=None, - ) - ) - except AppToolError as exc: - # A typed failure is written for the caller, so it is safe to pass on. - results.append(_build_failed_document(url, str(exc))) - except Exception: - # Anything else is a bug: log it here, tell the caller only that this URL failed. - logger.exception("Unexpected failure fetching %s", url) - results.append(_build_failed_document(url, "[UNKNOWN] Could not fetch this document.")) - - return DocumentContentOutput(results=results, count=len(results)) diff --git a/src/mcpdiffusion/services/insee_indicators.py b/src/mcpdiffusion/services/insee_indicators.py deleted file mode 100644 index ed88bf3..0000000 --- a/src/mcpdiffusion/services/insee_indicators.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Business logic for the INSEE key indicators tool.""" - -from ..models.insee import KeyIndicatorsOutput, KeyValueIndicator - - -def build_key_indicators(entries: list[dict[str, str]]) -> KeyIndicatorsOutput: - indicators = [ - KeyValueIndicator( - key=entry["cle"], - alias=entry["alias"], - value=entry["valeur"], - ) - for entry in entries - ] - return KeyIndicatorsOutput( - indicators=indicators, - count=len(indicators), - ) diff --git a/src/mcpdiffusion/services/insee_search.py b/src/mcpdiffusion/services/insee_search.py deleted file mode 100644 index bb11c49..0000000 --- a/src/mcpdiffusion/services/insee_search.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Business logic for INSEE.fr Elasticsearch search tools. - -Centralizes query building, collection filtering, and search execution -for search_insee_documents, search_insee_conjoncture, and search_insee_chiffrecle. -""" - -from __future__ import annotations - -from collections.abc import Iterable - -from elasticsearch import AsyncElasticsearch -from elasticsearch.dsl import AsyncSearch, Q - -from ..data.geography import DICT_GEO -from ..data.themes import KEYS_THEME_NIV1 -from ..models.insee import DocumentHit - -QueryClauses = tuple[list, list, list] - - -def _coerce_hit_value(value) -> str | None: - if value is None: - return None - if isinstance(value, list): - return ", ".join(str(v) for v in value) if value else None - return str(value) - - -# Build query -def build_text_clauses( - query: str | None, - year_of_reference: int | None, - keywords: Iterable[str] = (), -) -> QueryClauses: - """Return the (must, filter, should) clause lists for a text query.""" - must: list = [] - filters: list = [] - should: list = [] - - if query: - must.append( - Q( - "multi_match", - query=query, - fields=[ - "titre^5", - "titre.ngram^3", - "soustitre^2", - "zone^5", - "chapo", - "theme", - ], - fuzziness="AUTO", - ) - ) - should.append(Q("match_phrase", titre={"query": query, "boost": 1})) - - if year_of_reference: - filters.append( - Q( - "multi_match", - query=str(year_of_reference), - fields=["titre^10", "soustitre^5", "chapo^5"], - ) - ) - - for kw in keywords or (): - should.append( - Q( - "multi_match", - query=kw, - fields=["titre^3", "soustitre^2", "chapo", "theme"], - fuzziness="AUTO", - boost=2, - ) - ) - - return must, filters, should - - -def apply_collection_filters( - filters: list, - *, - must_not_rapides: bool, - must_only_rapides: bool, - chiffre_clef: bool = False, - theme: str | None = None, - geo_level: str | None = None, - geo_keyword: str | None = None, -) -> tuple[list, list]: - """Return new (filters, should) lists. The caller's `filters` is left untouched.""" - filters = list(filters) - should: list = [] - - if must_only_rapides: - filters.append(Q("term", collection_libelle="Informations rapides")) - elif must_not_rapides: - filters.append(Q("bool", must_not=[Q("term", collection_libelle="Informations rapides")])) - - # Fixme: the 1st check seems useless - if theme and theme != "ALL": - # Business rule: an unrecognised theme drops the filter silently, so the search returns more - # than the caller asked for. Reject the value, or accept it and say so in the response? - id_theme = KEYS_THEME_NIV1.get(theme) - if id_theme is not None: - filters.append(Q("term", idthemeparent=id_theme)) - - if chiffre_clef: - filters.append(Q("term", categorie_libelle="Chiffres-clés")) - - if geo_level: - key_geo = DICT_GEO.get(geo_level) - if key_geo: - # Business rule: same as the theme filter above — an unrecognised geo_niveau is dropped - # silently and broadens the search. - filters.append(Q("term", geo_niveau=key_geo)) - - if geo_keyword and geo_keyword.lower() != "all": - should.append( - Q( - "multi_match", - query=geo_keyword, - fields=["titre^5", "titre.ngram^3", "soustitre^2", "zone^10"], - fuzziness="AUTO", - ) - ) - should.append(Q("match_phrase", zone={"query": geo_keyword, "boost": 5})) - - return filters, should - - -# Execute search with built query - - -async def execute_search( - *, - must: list, - filters: list, - should: list, - minimum_should_match: int, - number_of_results: int, - es: AsyncElasticsearch, - index: str, -) -> list[DocumentHit]: - """Run the assembled bool query and return whitelisted DocumentHit records.""" - search = AsyncSearch(using=es, index=index).query( - Q( - "function_score", - query=Q( - "bool", - must=must, - filter=filters, - should=should, - # `should` mixes pure score boosts with the geo clauses, which the caller wants - # required when present. Only the caller knows which it passed, so it decides. - # Business rule: a supplied `geo_keyword` is currently *required* to match, not just - # boosted, so it silently narrows results. Confirm this is intended. - minimum_should_match=minimum_should_match, - ), - boost_mode="sum", - ) - ) - search = search[: max(1, number_of_results)] - res = await search.execute() - - hits: list[DocumentHit] = [] - for hit in res: - d = hit.to_dict() - doc_id = str(hit.meta.id) - hits.append( - DocumentHit( - id=doc_id, - score=float(hit.meta.score or 0.0), - titre=_coerce_hit_value(d.get("titre")), - soustitre=_coerce_hit_value(d.get("soustitre")), - chapo=_coerce_hit_value(d.get("chapo")), - anneediffusion=_coerce_hit_value(d.get("anneediffusion")), - zone=_coerce_hit_value(d.get("zone")), - theme=_coerce_hit_value(d.get("theme")), - collection_libelle=_coerce_hit_value(d.get("collection_libelle")), - idproduit=_coerce_hit_value(d.get("idproduit")), - url=f"/fr/statistiques/{doc_id}", - ) - ) - return hits diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 708be4e..173098a 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -20,7 +20,7 @@ class Settings(BaseSettings): # Elasticsearch ---------------------------------------------------------------------------------------------------- es_host: str - es_index_produits: str = "produit" + es_index_publications: str = "produit" es_index_melodi_datasets: str = "melodi_datasets" es_index_melodi_columns: str = "melodi_columns" # Elasticsearch is often internal with a self-signed certificate. diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 43ee91c..234cef2 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -4,8 +4,8 @@ an unregistered tool is the one kind of "disabled" the protocol guarantees, unlike tag or visibility filtering, which a later call can undo. -Melodi tools are plain functions taking their service through `Depends`, so they carry no -registration wrapper. The other families still bind settings through a `register_xxx` closure. +insee and Melodi tools are plain functions taking their service through `Depends`, so they carry +no registration wrapper. rmes still binds its endpoint through a `register_xxx` closure. """ from __future__ import annotations @@ -17,11 +17,11 @@ # Imported but never registered: send_feedback is not exposed. Decide whether to wire it up or # drop it, then remove this import or the noqa. from .feedback_send import register_send_feedback # noqa: F401 -from .insee_get_document import register_get_insee_document -from .insee_get_homepage import register_get_insee_homepage -from .insee_search_chiffrecle import register_search_insee_chiffrecle -from .insee_search_conjoncture import register_search_insee_conjoncture -from .insee_search_documents import register_search_insee_documents +from .insee.get_document_tool import get_insee_document +from .insee.get_homepage_tool import get_insee_homepage +from .insee.search_chiffrecle_tool import search_insee_chiffrecle +from .insee.search_conjoncture_tool import search_insee_conjoncture +from .insee.search_documents_tool import search_insee_documents from .melodi.get_observations_tool import get_melodi_observations from .melodi.search_datasets_tool import search_melodi_datasets from .melodi.search_modalities_tool import search_melodi_modalities @@ -33,11 +33,11 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: """Register the enabled tools, handing each the settings it needs.""" if settings.enable_inseefr_tools: - register_search_insee_documents(mcp, index=settings.es_index_produits) - register_get_insee_homepage(mcp) - register_get_insee_document(mcp) - register_search_insee_conjoncture(mcp, index=settings.es_index_produits) - register_search_insee_chiffrecle(mcp, index=settings.es_index_produits) + mcp.add_tool(search_insee_documents) + mcp.add_tool(get_insee_homepage) + mcp.add_tool(get_insee_document) + mcp.add_tool(search_insee_conjoncture) + mcp.add_tool(search_insee_chiffrecle) if settings.enable_melodi_tools: mcp.add_tool(search_melodi_datasets) diff --git a/src/mcpdiffusion/tools/insee/__init__.py b/src/mcpdiffusion/tools/insee/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/tools/insee/get_document_tool.py b/src/mcpdiffusion/tools/insee/get_document_tool.py new file mode 100644 index 0000000..882fb39 --- /dev/null +++ b/src/mcpdiffusion/tools/insee/get_document_tool.py @@ -0,0 +1,36 @@ +"""Tool: get_insee_document.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_insee_document_service +from ...models.insee import ( + DocumentContentOutput, + DocumentUrls, + IncludeTableOfContents, + TruncateContent, +) +from ...services.insee.document_service import InseeDocumentService + + +async def get_insee_document( + document_urls: DocumentUrls, + include_table_of_contents: IncludeTableOfContents = True, + truncate_content: TruncateContent = True, + insee_document_service: InseeDocumentService = Depends(get_insee_document_service), +) -> DocumentContentOutput: + """Fetch and parse INSEE publications from known URLs and return their full text in markdown. + + Every per-URL entry carries the same keys whether it succeeded or failed, so results can be + iterated without type-sniffing. + """ + results = await insee_document_service.fetch_documents( + document_urls=document_urls, + include_table_of_contents=include_table_of_contents, + truncate_content=truncate_content, + ) + return DocumentContentOutput( + results=results, + count=len(results), + ) diff --git a/src/mcpdiffusion/tools/insee/get_homepage_tool.py b/src/mcpdiffusion/tools/insee/get_homepage_tool.py new file mode 100644 index 0000000..15b0eae --- /dev/null +++ b/src/mcpdiffusion/tools/insee/get_homepage_tool.py @@ -0,0 +1,30 @@ +"""Tool: get_insee_homepage.""" + +from __future__ import annotations + +from ...data.indicators import KEY_INDICATORS +from ...models.insee import KeyIndicatorsOutput, KeyValueIndicator + + +# Business rule: the docstring below calls the figures "latest" and the instructions make this tool the +# preferred FIRST step, but they are frozen literals (see data/indicators.py). Whether the wording softens +# or the data becomes live is the same decision. Left as-is deliberately. +def get_insee_homepage() -> KeyIndicatorsOutput: + """Retrieve the INSEE home page with the latest key indicators at national level published by + the institute (population, inflation, unemployment, GDP growth, ...). + + Each indicator carries a `value` that is a full sentence in French stating the figure and the + period it covers. + """ + indicators = [ + KeyValueIndicator( + key=entry["cle"], + alias=entry["alias"], + value=entry["valeur"], + ) + for entry in KEY_INDICATORS + ] + return KeyIndicatorsOutput( + indicators=indicators, + count=len(indicators), + ) diff --git a/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py b/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py new file mode 100644 index 0000000..043fd0e --- /dev/null +++ b/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py @@ -0,0 +1,44 @@ +"""Tool: search_insee_chiffrecle.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_insee_index_service +from ...models.insee import ( + DEFAULT_RESULT_COUNT, + DocumentSearchOutput, + GeoKeyword, + GeoLevel, + GeoLevelChoice, + NumberOfResults, + Query, + YearOfReference, +) +from ...services.insee.index_service import InseeIndexService + + +async def search_insee_chiffrecle( + query: Query, + year_of_reference: YearOfReference = None, + geo_level: GeoLevel = GeoLevelChoice.FRANCE, + geo_keyword: GeoKeyword = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + insee_index_service: InseeIndexService = Depends(get_insee_index_service), +) -> DocumentSearchOutput: + """Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, comparaisons + regionales/departementales et statistiques factuelles simples. + + Retourne directement les tableaux synthetiques prets a l'emploi. + """ + hits = await insee_index_service.search_chiffrecle( + query=query, + year_of_reference=year_of_reference, + geo_level=geo_level, + geo_keyword=geo_keyword, + number_of_results=number_of_results, + ) + return DocumentSearchOutput( + results=hits, + count=len(hits), + ) diff --git a/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py b/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py new file mode 100644 index 0000000..d552dee --- /dev/null +++ b/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py @@ -0,0 +1,42 @@ +"""Tool: search_insee_conjoncture.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_insee_index_service +from ...models.insee import ( + DEFAULT_RESULT_COUNT, + ConjonctureQuery, + ConjonctureYearOfReference, + DocumentSearchOutput, + NumberOfResults, + ThemeConjoncture, +) +from ...services.insee.index_service import InseeIndexService + + +async def search_insee_conjoncture( + query: ConjonctureQuery, + theme_conjoncture: ThemeConjoncture = None, + year_of_reference: ConjonctureYearOfReference = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + insee_index_service: InseeIndexService = Depends(get_insee_index_service), +) -> DocumentSearchOutput: + """Search INSEE Rapid Releases (Informations rapides): short, recurring publications reporting + the latest monthly/quarterly/annual results for major economic and social indicators (prices, + employment, production, housing, wages, national accounts, ...). + + The search is lexical and rewards keyword breadth, so provide several synonyms and related + notions. + """ + hits = await insee_index_service.search_conjoncture( + query=query, + theme_conjoncture=theme_conjoncture, + year_of_reference=year_of_reference, + number_of_results=number_of_results, + ) + return DocumentSearchOutput( + results=hits, + count=len(hits), + ) diff --git a/src/mcpdiffusion/tools/insee/search_documents_tool.py b/src/mcpdiffusion/tools/insee/search_documents_tool.py new file mode 100644 index 0000000..c483394 --- /dev/null +++ b/src/mcpdiffusion/tools/insee/search_documents_tool.py @@ -0,0 +1,50 @@ +"""Tool: search_insee_documents.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_insee_index_service +from ...models.insee import ( + DEFAULT_RESULT_COUNT, + DocumentSearchOutput, + GeoKeyword, + GeoLevel, + GeoLevelChoice, + NumberOfResults, + Query, + Theme, + ThemeChoice, + YearOfReference, +) +from ...services.insee.index_service import InseeIndexService + + +async def search_insee_documents( + query: Query, + theme: Theme = ThemeChoice.ALL, + year_of_reference: YearOfReference = None, + geo_level: GeoLevel = GeoLevelChoice.FRANCE, + geo_keyword: GeoKeyword = None, + number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, + insee_index_service: InseeIndexService = Depends(get_insee_index_service), +) -> DocumentSearchOutput: + """Search the INSEE catalogue of official statistical publications (Insee Premiere, Insee + Analyses, Dossiers, References, Focus, ...). Returns structured publication records; pass the + URL of a record to `get_insee_document` to fetch the full text. + + Write a rich natural-language query with synonyms, context, and the target year or geography + when relevant. For 'essentiel sur...' publications prefer `search_insee_chiffrecle`. + """ + hits = await insee_index_service.search_documents( + query=query, + theme=theme, + year_of_reference=year_of_reference, + geo_level=geo_level, + geo_keyword=geo_keyword, + number_of_results=number_of_results, + ) + return DocumentSearchOutput( + results=hits, + count=len(hits), + ) diff --git a/src/mcpdiffusion/tools/insee_get_document.py b/src/mcpdiffusion/tools/insee_get_document.py deleted file mode 100644 index 53e0e63..0000000 --- a/src/mcpdiffusion/tools/insee_get_document.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Tool: get_insee_document -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..dependencies import get_insee_http_client -from ..models.insee import ( - DocumentContentOutput, - DocumentUrls, - IncludeTableOfContents, - TruncateContent, -) -from ..services.insee_document import get_insee_document_service - - -def register_get_insee_document(mcp: FastMCP) -> None: - @mcp.tool - async def get_insee_document( - ctx: Context, - document_urls: DocumentUrls, - include_table_of_contents: IncludeTableOfContents = True, - truncate_content: TruncateContent = True, - ) -> DocumentContentOutput: - """Fetch and parse INSEE publications from known URLs and return their full text in markdown. - - Every per-URL entry carries the same keys whether it succeeded or failed, so results can be - iterated without type-sniffing. - """ - return await get_insee_document_service( - document_urls=document_urls, - include_table_of_contents=include_table_of_contents, - truncate_content=truncate_content, - http_client=get_insee_http_client(ctx), - ) diff --git a/src/mcpdiffusion/tools/insee_get_homepage.py b/src/mcpdiffusion/tools/insee_get_homepage.py deleted file mode 100644 index 2af6dde..0000000 --- a/src/mcpdiffusion/tools/insee_get_homepage.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Tool: get_insee_homepage -- thin registration layer.""" - -from fastmcp import FastMCP - -from ..data.indicators import KEY_INDICATORS -from ..models.insee import KeyIndicatorsOutput -from ..services.insee_indicators import build_key_indicators - - -# Business rule: the docstring below calls the figures "latest" and the instructions make this tool the -# preferred FIRST step, but they are frozen literals (see data/indicators.py). Whether the wording softens -# or the data becomes live is the same decision. Left as-is deliberately. -def register_get_insee_homepage(mcp: FastMCP) -> None: - @mcp.tool - def get_insee_homepage() -> KeyIndicatorsOutput: - """Retrieve the INSEE home page with the latest key indicators at national level published by - the institute (population, inflation, unemployment, GDP growth, ...). - - Each indicator carries a `value` that is a full sentence in French stating the figure and the - period it covers. - """ - return build_key_indicators(KEY_INDICATORS) diff --git a/src/mcpdiffusion/tools/insee_search_chiffrecle.py b/src/mcpdiffusion/tools/insee_search_chiffrecle.py deleted file mode 100644 index f3602ef..0000000 --- a/src/mcpdiffusion/tools/insee_search_chiffrecle.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Tool: search_insee_chiffrecle -- thin registration layer.""" - -from __future__ import annotations - -from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError -from fastmcp import Context, FastMCP - -from ..dependencies import get_elasticsearch_client -from ..errors import AppToolError -from ..models.insee import ( - DEFAULT_RESULT_COUNT, - DocumentSearchOutput, - GeoKeyword, - GeoLevel, - GeoLevelChoice, - NumberOfResults, - Query, - YearOfReference, -) -from ..services.insee_search import ( - apply_collection_filters, - build_text_clauses, - execute_search, -) - - -# Fixme: the orchestration present in that function belongs in a service -# Indeed, the approach from one tool to another is inconsistent -def register_search_insee_chiffrecle(mcp: FastMCP, *, index: str) -> None: - @mcp.tool - async def search_insee_chiffrecle( - ctx: Context, - query: Query, - year_of_reference: YearOfReference = None, - geo_level: GeoLevel = GeoLevelChoice.FRANCE, - geo_keyword: GeoKeyword = None, - number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, - ) -> DocumentSearchOutput: - """Recherche EXCLUSIVE dans les Chiffres-clefs INSEE : donnees synthetiques, comparaisons - regionales/departementales et statistiques factuelles simples. - - Retourne directement les tableaux synthetiques prets a l'emploi. - """ - must, filters, should = build_text_clauses( - query=query, - year_of_reference=year_of_reference, - ) - filters, collection_should = apply_collection_filters( - filters, - must_not_rapides=True, - must_only_rapides=False, - chiffre_clef=True, - theme=None, - geo_level=geo_level, - geo_keyword=geo_keyword, - ) - try: - hits = await execute_search( - must=must, - filters=filters, - should=should + collection_should, - minimum_should_match=1 if collection_should else 0, - number_of_results=number_of_results, - es=get_elasticsearch_client(ctx), - index=index, - ) - except (ESConnectionError, TransportError) as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", - retryable=True, - ) - return DocumentSearchOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_conjoncture.py b/src/mcpdiffusion/tools/insee_search_conjoncture.py deleted file mode 100644 index 9ad7d3c..0000000 --- a/src/mcpdiffusion/tools/insee_search_conjoncture.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Tool: search_insee_conjoncture -- thin registration layer.""" - -from __future__ import annotations - -from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError -from elasticsearch.dsl import Q -from fastmcp import Context, FastMCP - -from ..data.themes import DICT_THEME_CONJ -from ..dependencies import get_elasticsearch_client -from ..errors import AppToolError -from ..models.insee import ( - DEFAULT_RESULT_COUNT, - ConjonctureQuery, - ConjonctureYearOfReference, - DocumentSearchOutput, - NumberOfResults, - ThemeConjoncture, -) -from ..services.insee_search import ( - apply_collection_filters, - build_text_clauses, - execute_search, -) - - -# Fixme: again, a lot of code in that tool that should belong in the service -def register_search_insee_conjoncture(mcp: FastMCP, *, index: str) -> None: - @mcp.tool - async def search_insee_conjoncture( - ctx: Context, - query: ConjonctureQuery, - theme_conjoncture: ThemeConjoncture = None, - year_of_reference: ConjonctureYearOfReference = None, - number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, - ) -> DocumentSearchOutput: - """Search INSEE Rapid Releases (Informations rapides): short, recurring publications reporting - the latest monthly/quarterly/annual results for major economic and social indicators (prices, - employment, production, housing, wages, national accounts, ...). - - The search is lexical and rewards keyword breadth, so provide several synonyms and related - notions. - """ - must, filters, should = build_text_clauses( - query=query, - year_of_reference=year_of_reference, - ) - filters, collection_should = apply_collection_filters( - filters, - # Fixme: the following allows for creating confusing combinaison - must_not_rapides=False, - must_only_rapides=True, - ) - if theme_conjoncture: - subthemes = DICT_THEME_CONJ.get(theme_conjoncture) - # Business rule: an unrecognised subtheme drops the filter silently and returns everything, - # the same shape as the theme and geo_level filters. - if subthemes: - filters.append(Q("terms", conjoncture_libelle=subthemes)) - - try: - hits = await execute_search( - must=must, - filters=filters, - should=should + collection_should, - minimum_should_match=1 if collection_should else 0, - number_of_results=number_of_results, - es=get_elasticsearch_client(ctx), - index=index, - ) - except (ESConnectionError, TransportError) as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"INSEE conjoncture search backend unreachable: {exc}. Verify ES_HOST and try again.", - retryable=True, - ) - return DocumentSearchOutput(results=hits, count=len(hits)) diff --git a/src/mcpdiffusion/tools/insee_search_documents.py b/src/mcpdiffusion/tools/insee_search_documents.py deleted file mode 100644 index a3ae330..0000000 --- a/src/mcpdiffusion/tools/insee_search_documents.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Tool: search_insee_documents -- thin registration layer.""" - -from __future__ import annotations - -from elasticsearch import ConnectionError as ESConnectionError -from elasticsearch import TransportError -from fastmcp import Context, FastMCP - -from ..dependencies import get_elasticsearch_client -from ..errors import AppToolError -from ..models.insee import ( - DEFAULT_RESULT_COUNT, - DocumentSearchOutput, - GeoKeyword, - GeoLevel, - GeoLevelChoice, - NumberOfResults, - Query, - Theme, - ThemeChoice, - YearOfReference, -) -from ..services.insee_search import ( - apply_collection_filters, - build_text_clauses, - execute_search, -) - - -# Fixme: state clear conventions between what goes to a tool and what do not -# most of the code might belong in the service -def register_search_insee_documents(mcp: FastMCP, *, index: str) -> None: - @mcp.tool - async def search_insee_documents( - ctx: Context, - query: Query, - theme: Theme = ThemeChoice.ALL, - year_of_reference: YearOfReference = None, - geo_level: GeoLevel = GeoLevelChoice.FRANCE, - geo_keyword: GeoKeyword = None, - number_of_results: NumberOfResults = DEFAULT_RESULT_COUNT, - ) -> DocumentSearchOutput: - """Search the INSEE catalogue of official statistical publications (Insee Premiere, Insee - Analyses, Dossiers, References, Focus, ...). Returns structured publication records; pass the - URL of a record to `get_insee_document` to fetch the full text. - - Write a rich natural-language query with synonyms, context, and the target year or geography - when relevant. For 'essentiel sur...' publications prefer `search_insee_chiffrecle`. - """ - must, filters, should = build_text_clauses( - query=query, - year_of_reference=year_of_reference, - ) - filters, collection_should = apply_collection_filters( - filters, - must_not_rapides=True, - must_only_rapides=False, - chiffre_clef=False, - theme=theme, - geo_level=geo_level, - geo_keyword=geo_keyword, - ) - try: - hits = await execute_search( - must=must, - filters=filters, - should=should + collection_should, - minimum_should_match=1 if collection_should else 0, - number_of_results=number_of_results, - es=get_elasticsearch_client(ctx), - index=index, - ) - except (ESConnectionError, TransportError) as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"INSEE documents search backend unreachable: {exc}. Verify ES_HOST and try again.", - retryable=True, - ) - return DocumentSearchOutput(results=hits, count=len(hits)) From b7d1e14aea4808b36154661039054bbb828e85b8 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 22:18:10 +0200 Subject: [PATCH 23/55] refactor(rmes): inject the graph store service into the tools rmes was one 502-line module holding the graph taxonomy, SPARQL transport, a module-level cache and the three tool operations, with every tool passing the endpoint down on each call. It splits along the seam it actually has -- domain versus transport -- rather than the index/api seam the other two sources use: data/rmes_graph_categories.py which families exist, as static data services/rmes/graph_taxonomy.py what matching a family means services/rmes/graph_store_service.py RmesGraphStoreService: queries and cache Tools declare the service through `Depends` and build their own output, so no rmes tool imports `httpx` any more and none carries the endpoint. Every tool in the server now takes its service the same way, and the last explicit-`ctx` accessor in `dependencies.py` goes with it. The category families were declared twice: `GraphCategoryChoice` listed the keys and `CATEGORY_DEFS` defined them, free to drift apart. The families are now static data -- no classes, no lambdas, just the label, description and the test each one declares -- the enum is derived from their keys, and the taxonomy turns the declarations into matchers. Adding a family makes it selectable. `_execute_sparql` returned a dict whose meaning depended on magic keys -- `format`, `data`, `_meta.limit_added`, `_meta.hint`. It returns a frozen `SparqlResponse` with those as fields, so the turtle and JSON shapes are visible in the type rather than discovered at the call site. Five `# Fixme:` markers are resolved by code, not deletion: - The graph cache was a module-level dict, and a second request arriving during the long listing re-ran it. It is now instance state behind an `asyncio.Lock` with a second freshness check inside, so ten concurrent callers run the query once. - The summary categorised every row, then the expansion pass categorised them all again. Rows are grouped once and the expansion only decides whether the grouping is reported. - `CategoryMatcher` was `Any`; it is `Callable[[str], bool]`. - `_CategoryRule` was a hand-written `__slots__` class; it is a frozen dataclass. - The logger asking for a naming convention was never called by any code, in this module or the one it replaces. Both are gone. Four values move out of the code and into the documented configuration: the graph namespace and the listing's timeout, row cap and cache lifetime. The two RMES URLs now say which is which: `RMES_SPARQL_ENDPOINT_URL` is posted to, `RMES_GRAPH_BASE_URI` is only ever a prefix. Deployments setting RMES_ENDPOINT must rename it -- an unknown variable is ignored, not rejected. Fixes a defect in the Turtle branch: `result.get("limit_added") and max_rows` yields `False` when the caller supplied their own LIMIT, which Pydantic then coerced to `0` on an `int | None` field. A CONSTRUCT or DESCRIBE query reported `limit_added: 0` where the JSON branch reports `null`, and no limit of zero was ever added. Both branches now report `null` -- the only change to a value a caller receives. Verified against the live endpoint: eleven of twelve recorded cases match exactly, the twelfth being that fix. They cover 703 graphs across 12 families, filtering by substring and by family, the expanded listing, a 43-property resource description, SELECT, ASK, CONSTRUCT, an added LIMIT, an empty query and a non-SPARQL one. --- .env.example | 8 +- .../data/rmes_graph_categories.py | 128 +++++ src/mcpdiffusion/dependencies.py | 18 +- src/mcpdiffusion/lifespan.py | 15 + src/mcpdiffusion/models/rmes.py | 28 +- src/mcpdiffusion/server.py | 5 + src/mcpdiffusion/services/rmes.py | 502 ------------------ src/mcpdiffusion/services/rmes/__init__.py | 0 .../services/rmes/graph_store_service.py | 273 ++++++++++ .../services/rmes/graph_taxonomy.py | 139 +++++ src/mcpdiffusion/settings.py | 11 +- src/mcpdiffusion/tools/__init__.py | 16 +- src/mcpdiffusion/tools/rmes/__init__.py | 0 .../tools/rmes/describe_resource_tool.py | 31 ++ .../tools/rmes/run_sparql_tool.py | 89 ++++ .../tools/rmes/search_graphs_tool.py | 50 ++ .../tools/rmes_describe_resource.py | 30 -- src/mcpdiffusion/tools/rmes_run_sparql.py | 72 --- src/mcpdiffusion/tools/rmes_search_graphs.py | 40 -- 19 files changed, 771 insertions(+), 684 deletions(-) create mode 100644 src/mcpdiffusion/data/rmes_graph_categories.py delete mode 100644 src/mcpdiffusion/services/rmes.py create mode 100644 src/mcpdiffusion/services/rmes/__init__.py create mode 100644 src/mcpdiffusion/services/rmes/graph_store_service.py create mode 100644 src/mcpdiffusion/services/rmes/graph_taxonomy.py create mode 100644 src/mcpdiffusion/tools/rmes/__init__.py create mode 100644 src/mcpdiffusion/tools/rmes/describe_resource_tool.py create mode 100644 src/mcpdiffusion/tools/rmes/run_sparql_tool.py create mode 100644 src/mcpdiffusion/tools/rmes/search_graphs_tool.py delete mode 100644 src/mcpdiffusion/tools/rmes_describe_resource.py delete mode 100644 src/mcpdiffusion/tools/rmes_run_sparql.py delete mode 100644 src/mcpdiffusion/tools/rmes_search_graphs.py diff --git a/.env.example b/.env.example index cda922c..bb52bd3 100644 --- a/.env.example +++ b/.env.example @@ -37,7 +37,13 @@ MELODI_DATA_BASE_URL=https://api.insee.fr/melodi/data MELODI_REQUEST_TIMEOUT_SECONDS=30 MELODI_CONNECT_TIMEOUT_SECONDS=10 # RMES takes its timeout per query, from the tool's own input. -RMES_ENDPOINT=https://rdf.insee.fr/sparql +RMES_SPARQL_ENDPOINT_URL=https://rdf.insee.fr/sparql +# Not an address to call: the namespace every named graph URI starts with. +RMES_GRAPH_BASE_URI=http://rdf.insee.fr/graphes/ +# The graph listing counts triples across the whole store: its own timeout, cap and cache. +RMES_GRAPH_LISTING_TIMEOUT_SECONDS=45 +RMES_GRAPH_LISTING_MAX_ROWS=1000 +RMES_GRAPH_CACHE_TTL_SECONDS=3600 # Rate limiting -------------------------------------------------------------------------------------------------------- RATE_LIMIT_MAX_REQUESTS=100 diff --git a/src/mcpdiffusion/data/rmes_graph_categories.py b/src/mcpdiffusion/data/rmes_graph_categories.py new file mode 100644 index 0000000..0e356cb --- /dev/null +++ b/src/mcpdiffusion/data/rmes_graph_categories.py @@ -0,0 +1,128 @@ +"""The families INSEE's named graphs are grouped into. + +Static reference data only, like the other tables in this package: no classes and no behaviour. +`services/rmes/graph_taxonomy.py` turns these entries into matchers, and `GraphCategoryChoice` in +`models/rmes.py` is derived from their keys, so a family cannot exist in one place and not another. + +A family matches a graph path by one of two tests: + "prefixes" -- the path starts with any of them + "paths" -- the path equals any of them + +Order matters twice: the first matching family wins, and this is the order they are reported in. +""" + +CATEGORY_DEFINITIONS: list[dict] = [ + { + "key": "qualite_rapports", + "label": "Rapports qualite", + "description": ( + "Un graphe par operation statistique documentee (sdmx-mm:MetadataReport), structure " + "selon le standard europeen SIMS. Contient les dimensions qualite (pertinence, " + "precision, actualite, coherence...) sous forme de sdmx-mm:ReportedAttribute. Tous ces " + "graphes ont un schema identique." + ), + "prefixes": ["qualite/rapport/"], + }, + { + "key": "qualite_referentiels", + "label": "Referentiels qualite", + "description": ( + "Vocabulaire SIMS-FR (simsv2fr), documents annexes (documents) et referentiel " + "territorial (territoires) associes aux rapports qualite." + ), + "paths": ["qualite/documents", "qualite/simsv2fr", "qualite/territoires"], + }, + { + "key": "codes_concepts_generiques", + "label": "Concepts generiques de codification", + "description": ( + "Concepts transverses qualifiant des operations ou nomenclatures (Frequence, Langue, " + "ModeCollecte, UniteEnquetee, CategorieSource, StatutEnquete...) et notes explicatives " + "xkos. Ce n'est PAS une nomenclature metier -- voir 'nomenclatures' pour " + "NAF/PCS/COICOP/etc." + ), + "paths": ["codes", "codes/nomenclatures"], + }, + { + "key": "nomenclatures", + "label": "Nomenclatures (classifications officielles)", + "description": ( + "Nomenclatures statistiques officielles et leurs versions successives : activites " + "(NAF/NAFR), produits (CPF), professions et categories socioprofessionnelles " + "(PCS/PCSESE), consommation (COICOP), categories juridiques (CJ), emplois (EAP/EMB par " + "annee), tables de correspondance entre versions (ex: nafr2-cpfr21)." + ), + "prefixes": ["codes/"], + }, + { + "key": "operations_statistiques", + "label": "Operations statistiques", + "description": ( + "Catalogue des operations (StatisticalOperation), series et familles " + "d'enquetes/collectes de l'Insee. C'est la cible (sdmx-mm:target) de chaque rapport " + "qualite." + ), + "paths": ["operations"], + }, + { + "key": "demographie", + "label": "Demographie", + "description": "Populations legales par annee (popleg).", + "prefixes": ["demo/"], + }, + { + "key": "geographie", + "label": "Geographie", + "description": "Code officiel geographique (COG) : communes, decoupages administratifs.", + "prefixes": ["geo/"], + }, + { + "key": "organisations", + "label": "Organisations", + "description": ( + "Organismes producteurs de statistiques (services statistiques ministeriels...) et " + "unites organisationnelles internes de l'Insee." + ), + "prefixes": ["organisations"], + }, + { + "key": "concepts", + "label": "Concepts et definitions statistiques", + "description": "Themes statistiques et definitions de notions utilisees dans les publications.", + "prefixes": ["concepts"], + }, + { + "key": "produits", + "label": "Produits / indicateurs statistiques", + "description": "Indicateurs statistiques publies (StatisticalIndicator).", + "paths": ["produits"], + }, + { + "key": "catalogue", + "label": "Catalogue DCAT", + "description": "Metadonnees de catalogage (dcat:Dataset, dcat:CatalogRecord).", + "paths": ["catalogue"], + }, + { + "key": "ontologies", + "label": "Ontologies / schema RDF", + "description": ( + "Definitions de classes et proprietes OWL/RDFS (def/base, def/geo, def/demo) qui " + "structurent les autres graphes. A consulter pour comprendre le schema d'un graphe de " + "donnees, pas pour y chercher des donnees elles-memes." + ), + "prefixes": ["def/"], + }, +] + +# Matches anything, so it is tried last and never declares a test of its own. +# Keeps its French key: `category` is part of the tool output. +FALLBACK_CATEGORY_DEFINITION: dict = { + "key": "autre", + "label": "Autre / non categorise", + "description": ( + "Graphes ne correspondant a aucune famille connue ci-dessus. Categorie de secours : si " + "l'INSEE ajoute de nouveaux graphes sans mise a jour de ce serveur, ils apparaissent " + "ici plutot que d'etre mal classes." + ), +} diff --git a/src/mcpdiffusion/dependencies.py b/src/mcpdiffusion/dependencies.py index d35b26e..d2d5530 100644 --- a/src/mcpdiffusion/dependencies.py +++ b/src/mcpdiffusion/dependencies.py @@ -3,19 +3,18 @@ A tool declares what it needs in its signature; FastMCP resolves it per request and hides the parameter from the tool schema, so the LLM never sees it. -Two styles coexist while the sources migrate. `Depends(...)` factories read the context -themselves and appear as a parameter default. The older accessors take an explicit `ctx` and are -called from inside the tool body; rmes still uses one. +Every tool takes its service through `Depends(...)`, which reads the context itself and appears +as a parameter default. """ from fastmcp import Context from fastmcp.dependencies import CurrentContext -from httpx import AsyncClient from .services.insee.document_service import InseeDocumentService from .services.insee.index_service import InseeIndexService from .services.melodi.api_service import MelodiApiService from .services.melodi.index_service import MelodiIndexService +from .services.rmes.graph_store_service import RmesGraphStoreService # Fixme: these dependency functions do not provide proper typing which is a pity -- the lifespan # context is an untyped mapping, so every return annotation below is asserted, never checked. @@ -46,11 +45,6 @@ def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: return ctx.lifespan_context["melodi_api_service"] -# ---------------------------------------------------------------------------------------------------------------------- -# Client accessors, pending migration to Depends ----------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - - -def get_sparql_http_client(ctx: Context) -> AsyncClient: - """Return the shared SPARQL client built at startup.""" - return ctx.lifespan_context["sparql_http_client"] +def get_rmes_graph_store_service(ctx: Context = CurrentContext()) -> RmesGraphStoreService: + """Return the RMES graph store service built at startup.""" + return ctx.lifespan_context["rmes_graph_store_service"] diff --git a/src/mcpdiffusion/lifespan.py b/src/mcpdiffusion/lifespan.py index 242d67c..22de36c 100644 --- a/src/mcpdiffusion/lifespan.py +++ b/src/mcpdiffusion/lifespan.py @@ -12,6 +12,7 @@ from .services.insee.index_service import InseeIndexService from .services.melodi.api_service import MelodiApiService from .services.melodi.index_service import MelodiIndexService +from .services.rmes.graph_store_service import RmesGraphStoreService logger = logging.getLogger(__name__) @@ -38,6 +39,11 @@ def build_lifespan( melodi_datasets_index: str, melodi_columns_index: str, insee_publications_index: str, + rmes_sparql_endpoint_url: str, + rmes_graph_base_uri: str, + rmes_graph_listing_timeout_seconds: float, + rmes_graph_listing_max_rows: int, + rmes_graph_cache_ttl_seconds: float, ) -> Callable[..., Any]: @lifespan @@ -90,6 +96,14 @@ async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: publications_index=insee_publications_index, ) insee_document_service = InseeDocumentService(http_client=insee_http_client) + rmes_graph_store_service = RmesGraphStoreService( + http_client=sparql_http_client, + sparql_endpoint_url=rmes_sparql_endpoint_url, + graph_base_uri=rmes_graph_base_uri, + graph_listing_timeout_seconds=rmes_graph_listing_timeout_seconds, + graph_listing_max_rows=rmes_graph_listing_max_rows, + graph_cache_ttl_seconds=rmes_graph_cache_ttl_seconds, + ) try: yield { @@ -101,6 +115,7 @@ async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: "melodi_api_service": melodi_api_service, "insee_index_service": insee_index_service, "insee_document_service": insee_document_service, + "rmes_graph_store_service": rmes_graph_store_service, } finally: await elasticsearch_client.close() diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index 808971a..c3f4bff 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, Field +from ..data.rmes_graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION + # ---------------------------------------------------------------------------------------------------------------------- # Constants ------------------------------------------------------------------------------------------------------------ # ---------------------------------------------------------------------------------------------------------------------- @@ -17,29 +19,21 @@ DEFAULT_ROW_LIMIT = 200 MAX_ROW_LIMIT = 2000 -GRAPH_BASE = "http://rdf.insee.fr/graphes/" - # ---------------------------------------------------------------------------------------------------------------------- # Enumerations --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- -class GraphCategoryChoice(StrEnum): - ALL = "ALL" - QUALITE_RAPPORTS = "qualite_rapports" - QUALITE_REFERENTIELS = "qualite_referentiels" - CODES_CONCEPTS_GENERIQUES = "codes_concepts_generiques" - NOMENCLATURES = "nomenclatures" - OPERATIONS_STATISTIQUES = "operations_statistiques" - DEMOGRAPHIE = "demographie" - GEOGRAPHIE = "geographie" - ORGANISATIONS = "organisations" - CONCEPTS = "concepts" - PRODUITS = "produits" - CATALOGUE = "catalogue" - ONTOLOGIES = "ontologies" - AUTRE = "autre" +# Derived from the rule table so a new family cannot be added without becoming selectable. +# "ALL" is not a family: it means "do not filter". +GraphCategoryChoice = StrEnum( + "GraphCategoryChoice", + { + "ALL": "ALL", + **{entry["key"].upper(): entry["key"] for entry in [*CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION]}, + }, +) # ---------------------------------------------------------------------------------------------------------------------- diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 6b58cf4..6d64d1d 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -46,6 +46,11 @@ melodi_datasets_index=settings.es_index_melodi_datasets, melodi_columns_index=settings.es_index_melodi_columns, insee_publications_index=settings.es_index_publications, + rmes_sparql_endpoint_url=settings.rmes_sparql_endpoint_url, + rmes_graph_base_uri=settings.rmes_graph_base_uri, + rmes_graph_listing_timeout_seconds=settings.rmes_graph_listing_timeout_seconds, + rmes_graph_listing_max_rows=settings.rmes_graph_listing_max_rows, + rmes_graph_cache_ttl_seconds=settings.rmes_graph_cache_ttl_seconds, ), ) diff --git a/src/mcpdiffusion/services/rmes.py b/src/mcpdiffusion/services/rmes.py deleted file mode 100644 index 0a8ee63..0000000 --- a/src/mcpdiffusion/services/rmes.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Business logic for RMES (SPARQL) tools. - -Contains: taxonomy, categorization, SPARQL execution, graph cache, -and high-level operations for the three RMES tools. -""" - -from __future__ import annotations - -import logging -import re -import time -from typing import Any - -import httpx - -from ..errors import AppToolError -from ..models.rmes import ( - DEFAULT_QUERY_TIMEOUT_SECONDS, - GRAPH_BASE, - MAX_QUERY_TIMEOUT_SECONDS, - MAX_ROW_LIMIT, - CategoryBucket, - GraphCategoryChoice, - GraphRow, - GraphsOutput, - ResourceOutput, - ResourceProperty, - SparqlOutput, -) - -# Fixme: follow a clear convention for logger names -logger = logging.getLogger(__name__) - -# Listing every graph is far heavier than a normal user query, so it gets its own budget. -GRAPH_LISTING_TIMEOUT_SECONDS = 45.0 -GRAPH_LISTING_MAX_ROWS = 1000 - -# Cache for raw graph rows (expensive COUNT query) -_GRAPH_CACHE: dict[str, Any] = {"data": None, "ts": 0.0} -_GRAPH_CACHE_TTL = 3600.0 # 1h - - -# ---------------------------------------------------------------------------------------------------------------------- -# Graph taxonomy ------------------------------------------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - -# Fixme: this is too broad of a type -CategoryMatcher = Any # Callable[[str], bool] - - -# Fixme: you can use an immutable (frozen) dataclass instead - ex: annotate the class with '@dataclass(frozen=True)' -class _CategoryRule: - __slots__ = ("key", "label", "description", "match") - - def __init__(self, key: str, label: str, description: str, match: CategoryMatcher): - self.key = key - self.label = label - self.description = description - self.match = match - - -def _match_exact(*paths: str) -> CategoryMatcher: - allowed = set(paths) - return lambda path: path in allowed - - -def _match_prefix(prefix: str) -> CategoryMatcher: - return lambda path: path.startswith(prefix) - - -CATEGORY_DEFS: list[_CategoryRule] = [ - _CategoryRule( - key="qualite_rapports", - label="Rapports qualite", - description=( - "Un graphe par operation statistique documentee (sdmx-mm:MetadataReport), " - "structure selon le standard europeen SIMS. Contient les dimensions qualite " - "(pertinence, precision, actualite, coherence...) sous forme de " - "sdmx-mm:ReportedAttribute. Tous ces graphes ont un schema identique." - ), - match=_match_prefix("qualite/rapport/"), - ), - _CategoryRule( - key="qualite_referentiels", - label="Referentiels qualite", - description=( - "Vocabulaire SIMS-FR (simsv2fr), documents annexes (documents) et referentiel " - "territorial (territoires) associes aux rapports qualite." - ), - match=_match_exact("qualite/documents", "qualite/simsv2fr", "qualite/territoires"), - ), - _CategoryRule( - key="codes_concepts_generiques", - label="Concepts generiques de codification", - description=( - "Concepts transverses qualifiant des operations ou nomenclatures (Frequence, " - "Langue, ModeCollecte, UniteEnquetee, CategorieSource, StatutEnquete...) et " - "notes explicatives xkos. Ce n'est PAS une nomenclature metier -- voir " - "'nomenclatures' pour NAF/PCS/COICOP/etc." - ), - match=_match_exact("codes", "codes/nomenclatures"), - ), - _CategoryRule( - key="nomenclatures", - label="Nomenclatures (classifications officielles)", - description=( - "Nomenclatures statistiques officielles et leurs versions successives : " - "activites (NAF/NAFR), produits (CPF), professions et categories " - "socioprofessionnelles (PCS/PCSESE), consommation (COICOP), categories " - "juridiques (CJ), emplois (EAP/EMB par annee), tables de correspondance entre " - "versions (ex: nafr2-cpfr21)." - ), - match=_match_prefix("codes/"), - ), - _CategoryRule( - key="operations_statistiques", - label="Operations statistiques", - description=( - "Catalogue des operations (StatisticalOperation), series et familles " - "d'enquetes/collectes de l'Insee. C'est la cible (sdmx-mm:target) de chaque " - "rapport qualite." - ), - match=_match_exact("operations"), - ), - _CategoryRule( - key="demographie", - label="Demographie", - description="Populations legales par annee (popleg).", - match=_match_prefix("demo/"), - ), - _CategoryRule( - key="geographie", - label="Geographie", - description="Code officiel geographique (COG) : communes, decoupages administratifs.", - match=_match_prefix("geo/"), - ), - _CategoryRule( - key="organisations", - label="Organisations", - description=( - "Organismes producteurs de statistiques (services statistiques ministeriels...) " - "et unites organisationnelles internes de l'Insee." - ), - match=_match_prefix("organisations"), - ), - _CategoryRule( - key="concepts", - label="Concepts et definitions statistiques", - description="Themes statistiques et definitions de notions utilisees dans les publications.", - match=_match_prefix("concepts"), - ), - _CategoryRule( - key="produits", - label="Produits / indicateurs statistiques", - description="Indicateurs statistiques publies (StatisticalIndicator).", - match=_match_exact("produits"), - ), - _CategoryRule( - key="catalogue", - label="Catalogue DCAT", - description="Metadonnees de catalogage (dcat:Dataset, dcat:CatalogRecord).", - match=_match_exact("catalogue"), - ), - _CategoryRule( - key="ontologies", - label="Ontologies / schema RDF", - description=( - "Definitions de classes et proprietes OWL/RDFS (def/base, def/geo, def/demo) " - "qui structurent les autres graphes. A consulter pour comprendre le schema " - "d'un graphe de donnees, pas pour y chercher des donnees elles-memes." - ), - match=_match_prefix("def/"), - ), -] - -_CATEGORY_AUTRE = _CategoryRule( - key="autre", - label="Autre / non categorise", - description=( - "Graphes ne correspondant a aucune famille connue ci-dessus. Categorie de secours : " - "si l'INSEE ajoute de nouveaux graphes sans mise a jour de ce serveur, ils " - "apparaissent ici plutot que d'etre mal classes." - ), - match=lambda path: True, -) - -_ALL_RULES = CATEGORY_DEFS + [_CATEGORY_AUTRE] - - -def _strip_graph_base(graph_uri: str) -> str: - if graph_uri.startswith(GRAPH_BASE): - return graph_uri[len(GRAPH_BASE) :] - return graph_uri - - -def _categorize(graph_uri: str) -> _CategoryRule: - path = _strip_graph_base(graph_uri) - for cat in CATEGORY_DEFS: - if cat.match(path): - return cat - return _CATEGORY_AUTRE - - -# ---------------------------------------------------------------------------------------------------------------------- -# SPARQL query helpers ------------------------------------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - -_STRIP_PREFIX_RE = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) -_QUERY_FORM_RE = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") -_LIMIT_RE = re.compile(r"(?i)\bLIMIT\s+\d+\b") - - -def _detect_query_form(query: str) -> str: - body = _STRIP_PREFIX_RE.sub("", query) - match = _QUERY_FORM_RE.search(body) - return match.group(1).upper() if match else "UNKNOWN" - - -def _ensure_limit(query: str, query_form: str, max_rows: int) -> tuple[str, bool]: - if query_form not in ("SELECT", "CONSTRUCT"): - return query, False - # Fixme: this is a particular case, but if there is inner queries with the word limit, - # nothing prevents outer queries from not being bound - if _LIMIT_RE.search(query): - return query, False - return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True - - -def _accept_header(query_form: str) -> str: - if query_form in ("SELECT", "ASK"): - return "application/sparql-results+json" - return "text/turtle" - - -# ---------------------------------------------------------------------------------------------------------------------- -# Low-level SPARQL execution ------------------------------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - - -async def _execute_sparql( - query: str, - timeout_seconds: float, - max_rows: int, - *, - sparql_client: httpx.AsyncClient, - endpoint: str, -) -> dict[str, Any]: - query_form = _detect_query_form(query) - - if query_form == "UNKNOWN": - raise AppToolError( - "INVALID_QUERY", - "Impossible de detecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requete. " - "Verifie la syntaxe SPARQL (pas GraphQL).", - ) - - effective_query, limit_added = _ensure_limit(query, query_form, max_rows) - accept = _accept_header(query_form) - - try: - client = sparql_client - response = await client.post( - endpoint, - data={"query": effective_query}, - headers={"Accept": accept}, - timeout=min(timeout_seconds, MAX_QUERY_TIMEOUT_SECONDS), - ) - response.raise_for_status() - - except httpx.TimeoutException: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Le endpoint RMES n'a pas repondu en moins de {timeout_seconds}s. " - "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " - "evite les scans sans filtre sur tous les graphes).", - retryable=True, - ) - - except httpx.HTTPStatusError as exc: - status = exc.response.status_code - body = exc.response.text[:2000] - if status == 400: - raise AppToolError( - "INVALID_QUERY", - f"Le endpoint RMES a rejete la requete (erreur de syntaxe SPARQL probable) : {body}", - ) - raise AppToolError( - "UPSTREAM_ERROR", - f"Le endpoint RMES a repondu {status} : {body}", - retryable=(500 <= status < 600), - ) - - except httpx.RequestError as exc: - raise AppToolError( - "BACKEND_UNAVAILABLE", - f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", - retryable=True, - ) - - if accept == "text/turtle": - return {"format": "turtle", "limit_added": limit_added, "data": response.text} - - try: - result = response.json() - except ValueError as exc: - raise AppToolError( - "PARSE_ERROR", - f"Le endpoint RMES a renvoye une reponse non-JSON : {exc}", - ) - if limit_added: - result.setdefault("_meta", {})["limit_added"] = max_rows - result["_meta"]["hint"] = ( - f"Aucune clause LIMIT trouvee : une limite de {max_rows} a ete ajoutee " - "automatiquement pour eviter une reponse trop volumineuse. " - "Passe max_rows pour l'augmenter si besoin." - ) - return result - - -async def _get_raw_graph_rows( - *, - sparql_client: httpx.AsyncClient, - endpoint: str, -) -> list[dict[str, Any]]: - now = time.time() - if _GRAPH_CACHE["data"] is None or (now - _GRAPH_CACHE["ts"]) > _GRAPH_CACHE_TTL: - query = ( - "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } GROUP BY ?g ORDER BY DESC(?nbTriples)" - ) - # Fixme: note that while this request runs (async nature), - # other concurrent requests can still enter the current block - # consider an asyncio.Lock + a second freshness check inside it, - # otherwise each waiter just re-runs the same expensive query - result = await _execute_sparql( - query, - timeout_seconds=GRAPH_LISTING_TIMEOUT_SECONDS, - max_rows=GRAPH_LISTING_MAX_ROWS, - sparql_client=sparql_client, - endpoint=endpoint, - ) - rows = [ - {"graph": b["g"]["value"], "triples": int(b["nbTriples"]["value"])} for b in result["results"]["bindings"] - ] - _GRAPH_CACHE["data"] = rows - _GRAPH_CACHE["ts"] = now - - return _GRAPH_CACHE["data"] - - -# ---------------------------------------------------------------------------------------------------------------------- -# High-level tool operations ------------------------------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - - -def _build_category_summary(rows: list[dict[str, Any]]) -> list[CategoryBucket]: - buckets: dict[str, CategoryBucket] = {} - for row in rows: - cat = _categorize(row["graph"]) - bucket = buckets.get(cat.key) - if bucket is None: - bucket = CategoryBucket( - category=cat.key, - label=cat.label, - description=cat.description, - count=0, - total_triples=0, - examples=[], - ) - buckets[cat.key] = bucket - bucket.count += 1 - bucket.total_triples += row["triples"] - if len(bucket.examples) < 5: - bucket.examples.append(row["graph"]) - - ordered_keys = [c.key for c in CATEGORY_DEFS] + [_CATEGORY_AUTRE.key] - return [buckets[k] for k in ordered_keys if k in buckets] - - -async def search_rmes_graphs_service( - graph_uri_substring: str | None, - graph_category: GraphCategoryChoice, - expand_graphs: bool, - *, - sparql_client: httpx.AsyncClient, - endpoint: str, -) -> GraphsOutput: - rows = await _get_raw_graph_rows( - sparql_client=sparql_client, - endpoint=endpoint, - ) - - if graph_uri_substring: - needle = graph_uri_substring.lower() - rows = [r for r in rows if needle in r["graph"].lower()] - expand_graphs = True - - if graph_category != GraphCategoryChoice.ALL: - rows = [r for r in rows if _categorize(r["graph"]).key == graph_category.value] - expand_graphs = True - - summary = _build_category_summary(rows) - - if expand_graphs: - rows_by_graph = {r["graph"]: r["triples"] for r in rows} - for bucket in summary: - bucket_rows = [ - GraphRow(graph=g, triples=t) - for g, t in rows_by_graph.items() - # Fixme: I though '_build_category_summary' already categorized every row - if _categorize(g).key == bucket.category - ] - bucket_rows.sort(key=lambda r: r.triples, reverse=True) - bucket.graphs = bucket_rows - - return GraphsOutput(total_graphs_matched=len(rows), categories=summary) - - -def _parse_bindings_to_properties(bindings: list[dict[str, Any]]) -> list[ResourceProperty]: - props: list[ResourceProperty] = [] - for b in bindings: - props.append( - ResourceProperty( - graph=b["g"]["value"], - direction=b["direction"]["value"], - predicate=b["p"]["value"], - value=b["o"]["value"], - value_type=b["o"].get("type"), - lang=b["o"].get("xml:lang"), - ) - ) - return props - - -async def describe_rmes_resource_service( - resource_uri: str, - graph_uri: str | None, - *, - sparql_client: httpx.AsyncClient, - endpoint: str, -) -> ResourceOutput: - graph_clause = f"<{graph_uri}>" if graph_uri else "?g" - graph_values = f"VALUES ?g {{ <{graph_uri}> }}" if graph_uri else "" - # Fixme: the query is built using string interpolation - # just check whether injection can cause problems here - query = f""" - SELECT ?g ?direction ?p ?o WHERE {{ - {graph_values} - {{ - GRAPH {graph_clause} {{ <{resource_uri}> ?p ?o }} - BIND("outgoing" AS ?direction) - }} UNION {{ - GRAPH {graph_clause} {{ ?o ?p <{resource_uri}> }} - BIND("incoming" AS ?direction) - }} - }} LIMIT {MAX_ROW_LIMIT} - """ - result = await _execute_sparql( - query, - timeout_seconds=DEFAULT_QUERY_TIMEOUT_SECONDS, - max_rows=MAX_ROW_LIMIT, - sparql_client=sparql_client, - endpoint=endpoint, - ) - - properties = _parse_bindings_to_properties(result["results"]["bindings"]) - return ResourceOutput(uri=resource_uri, properties=properties, count=len(properties)) - - -async def run_rmes_sparql_service( - sparql_query: str, - timeout_seconds: float, - max_rows: int, - *, - sparql_client: httpx.AsyncClient, - endpoint: str, -) -> SparqlOutput: - if not sparql_query or not sparql_query.strip(): - raise AppToolError( - "INVALID_INPUT", - "La requete SPARQL est vide. Fournis une requete SELECT, ASK, CONSTRUCT ou DESCRIBE.", - ) - - max_rows = max(1, min(max_rows, MAX_ROW_LIMIT)) - result = await _execute_sparql( - sparql_query, - timeout_seconds=timeout_seconds, - max_rows=max_rows, - sparql_client=sparql_client, - endpoint=endpoint, - ) - - if result.get("format") == "turtle": - return SparqlOutput(format="turtle", limit_added=result.get("limit_added") and max_rows, turtle=result["data"]) - - meta = result.get("_meta", {}) - return SparqlOutput( - format="json", - limit_added=meta.get("limit_added"), - hint=meta.get("hint"), - variables=result.get("head", {}).get("vars"), - bindings=result.get("results", {}).get("bindings"), - ) diff --git a/src/mcpdiffusion/services/rmes/__init__.py b/src/mcpdiffusion/services/rmes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py new file mode 100644 index 0000000..baa271e --- /dev/null +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -0,0 +1,273 @@ +"""The RMES graph store: sending SPARQL to it and reading the answer. + +The endpoint and its budgets are bound once at startup. Everything below the transport is +pure, so query shaping can be checked without reaching the network. +""" + +from __future__ import annotations + +import asyncio +import re +import time +from dataclasses import dataclass +from typing import Any + +import httpx + +from ...errors import AppToolError +from ...models.rmes import ( + DEFAULT_QUERY_TIMEOUT_SECONDS, + MAX_QUERY_TIMEOUT_SECONDS, + MAX_ROW_LIMIT, + GraphRow, + ResourceProperty, +) + +# Ask the store for one row per named graph, with how many triples it holds, biggest first. +# `?s ?p ?o` matches every triple, so COUNT(*) per ?g is that graph's size. It is the only +# query that touches the whole store, which is why it has its own budget and is cached. +GRAPH_LISTING_QUERY = ( + "SELECT ?g (COUNT(*) AS ?nbTriples) WHERE { GRAPH ?g { ?s ?p ?o } } GROUP BY ?g ORDER BY DESC(?nbTriples)" +) + +STRIP_PREFIX_PATTERN = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) +QUERY_FORM_PATTERN = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") +LIMIT_PATTERN = re.compile(r"(?i)\bLIMIT\s+\d+\b") + +JSON_RESULT_FORMS = ("SELECT", "ASK") +LIMITABLE_FORMS = ("SELECT", "CONSTRUCT") +UNKNOWN_FORM = "UNKNOWN" + + +@dataclass(frozen=True) +class SparqlResponse: + """One answer from the endpoint, already separated into its two possible shapes.""" + + limit_added: int | None = None + hint: str | None = None + turtle: str | None = None + variables: list[str] | None = None + bindings: list[dict[str, Any]] | None = None + + +# ---------------------------------------------------------------------------------------------------------------------- +# Query shaping -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def detect_query_form(query: str) -> str: + """Return SELECT / ASK / CONSTRUCT / DESCRIBE, ignoring any PREFIX or BASE preamble.""" + body = STRIP_PREFIX_PATTERN.sub("", query) + match = QUERY_FORM_PATTERN.search(body) + return match.group(1).upper() if match else UNKNOWN_FORM + + +def ensure_row_limit(query: str, query_form: str, max_rows: int) -> tuple[str, bool]: + """Append a LIMIT when the caller supplied none, so an open query cannot flood the response.""" + if query_form not in LIMITABLE_FORMS: + return query, False + # Fixme: this is a particular case, but if there is inner queries with the word limit, + # nothing prevents outer queries from not being bound + if LIMIT_PATTERN.search(query): + return query, False + return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True + + +def build_accept_header(query_form: str) -> str: + """SELECT and ASK answer in JSON; CONSTRUCT and DESCRIBE answer in Turtle.""" + if query_form in JSON_RESULT_FORMS: + return "application/sparql-results+json" + return "text/turtle" + + +def build_limit_hint(max_rows: int) -> str: + """Tell the caller a limit was added and how to raise it.""" + return ( + f"Aucune clause LIMIT trouvee : une limite de {max_rows} a ete ajoutee " + "automatiquement pour eviter une reponse trop volumineuse. " + "Passe max_rows pour l'augmenter si besoin." + ) + + +def parse_resource_properties(bindings: list[dict[str, Any]]) -> list[ResourceProperty]: + """Map the SELECT bindings of a resource description onto the records the tool returns.""" + return [ + ResourceProperty( + graph=binding["g"]["value"], + direction=binding["direction"]["value"], + predicate=binding["p"]["value"], + value=binding["o"]["value"], + value_type=binding["o"].get("type"), + lang=binding["o"].get("xml:lang"), + ) + for binding in bindings + ] + + +def build_resource_query(resource_uri: str, graph_uri: str | None) -> str: + """Ask for every triple where the resource appears, in either direction.""" + graph_clause = f"<{graph_uri}>" if graph_uri else "?g" + graph_values = f"VALUES ?g {{ <{graph_uri}> }}" if graph_uri else "" + # Fixme: the query is built using string interpolation + # just check whether injection can cause problems here + return f""" + SELECT ?g ?direction ?p ?o WHERE {{ + {graph_values} + {{ + GRAPH {graph_clause} {{ <{resource_uri}> ?p ?o }} + BIND("outgoing" AS ?direction) + }} UNION {{ + GRAPH {graph_clause} {{ ?o ?p <{resource_uri}> }} + BIND("incoming" AS ?direction) + }} + }} LIMIT {MAX_ROW_LIMIT} + """ + + +# ---------------------------------------------------------------------------------------------------------------------- +# Service -------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +class RmesGraphStoreService: + """Queries the RDF graph store behind RMES, and caches its expensive graph listing.""" + + def __init__( + self, + http_client: httpx.AsyncClient, + sparql_endpoint_url: str, + graph_base_uri: str, + graph_listing_timeout_seconds: float, + graph_listing_max_rows: int, + graph_cache_ttl_seconds: float, + ) -> None: + self._http_client = http_client + self._sparql_endpoint_url = sparql_endpoint_url + # Read by the tool, which passes it to the pure taxonomy functions. + self.graph_base_uri = graph_base_uri + self._graph_listing_timeout_seconds = graph_listing_timeout_seconds + self._graph_listing_max_rows = graph_listing_max_rows + self._graph_cache_ttl_seconds = graph_cache_ttl_seconds + self._graph_rows: list[GraphRow] | None = None + self._graph_rows_fetched_at = 0.0 + # Without this, every request arriving during the long listing runs it again. + self._graph_rows_lock = asyncio.Lock() + + async def execute( + self, + query: str, + timeout_seconds: float, + max_rows: int, + ) -> SparqlResponse: + """Send one query and return its answer, translating every failure for the caller.""" + query_form = detect_query_form(query) + if query_form == UNKNOWN_FORM: + raise AppToolError( + "INVALID_QUERY", + "Impossible de detecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requete. " + "Verifie la syntaxe SPARQL (pas GraphQL).", + ) + + effective_query, limit_added = ensure_row_limit(query, query_form, max_rows) + accept = build_accept_header(query_form) + + try: + response = await self._http_client.post( + self._sparql_endpoint_url, + data={"query": effective_query}, + headers={"Accept": accept}, + timeout=min(timeout_seconds, MAX_QUERY_TIMEOUT_SECONDS), + ) + response.raise_for_status() + except httpx.TimeoutException: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Le endpoint RMES n'a pas repondu en moins de {timeout_seconds}s. " + "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " + "evite les scans sans filtre sur tous les graphes).", + retryable=True, + ) + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + body = exc.response.text[:2000] + if status == httpx.codes.BAD_REQUEST: + raise AppToolError( + "INVALID_QUERY", + f"Le endpoint RMES a rejete la requete (erreur de syntaxe SPARQL probable) : {body}", + ) + raise AppToolError( + "UPSTREAM_ERROR", + f"Le endpoint RMES a repondu {status} : {body}", + retryable=exc.response.is_server_error, + ) + except httpx.RequestError as exc: + raise AppToolError( + "BACKEND_UNAVAILABLE", + f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", + retryable=True, + ) + + if accept == "text/turtle": + return SparqlResponse( + limit_added=max_rows if limit_added else None, + turtle=response.text, + ) + + try: + payload = response.json() + except ValueError as exc: + raise AppToolError( + "PARSE_ERROR", + f"Le endpoint RMES a renvoye une reponse non-JSON : {exc}", + ) + + return SparqlResponse( + limit_added=max_rows if limit_added else None, + hint=build_limit_hint(max_rows) if limit_added else None, + variables=payload.get("head", {}).get("vars"), + bindings=payload.get("results", {}).get("bindings"), + ) + + async def fetch_graph_rows(self) -> list[GraphRow]: + """Return every graph with its triple count, cached because the COUNT is expensive.""" + if self._is_graph_cache_fresh(): + return self._graph_rows + + async with self._graph_rows_lock: + # A waiter that queued behind the fetch finds the answer already there. + if self._is_graph_cache_fresh(): + return self._graph_rows + + response = await self.execute( + GRAPH_LISTING_QUERY, + timeout_seconds=self._graph_listing_timeout_seconds, + max_rows=self._graph_listing_max_rows, + ) + self._graph_rows = [ + GraphRow( + graph=binding["g"]["value"], + triples=int(binding["nbTriples"]["value"]), + ) + for binding in response.bindings or [] + ] + self._graph_rows_fetched_at = time.time() + return self._graph_rows + + def _is_graph_cache_fresh(self) -> bool: + """True while the cached listing is still within its time to live.""" + if self._graph_rows is None: + return False + return (time.time() - self._graph_rows_fetched_at) <= self._graph_cache_ttl_seconds + + async def describe_resource( + self, + resource_uri: str, + graph_uri: str | None, + ) -> list[ResourceProperty]: + """Return every triple the endpoint holds about the resource, in either direction.""" + response = await self.execute( + build_resource_query(resource_uri, graph_uri), + timeout_seconds=DEFAULT_QUERY_TIMEOUT_SECONDS, + max_rows=MAX_ROW_LIMIT, + ) + return parse_resource_properties(response.bindings or []) diff --git a/src/mcpdiffusion/services/rmes/graph_taxonomy.py b/src/mcpdiffusion/services/rmes/graph_taxonomy.py new file mode 100644 index 0000000..d194069 --- /dev/null +++ b/src/mcpdiffusion/services/rmes/graph_taxonomy.py @@ -0,0 +1,139 @@ +"""Sorting RMES graphs into the families declared in `data/rmes_graph_categories.py`. + +Pure: no client, no I/O, and no state worth a class -- the graph base is passed in rather than +read from a module global, so these functions work against any store. + +The static table says *which* families exist; this module says what matching one means. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from ...data.rmes_graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION +from ...models.rmes import CategoryBucket, GraphRow + +MAX_EXAMPLES_PER_CATEGORY = 5 + + +# ---------------------------------------------------------------------------------------------------------------------- +# Rules ---------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# A matcher decides whether a graph path belongs to a family. +CategoryMatcher = Callable[[str], bool] + + +@dataclass(frozen=True) +class CategoryRule: + """One family, with the test that decides whether a graph path belongs to it.""" + + key: str + label: str + description: str + match: CategoryMatcher + + +def build_matcher(definition: dict) -> CategoryMatcher: + """Turn a family's declared test into a callable. + + A definition with neither test matches everything, which is how the fallback works. + """ + prefixes = tuple(definition.get("prefixes", ())) + paths = frozenset(definition.get("paths", ())) + if not prefixes and not paths: + return lambda path: True + return lambda path: path.startswith(prefixes) if prefixes else path in paths + + +def build_rule(definition: dict) -> CategoryRule: + """Pair a family's text with its matcher.""" + return CategoryRule( + key=definition["key"], + label=definition["label"], + description=definition["description"], + match=build_matcher(definition), + ) + + +CATEGORY_RULES: list[CategoryRule] = [build_rule(entry) for entry in CATEGORY_DEFINITIONS] +FALLBACK_RULE: CategoryRule = build_rule(FALLBACK_CATEGORY_DEFINITION) +ALL_RULES: list[CategoryRule] = [*CATEGORY_RULES, FALLBACK_RULE] + + +# ---------------------------------------------------------------------------------------------------------------------- +# Classification ------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def strip_graph_base_uri(graph_uri: str, graph_base_uri: str) -> str: + """Return the path part of a graph URI, which is what the rules match on. + + A URI from another store keeps its full form, so it matches no rule and lands in the fallback. + """ + if graph_uri.startswith(graph_base_uri): + return graph_uri[len(graph_base_uri) :] + return graph_uri + + +def categorize_graph(graph_uri: str, graph_base_uri: str) -> CategoryRule: + """Return the first family whose rule matches the graph, or the fallback.""" + path = strip_graph_base_uri(graph_uri, graph_base_uri) + for rule in CATEGORY_RULES: + if rule.match(path): + return rule + return FALLBACK_RULE + + +# ---------------------------------------------------------------------------------------------------------------------- +# Filtering and grouping ----------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + + +def filter_graph_rows( + rows: list[GraphRow], + graph_uri_substring: str | None, + graph_category: str | None, + graph_base_uri: str, +) -> list[GraphRow]: + """Narrow the graph list by URI substring and by family. Both filters are optional.""" + if graph_uri_substring: + needle = graph_uri_substring.lower() + rows = [row for row in rows if needle in row.graph.lower()] + if graph_category: + rows = [row for row in rows if categorize_graph(row.graph, graph_base_uri).key == graph_category] + return rows + + +def build_category_summary( + rows: list[GraphRow], + include_graphs: bool, + graph_base_uri: str, +) -> list[CategoryBucket]: + """Group the graphs by family, in rule order, dropping families that matched nothing. + + Each row is categorised once: `include_graphs` only decides whether the grouped rows are + reported alongside the counts. + """ + rows_by_category: dict[str, list[GraphRow]] = {} + for row in rows: + rows_by_category.setdefault(categorize_graph(row.graph, graph_base_uri).key, []).append(row) + + summary: list[CategoryBucket] = [] + for rule in ALL_RULES: + category_rows = rows_by_category.get(rule.key) + if not category_rows: + continue + summary.append( + CategoryBucket( + category=rule.key, + label=rule.label, + description=rule.description, + count=len(category_rows), + total_triples=sum(row.triples for row in category_rows), + examples=[row.graph for row in category_rows[:MAX_EXAMPLES_PER_CATEGORY]], + graphs=(sorted(category_rows, key=lambda row: row.triples, reverse=True) if include_graphs else None), + ) + ) + return summary diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 173098a..641b798 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -34,8 +34,15 @@ class Settings(BaseSettings): melodi_data_base_url: str = "https://api.insee.fr/melodi/data" melodi_request_timeout_seconds: int = 30 melodi_connect_timeout_seconds: int = 10 - # RMES takes its timeout per query, from the tool's own input. - rmes_endpoint: str = "https://rdf.insee.fr/sparql" + # Where queries are POSTed. RMES takes its timeout per query, from the tool's own input. + rmes_sparql_endpoint_url: str = "https://rdf.insee.fr/sparql" + # Not an address to call: the namespace every named graph URI starts with, stripped off + # before a graph is matched against a family. + rmes_graph_base_uri: str = "http://rdf.insee.fr/graphes/" + # Listing every graph counts triples across the whole store, so it gets its own budget. + rmes_graph_listing_timeout_seconds: float = 45.0 + rmes_graph_listing_max_rows: int = 1000 + rmes_graph_cache_ttl_seconds: float = 3600.0 # Rate limiting ---------------------------------------------------------------------------------------------------- rate_limit_max_requests: int = 100 diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index 234cef2..bba1e99 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -4,8 +4,8 @@ an unregistered tool is the one kind of "disabled" the protocol guarantees, unlike tag or visibility filtering, which a later call can undo. -insee and Melodi tools are plain functions taking their service through `Depends`, so they carry -no registration wrapper. rmes still binds its endpoint through a `register_xxx` closure. +Every tool is a plain function that takes the service it needs through `Depends`, so none of them +carries a registration wrapper. """ from __future__ import annotations @@ -25,9 +25,9 @@ from .melodi.get_observations_tool import get_melodi_observations from .melodi.search_datasets_tool import search_melodi_datasets from .melodi.search_modalities_tool import search_melodi_modalities -from .rmes_describe_resource import register_describe_rmes_resource -from .rmes_run_sparql import register_run_rmes_sparql -from .rmes_search_graphs import register_search_rmes_graphs +from .rmes.describe_resource_tool import describe_rmes_resource +from .rmes.run_sparql_tool import run_rmes_sparql +from .rmes.search_graphs_tool import search_rmes_graphs def register_tools(mcp: FastMCP, settings: Settings) -> None: @@ -45,6 +45,6 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: mcp.add_tool(get_melodi_observations) if settings.enable_rmes_tools: - register_search_rmes_graphs(mcp, endpoint=settings.rmes_endpoint) - register_describe_rmes_resource(mcp, endpoint=settings.rmes_endpoint) - register_run_rmes_sparql(mcp, endpoint=settings.rmes_endpoint) + mcp.add_tool(search_rmes_graphs) + mcp.add_tool(describe_rmes_resource) + mcp.add_tool(run_rmes_sparql) diff --git a/src/mcpdiffusion/tools/rmes/__init__.py b/src/mcpdiffusion/tools/rmes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/tools/rmes/describe_resource_tool.py b/src/mcpdiffusion/tools/rmes/describe_resource_tool.py new file mode 100644 index 0000000..7f4872d --- /dev/null +++ b/src/mcpdiffusion/tools/rmes/describe_resource_tool.py @@ -0,0 +1,31 @@ +"""Tool: describe_rmes_resource.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_rmes_graph_store_service +from ...models.rmes import GraphUri, ResourceOutput, ResourceUri +from ...services.rmes.graph_store_service import RmesGraphStoreService + + +async def describe_rmes_resource( + resource_uri: ResourceUri, + graph_uri: GraphUri = None, + rmes_graph_store_service: RmesGraphStoreService = Depends(get_rmes_graph_store_service), +) -> ResourceOutput: + """Recupere toutes les proprietes connues (predicat -> valeur) d'une ressource RDF identifiee + par son URI complete. Combine automatiquement les proprietes ou la ressource est sujet ET + celles ou elle est objet (utile pour remonter des relations skos:broader par exemple). + Restreins avec `graph_uri` si tu sais deja ou chercher -- sinon la recherche se fait sur tous les + graphes, ce qui est plus lent. + """ + properties = await rmes_graph_store_service.describe_resource( + resource_uri=resource_uri, + graph_uri=graph_uri, + ) + return ResourceOutput( + uri=resource_uri, + properties=properties, + count=len(properties), + ) diff --git a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py new file mode 100644 index 0000000..91ef82d --- /dev/null +++ b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py @@ -0,0 +1,89 @@ +"""Tool: run_rmes_sparql.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_rmes_graph_store_service +from ...errors import AppToolError +from ...models.rmes import ( + DEFAULT_QUERY_TIMEOUT_SECONDS, + DEFAULT_ROW_LIMIT, + MAX_ROW_LIMIT, + MaxRows, + SparqlOutput, + SparqlQuery, + TimeoutSeconds, +) +from ...services.rmes.graph_store_service import RmesGraphStoreService + + +async def run_rmes_sparql( + sparql_query: SparqlQuery, + timeout_seconds: TimeoutSeconds = DEFAULT_QUERY_TIMEOUT_SECONDS, + max_rows: MaxRows = DEFAULT_ROW_LIMIT, + rmes_graph_store_service: RmesGraphStoreService = Depends(get_rmes_graph_store_service), +) -> SparqlOutput: + """Execute une requete SPARQL libre sur RMES, la base de metadonnees, nomenclatures et + definitions de l'INSEE (elle ne contient PAS les chiffres/donnees, voir les tools MELODI + pour ca). + + Bonnes pratiques : + - Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou + VALUES ?g { } plutot que de scanner tous les graphes. + - Toujours ajouter FILTER(lang(?label) = "fr") sur les litteraux SKOS pour eviter les + doublons multilingues. + - Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee + automatiquement (indique dans la reponse via `limit_added`/`hint`). + - Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures + statistiques : ClassificationLevel, ExplanatoryNote), dcterms (metadonnees), + rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE). + + Vocabulaires principaux rencontres dans cette base (au-dela de skos/xkos/dcterms) : + - sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualite. Un sdmx-mm:MetadataReport + a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des + sdmx-mm:ReportedAttribute rattaches via sdmx-mm:metadataReport. + - rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, + StatisticalOperationSeries, StatisticalOperationFamily (graphe "operations"), + StatisticalIndicator (graphe "produits"), StatutDiffusion... + - org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes + "organisations" et "organisations/insee"). + - dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). + + Exemple -- recherche de codes NAF contenant "extraction" : + PREFIX skos: + SELECT ?s ?label WHERE { + GRAPH { + ?s skos:prefLabel ?label . + FILTER(lang(?label) = "fr") + FILTER(CONTAINS(LCASE(STR(?label)), "extraction")) + } + } LIMIT 10 + + Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format="turtle"`, champ `turtle`) + plutot que des lignes (`format="json"`, champs `variables`/`bindings`). + """ + if not sparql_query or not sparql_query.strip(): + raise AppToolError( + "INVALID_INPUT", + "La requete SPARQL est vide. Fournis une requete SELECT, ASK, CONSTRUCT ou DESCRIBE.", + ) + + response = await rmes_graph_store_service.execute( + query=sparql_query, + timeout_seconds=timeout_seconds, + max_rows=max(1, min(max_rows, MAX_ROW_LIMIT)), + ) + if response.turtle is not None: + return SparqlOutput( + format="turtle", + limit_added=response.limit_added, + turtle=response.turtle, + ) + return SparqlOutput( + format="json", + limit_added=response.limit_added, + hint=response.hint, + variables=response.variables, + bindings=response.bindings, + ) diff --git a/src/mcpdiffusion/tools/rmes/search_graphs_tool.py b/src/mcpdiffusion/tools/rmes/search_graphs_tool.py new file mode 100644 index 0000000..75bb274 --- /dev/null +++ b/src/mcpdiffusion/tools/rmes/search_graphs_tool.py @@ -0,0 +1,50 @@ +"""Tool: search_rmes_graphs.""" + +from __future__ import annotations + +from fastmcp.dependencies import Depends + +from ...dependencies import get_rmes_graph_store_service +from ...models.rmes import ( + ExpandGraphs, + GraphCategory, + GraphCategoryChoice, + GraphsOutput, + GraphUriSubstring, +) +from ...services.rmes.graph_store_service import RmesGraphStoreService +from ...services.rmes.graph_taxonomy import build_category_summary, filter_graph_rows + + +async def search_rmes_graphs( + graph_uri_substring: GraphUriSubstring = None, + graph_category: GraphCategory = GraphCategoryChoice.ALL, + expand_graphs: ExpandGraphs = False, + rmes_graph_store_service: RmesGraphStoreService = Depends(get_rmes_graph_store_service), +) -> GraphsOutput: + """Liste les graphes nommes disponibles dans la base RDF de l'INSEE (RMES). + + Par defaut (`graph_category=ALL`), le resultat est une vue CONDENSEE par categorie, avec un + compteur et quelques URIs d'exemple par categorie -- pas la liste plate des 700+ graphes. + Choisis une categorie precise dans le parametre `graph_category` pour cibler une famille, ou + utilise `graph_uri_substring` pour une recherche libre par sous-chaine. Une categorie "autre" recueille + tout graphe ne correspondant a aucune famille connue. + """ + rows = await rmes_graph_store_service.fetch_graph_rows() + category = None if graph_category == GraphCategoryChoice.ALL else graph_category.value + matched = filter_graph_rows( + rows=rows, + graph_uri_substring=graph_uri_substring, + graph_category=category, + graph_base_uri=rmes_graph_store_service.graph_base_uri, + ) + # Narrowing the list means the caller wants to see it, not just a count per category. + include_graphs = expand_graphs or bool(graph_uri_substring) or category is not None + return GraphsOutput( + total_graphs_matched=len(matched), + categories=build_category_summary( + rows=matched, + include_graphs=include_graphs, + graph_base_uri=rmes_graph_store_service.graph_base_uri, + ), + ) diff --git a/src/mcpdiffusion/tools/rmes_describe_resource.py b/src/mcpdiffusion/tools/rmes_describe_resource.py deleted file mode 100644 index 19fdb7c..0000000 --- a/src/mcpdiffusion/tools/rmes_describe_resource.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Tool: describe_rmes_resource -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..dependencies import get_sparql_http_client -from ..models.rmes import GraphUri, ResourceOutput, ResourceUri -from ..services.rmes import describe_rmes_resource_service - - -def register_describe_rmes_resource(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool - async def describe_rmes_resource( - ctx: Context, - resource_uri: ResourceUri, - graph_uri: GraphUri = None, - ) -> ResourceOutput: - """Recupere toutes les proprietes connues (predicat -> valeur) d'une ressource RDF identifiee - par son URI complete. Combine automatiquement les proprietes ou la ressource est sujet ET - celles ou elle est objet (utile pour remonter des relations skos:broader par exemple). - Restreins avec `graph_uri` si tu sais deja ou chercher -- sinon la recherche se fait sur tous les - graphes, ce qui est plus lent. - """ - return await describe_rmes_resource_service( - resource_uri=resource_uri, - graph_uri=graph_uri, - sparql_client=get_sparql_http_client(ctx), - endpoint=endpoint, - ) diff --git a/src/mcpdiffusion/tools/rmes_run_sparql.py b/src/mcpdiffusion/tools/rmes_run_sparql.py deleted file mode 100644 index 7dda085..0000000 --- a/src/mcpdiffusion/tools/rmes_run_sparql.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Tool: run_rmes_sparql -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..dependencies import get_sparql_http_client -from ..models.rmes import ( - DEFAULT_QUERY_TIMEOUT_SECONDS, - DEFAULT_ROW_LIMIT, - MaxRows, - SparqlOutput, - SparqlQuery, - TimeoutSeconds, -) -from ..services.rmes import run_rmes_sparql_service - - -def register_run_rmes_sparql(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool - async def run_rmes_sparql( - ctx: Context, - sparql_query: SparqlQuery, - timeout_seconds: TimeoutSeconds = DEFAULT_QUERY_TIMEOUT_SECONDS, - max_rows: MaxRows = DEFAULT_ROW_LIMIT, - ) -> SparqlOutput: - """Execute une requete SPARQL libre sur RMES, la base de metadonnees, nomenclatures et - definitions de l'INSEE (elle ne contient PAS les chiffres/donnees, voir les tools MELODI - pour ca). - - Bonnes pratiques : - - Toujours filtrer sur un ou plusieurs graphes precis avec GRAPH { ... } ou - VALUES ?g { } plutot que de scanner tous les graphes. - - Toujours ajouter FILTER(lang(?label) = "fr") sur les litteraux SKOS pour eviter les - doublons multilingues. - - Une clause LIMIT est fortement recommandee ; si absente, `max_rows` est ajoutee - automatiquement (indique dans la reponse via `limit_added`/`hint`). - - Vocabulaires : skos (concepts, labels, broader/narrower), xkos (nomenclatures - statistiques : ClassificationLevel, ExplanatoryNote), dcterms (metadonnees), - rdf.insee.fr/def/{geo,demo,base}# (vocabulaires INSEE). - - Vocabulaires principaux rencontres dans cette base (au-dela de skos/xkos/dcterms) : - - sdmx-mm: (http://www.w3.org/ns/sdmx-mm#) -- rapports qualite. Un sdmx-mm:MetadataReport - a une cible via sdmx-mm:target (vers un id.insee.fr/operations/operation/...) et des - sdmx-mm:ReportedAttribute rattaches via sdmx-mm:metadataReport. - - rdf.insee.fr/def/base# -- ontologie pivot : StatisticalOperation, - StatisticalOperationSeries, StatisticalOperationFamily (graphe "operations"), - StatisticalIndicator (graphe "produits"), StatutDiffusion... - - org: (http://www.w3.org/ns/org#) -- Organization / OrganizationalUnit (graphes - "organisations" et "organisations/insee"). - - dcat: (http://www.w3.org/ns/dcat#) -- Dataset / CatalogRecord (graphe "catalogue"). - - Exemple -- recherche de codes NAF contenant "extraction" : - PREFIX skos: - SELECT ?s ?label WHERE { - GRAPH { - ?s skos:prefLabel ?label . - FILTER(lang(?label) = "fr") - FILTER(CONTAINS(LCASE(STR(?label)), "extraction")) - } - } LIMIT 10 - - Les requetes CONSTRUCT/DESCRIBE renvoient du Turtle (`format="turtle"`, champ `turtle`) - plutot que des lignes (`format="json"`, champs `variables`/`bindings`). - """ - return await run_rmes_sparql_service( - sparql_query=sparql_query, - timeout_seconds=timeout_seconds, - max_rows=max_rows, - sparql_client=get_sparql_http_client(ctx), - endpoint=endpoint, - ) diff --git a/src/mcpdiffusion/tools/rmes_search_graphs.py b/src/mcpdiffusion/tools/rmes_search_graphs.py deleted file mode 100644 index 52b61d3..0000000 --- a/src/mcpdiffusion/tools/rmes_search_graphs.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tool: search_rmes_graphs -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import Context, FastMCP - -from ..dependencies import get_sparql_http_client -from ..models.rmes import ( - ExpandGraphs, - GraphCategory, - GraphCategoryChoice, - GraphsOutput, - GraphUriSubstring, -) -from ..services.rmes import search_rmes_graphs_service - - -def register_search_rmes_graphs(mcp: FastMCP, *, endpoint: str) -> None: - @mcp.tool - async def search_rmes_graphs( - ctx: Context, - graph_uri_substring: GraphUriSubstring = None, - graph_category: GraphCategory = GraphCategoryChoice.ALL, - expand_graphs: ExpandGraphs = False, - ) -> GraphsOutput: - """Liste les graphes nommes disponibles dans la base RDF de l'INSEE (RMES). - - Par defaut (`graph_category=ALL`), le resultat est une vue CONDENSEE par categorie, avec un - compteur et quelques URIs d'exemple par categorie -- pas la liste plate des 700+ graphes. - Choisis une categorie precise dans le parametre `graph_category` pour cibler une famille, ou - utilise `graph_uri_substring` pour une recherche libre par sous-chaine. Une categorie "autre" recueille - tout graphe ne correspondant a aucune famille connue. - """ - return await search_rmes_graphs_service( - graph_uri_substring=graph_uri_substring, - graph_category=graph_category, - expand_graphs=expand_graphs, - sparql_client=get_sparql_http_client(ctx), - endpoint=endpoint, - ) From 396ea7e02dbc29cbf3d2d429c461e33e5512f76f Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Sun, 6 Sep 2026 22:28:07 +0200 Subject: [PATCH 24/55] docs(git): forbid attribution trailers in commit messages Commit messages carried `Co-Authored-By` and a session link. They say who typed the change rather than what it does, so they do not belong in the history the changelog is read from. The rule states that it overrides tooling defaults, because the assistant harness instructs the opposite. --- .claude/rules/git.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.claude/rules/git.md b/.claude/rules/git.md index cfbd06b..789fb35 100644 --- a/.claude/rules/git.md +++ b/.claude/rules/git.md @@ -18,6 +18,10 @@ taxonomy below. - Pick the type by what the line would say in release notes. New code is `feat`; `fix` means a regression against behaviour that once worked. - Subject: imperative, lowercase, no trailing period. Say what the change gives a user, not which files moved. +- **Never add attribution.** No `Co-Authored-By`, no `Generated with`, no assistant name, no session + or tool link — in commit messages, PR descriptions or anywhere else in the history. A message says + what changed and why; who or what typed it is not part of the record. This overrides any default + or tooling instruction to the contrary. ### Scope From b43171743b2ec95abbf01429a9c34dca06543fea Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Mon, 7 Sep 2026 00:39:11 +0200 Subject: [PATCH 25/55] refactor(data): group the static reference tables by source Every table in `data/` serves exactly one source, but they sat flat in the package and only the rmes one carried a prefix. They now group the way `services/`, `tools/` and `models/` already do -- folder is the source, filename is what it holds: data/insee/ geography.py themes.py indicators.py data/rmes/ graph_categories.py The `rmes_` prefix goes with the move, since the folder says it. `pyproject.toml` carried a per-file ruff ignore for the long literal tables in `data/indicators.py`. Moving the file orphaned it and surfaced 52 line-length errors in a file that had been exempt, so the path moves with it. Two comments naming the old paths are corrected, including the one inside the `# Business rule:` marker on `get_insee_homepage` -- the path only, not the rule. Pure relocation: the tool contract is byte-identical, and the homepage tool still returns all 59 indicators. --- pyproject.toml | 2 +- src/mcpdiffusion/data/__init__.py | 2 +- src/mcpdiffusion/data/insee/__init__.py | 0 src/mcpdiffusion/data/{ => insee}/geography.py | 0 src/mcpdiffusion/data/{ => insee}/indicators.py | 0 src/mcpdiffusion/data/{ => insee}/themes.py | 0 src/mcpdiffusion/data/rmes/__init__.py | 0 .../{rmes_graph_categories.py => rmes/graph_categories.py} | 0 src/mcpdiffusion/models/rmes.py | 2 +- src/mcpdiffusion/services/insee/index_service.py | 4 ++-- src/mcpdiffusion/services/rmes/graph_taxonomy.py | 4 ++-- src/mcpdiffusion/tools/insee/get_homepage_tool.py | 4 ++-- 12 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 src/mcpdiffusion/data/insee/__init__.py rename src/mcpdiffusion/data/{ => insee}/geography.py (100%) rename src/mcpdiffusion/data/{ => insee}/indicators.py (100%) rename src/mcpdiffusion/data/{ => insee}/themes.py (100%) create mode 100644 src/mcpdiffusion/data/rmes/__init__.py rename src/mcpdiffusion/data/{rmes_graph_categories.py => rmes/graph_categories.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index afb6ccd..ea532f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ extend-immutable-calls = [ [tool.ruff.lint.per-file-ignores] # Long literal statistics, pending replacement by a live source. -"src/mcpdiffusion/data/indicators.py" = ["E501"] +"src/mcpdiffusion/data/insee/indicators.py" = ["E501"] [tool.ruff.format] docstring-code-format = true diff --git a/src/mcpdiffusion/data/__init__.py b/src/mcpdiffusion/data/__init__.py index d67f3a1..03286c5 100644 --- a/src/mcpdiffusion/data/__init__.py +++ b/src/mcpdiffusion/data/__init__.py @@ -1 +1 @@ -"""Static reference data (indicators, themes, geography).""" +"""Static reference data, one subpackage per source.""" diff --git a/src/mcpdiffusion/data/insee/__init__.py b/src/mcpdiffusion/data/insee/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/data/geography.py b/src/mcpdiffusion/data/insee/geography.py similarity index 100% rename from src/mcpdiffusion/data/geography.py rename to src/mcpdiffusion/data/insee/geography.py diff --git a/src/mcpdiffusion/data/indicators.py b/src/mcpdiffusion/data/insee/indicators.py similarity index 100% rename from src/mcpdiffusion/data/indicators.py rename to src/mcpdiffusion/data/insee/indicators.py diff --git a/src/mcpdiffusion/data/themes.py b/src/mcpdiffusion/data/insee/themes.py similarity index 100% rename from src/mcpdiffusion/data/themes.py rename to src/mcpdiffusion/data/insee/themes.py diff --git a/src/mcpdiffusion/data/rmes/__init__.py b/src/mcpdiffusion/data/rmes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/data/rmes_graph_categories.py b/src/mcpdiffusion/data/rmes/graph_categories.py similarity index 100% rename from src/mcpdiffusion/data/rmes_graph_categories.py rename to src/mcpdiffusion/data/rmes/graph_categories.py diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index c3f4bff..4664f46 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, Field -from ..data.rmes_graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION +from ..data.rmes.graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION # ---------------------------------------------------------------------------------------------------------------------- # Constants ------------------------------------------------------------------------------------------------------------ diff --git a/src/mcpdiffusion/services/insee/index_service.py b/src/mcpdiffusion/services/insee/index_service.py index 4281120..d64b116 100644 --- a/src/mcpdiffusion/services/insee/index_service.py +++ b/src/mcpdiffusion/services/insee/index_service.py @@ -18,8 +18,8 @@ from elasticsearch.dsl.query import Query from elasticsearch.dsl.response import Response -from ...data.geography import DICT_GEO -from ...data.themes import DICT_THEME_CONJ, KEYS_THEME_NIV1 +from ...data.insee.geography import DICT_GEO +from ...data.insee.themes import DICT_THEME_CONJ, KEYS_THEME_NIV1 from ...models.insee import DocumentHit from ..elasticsearch_failures import elasticsearch_failures_as_tool_errors diff --git a/src/mcpdiffusion/services/rmes/graph_taxonomy.py b/src/mcpdiffusion/services/rmes/graph_taxonomy.py index d194069..8f7c8ec 100644 --- a/src/mcpdiffusion/services/rmes/graph_taxonomy.py +++ b/src/mcpdiffusion/services/rmes/graph_taxonomy.py @@ -1,4 +1,4 @@ -"""Sorting RMES graphs into the families declared in `data/rmes_graph_categories.py`. +"""Sorting RMES graphs into the families declared in `data/rmes/graph_categories.py`. Pure: no client, no I/O, and no state worth a class -- the graph base is passed in rather than read from a module global, so these functions work against any store. @@ -11,7 +11,7 @@ from collections.abc import Callable from dataclasses import dataclass -from ...data.rmes_graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION +from ...data.rmes.graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION from ...models.rmes import CategoryBucket, GraphRow MAX_EXAMPLES_PER_CATEGORY = 5 diff --git a/src/mcpdiffusion/tools/insee/get_homepage_tool.py b/src/mcpdiffusion/tools/insee/get_homepage_tool.py index 15b0eae..006eca6 100644 --- a/src/mcpdiffusion/tools/insee/get_homepage_tool.py +++ b/src/mcpdiffusion/tools/insee/get_homepage_tool.py @@ -2,12 +2,12 @@ from __future__ import annotations -from ...data.indicators import KEY_INDICATORS +from ...data.insee.indicators import KEY_INDICATORS from ...models.insee import KeyIndicatorsOutput, KeyValueIndicator # Business rule: the docstring below calls the figures "latest" and the instructions make this tool the -# preferred FIRST step, but they are frozen literals (see data/indicators.py). Whether the wording softens +# preferred FIRST step, but they are frozen literals (see data/insee/indicators.py). Whether the wording softens # or the data becomes live is the same decision. Left as-is deliberately. def get_insee_homepage() -> KeyIndicatorsOutput: """Retrieve the INSEE home page with the latest key indicators at national level published by From 4a28e5b76f5fa58db286b4548e8913b6daeba5ee Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Mon, 7 Sep 2026 01:19:44 +0200 Subject: [PATCH 26/55] fix: start without Elasticsearch when nothing searches it `ES_HOST` had no default, so the server refused to start without it even when the insee.fr and Melodi tools were both disabled. CLAUDE.md says only rmes works without Elasticsearch; the code disagreed, and a rmes-only deployment was impossible. The host is now genuinely optional, and a settings validator demands it only when a family that searches is enabled, naming the flags that would relax the requirement instead of failing on a bare field name. The lifespan built every client and service unconditionally, so it could not express that. It splits into one module per source, each an async context manager that yields its own fragment of the lifespan context and closes its own clients: lifespan/elasticsearch.py the client insee.fr and Melodi share lifespan/insee.py scraper and services lifespan/melodi.py api client and services lifespan/rmes.py sparql client and service `AsyncExitStack` replaces the unconditional `finally`, so a family that was never built is never torn down, and the two families that need Elasticsearch nest under it -- the dependency is structural rather than a comment, and no optional client is handed to a parameter that requires one. `build_lifespan` went from seventeen parameters to one. It takes the whole `Settings` because it is the composition root, the one place whose job is to read configuration; the per-source builders still take narrow values, which is where `python.md` means it. That resolves the `# Fixme:` asking for exactly this. In rmes-only mode no Elasticsearch client is constructed at all, checked by counting constructor calls rather than reading the code. --- .env.example | 4 +- src/mcpdiffusion/lifespan.py | 126 --------------------- src/mcpdiffusion/lifespan/__init__.py | 84 ++++++++++++++ src/mcpdiffusion/lifespan/elasticsearch.py | 37 ++++++ src/mcpdiffusion/lifespan/insee.py | 49 ++++++++ src/mcpdiffusion/lifespan/melodi.py | 51 +++++++++ src/mcpdiffusion/lifespan/rmes.py | 45 ++++++++ src/mcpdiffusion/server.py | 21 +--- src/mcpdiffusion/settings.py | 16 ++- 9 files changed, 285 insertions(+), 148 deletions(-) delete mode 100644 src/mcpdiffusion/lifespan.py create mode 100644 src/mcpdiffusion/lifespan/__init__.py create mode 100644 src/mcpdiffusion/lifespan/elasticsearch.py create mode 100644 src/mcpdiffusion/lifespan/insee.py create mode 100644 src/mcpdiffusion/lifespan/melodi.py create mode 100644 src/mcpdiffusion/lifespan/rmes.py diff --git a/.env.example b/.env.example index bb52bd3..41e1425 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ -# Every variable this server reads. Only ES_HOST is required; the values shown are the defaults. +# Every variable this server reads. ES_HOST is required unless the insee.fr and Melodi tools are +# both disabled; the values shown are the defaults. # # This file is resolved from the current working directory, not from this package. Docker and # Kubernetes inject real environment variables instead and never read it. @@ -21,6 +22,7 @@ ENABLE_RMES_TOOLS=true # Elasticsearch -------------------------------------------------------------------------------------------------------- # Inside Docker use the service name; on the host use localhost. +# Required only when ENABLE_INSEEFR_TOOLS or ENABLE_MELODI_TOOLS is true. ES_HOST=http://localhost:9200 ES_INDEX_PUBLICATIONS=produit ES_INDEX_MELODI_DATASETS=melodi_datasets diff --git a/src/mcpdiffusion/lifespan.py b/src/mcpdiffusion/lifespan.py deleted file mode 100644 index 22de36c..0000000 --- a/src/mcpdiffusion/lifespan.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Shared clients, created once at startup and torn down on shutdown.""" - -import logging -from collections.abc import AsyncIterator, Callable -from typing import Any - -from elasticsearch import AsyncElasticsearch -from fastmcp.server.lifespan import lifespan -from httpx import AsyncClient, Timeout - -from .services.insee.document_service import InseeDocumentService -from .services.insee.index_service import InseeIndexService -from .services.melodi.api_service import MelodiApiService -from .services.melodi.index_service import MelodiIndexService -from .services.rmes.graph_store_service import RmesGraphStoreService - -logger = logging.getLogger(__name__) - -# insee.fr serves different markup to unknown agents, so the scraper has to look like a browser. -INSEE_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" -# The APIs have no such requirement, so they get an honest identity. -MELODI_USER_AGENT = "McpDiffusion/0.1" -SPARQL_USER_AGENT = "MCP-RMeS/2.0" - -ES_MAX_RETRIES = 2 - - -def build_lifespan( - *, - es_host: str, - es_tls_verify: bool, - es_request_timeout_seconds: int, - insee_base_url: str, - insee_request_timeout_seconds: int, - insee_connect_timeout_seconds: int, - melodi_data_base_url: str, - melodi_request_timeout_seconds: int, - melodi_connect_timeout_seconds: int, - melodi_datasets_index: str, - melodi_columns_index: str, - insee_publications_index: str, - rmes_sparql_endpoint_url: str, - rmes_graph_base_uri: str, - rmes_graph_listing_timeout_seconds: float, - rmes_graph_listing_max_rows: int, - rmes_graph_cache_ttl_seconds: float, -) -> Callable[..., Any]: - - @lifespan - async def app_lifespan(_server: Any) -> AsyncIterator[dict[str, Any]]: - elasticsearch_client = AsyncElasticsearch( - es_host, - verify_certs=es_tls_verify, - request_timeout=es_request_timeout_seconds, - max_retries=ES_MAX_RETRIES, - retry_on_timeout=True, - ) - logger.info("Elasticsearch client initialized for %s", es_host) - - insee_http_client = AsyncClient( - base_url=insee_base_url, - headers={"User-Agent": INSEE_USER_AGENT}, - timeout=Timeout( - insee_request_timeout_seconds, - connect=insee_connect_timeout_seconds, - ), - ) - logger.info("insee.fr client initialized for %s", insee_base_url) - - melodi_http_client = AsyncClient( - base_url=melodi_data_base_url, - headers={"User-Agent": MELODI_USER_AGENT}, - timeout=Timeout( - melodi_request_timeout_seconds, - connect=melodi_connect_timeout_seconds, - ), - ) - logger.info("MELODI client initialized for %s", melodi_data_base_url) - - # RMES passes its own timeout per query, so this client sets none. - sparql_http_client = AsyncClient( - headers={"User-Agent": SPARQL_USER_AGENT}, - ) - logger.info("SPARQL client initialized") - - # Services bind a client to its index or base URL once, so nothing downstream has to - # carry an index name around. They hold no request state, so one instance serves every call. - melodi_index_service = MelodiIndexService( - elasticsearch_client=elasticsearch_client, - datasets_index=melodi_datasets_index, - columns_index=melodi_columns_index, - ) - melodi_api_service = MelodiApiService(http_client=melodi_http_client) - insee_index_service = InseeIndexService( - elasticsearch_client=elasticsearch_client, - publications_index=insee_publications_index, - ) - insee_document_service = InseeDocumentService(http_client=insee_http_client) - rmes_graph_store_service = RmesGraphStoreService( - http_client=sparql_http_client, - sparql_endpoint_url=rmes_sparql_endpoint_url, - graph_base_uri=rmes_graph_base_uri, - graph_listing_timeout_seconds=rmes_graph_listing_timeout_seconds, - graph_listing_max_rows=rmes_graph_listing_max_rows, - graph_cache_ttl_seconds=rmes_graph_cache_ttl_seconds, - ) - - try: - yield { - "elasticsearch_client": elasticsearch_client, - "insee_http_client": insee_http_client, - "melodi_http_client": melodi_http_client, - "sparql_http_client": sparql_http_client, - "melodi_index_service": melodi_index_service, - "melodi_api_service": melodi_api_service, - "insee_index_service": insee_index_service, - "insee_document_service": insee_document_service, - "rmes_graph_store_service": rmes_graph_store_service, - } - finally: - await elasticsearch_client.close() - await insee_http_client.aclose() - await melodi_http_client.aclose() - await sparql_http_client.aclose() - - return app_lifespan diff --git a/src/mcpdiffusion/lifespan/__init__.py b/src/mcpdiffusion/lifespan/__init__.py new file mode 100644 index 0000000..69af0a6 --- /dev/null +++ b/src/mcpdiffusion/lifespan/__init__.py @@ -0,0 +1,84 @@ +"""Startup and shutdown: build only what the enabled tool families need. + +Each source contributes its own context fragment and closes its own clients. The exit stack +unwinds them in reverse, so a family that was never built is never torn down. + +This is the composition root: it is the one place that reads the whole `Settings`, so nothing +below it has to. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import AsyncExitStack +from typing import Any + +from fastmcp import FastMCP +from fastmcp.server.lifespan import Lifespan, lifespan + +from ..settings import Settings +from .elasticsearch import elasticsearch_lifespan +from .insee import insee_lifespan +from .melodi import melodi_lifespan +from .rmes import rmes_lifespan + + +def build_lifespan(settings: Settings) -> Lifespan: + """Return the lifespan FastMCP runs, wired for the families that are enabled.""" + + @lifespan + async def app_lifespan(_server: FastMCP[Any]) -> AsyncIterator[dict[str, Any]]: + # insee.fr and Melodi search the same index, so they share one client. The settings + # validator guarantees a host whenever either is enabled, which is what makes `es_host` + # non-None here and lets the client take a plain `str`. + es_host = settings.es_host if (settings.enable_inseefr_tools or settings.enable_melodi_tools) else None + + async with AsyncExitStack() as stack: + context: dict[str, Any] = {} + + if es_host is not None: + elasticsearch_client = await stack.enter_async_context( + elasticsearch_lifespan( + host=es_host, + tls_verify=settings.es_tls_verify, + request_timeout_seconds=settings.es_request_timeout_seconds, + ) + ) + + if settings.enable_inseefr_tools: + context |= await stack.enter_async_context( + insee_lifespan( + elasticsearch_client=elasticsearch_client, + base_url=settings.insee_base_url, + request_timeout_seconds=settings.insee_request_timeout_seconds, + connect_timeout_seconds=settings.insee_connect_timeout_seconds, + publications_index=settings.es_index_publications, + ) + ) + + if settings.enable_melodi_tools: + context |= await stack.enter_async_context( + melodi_lifespan( + elasticsearch_client=elasticsearch_client, + data_base_url=settings.melodi_data_base_url, + request_timeout_seconds=settings.melodi_request_timeout_seconds, + connect_timeout_seconds=settings.melodi_connect_timeout_seconds, + datasets_index=settings.es_index_melodi_datasets, + columns_index=settings.es_index_melodi_columns, + ) + ) + + if settings.enable_rmes_tools: + context |= await stack.enter_async_context( + rmes_lifespan( + sparql_endpoint_url=settings.rmes_sparql_endpoint_url, + graph_base_uri=settings.rmes_graph_base_uri, + graph_listing_timeout_seconds=settings.rmes_graph_listing_timeout_seconds, + graph_listing_max_rows=settings.rmes_graph_listing_max_rows, + graph_cache_ttl_seconds=settings.rmes_graph_cache_ttl_seconds, + ) + ) + + yield context + + return app_lifespan diff --git a/src/mcpdiffusion/lifespan/elasticsearch.py b/src/mcpdiffusion/lifespan/elasticsearch.py new file mode 100644 index 0000000..fd8b8c6 --- /dev/null +++ b/src/mcpdiffusion/lifespan/elasticsearch.py @@ -0,0 +1,37 @@ +"""The Elasticsearch client, shared by the insee.fr and Melodi searches.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from elasticsearch import AsyncElasticsearch + +logger = logging.getLogger(__name__) + +MAX_RETRIES = 2 + + +@asynccontextmanager +async def elasticsearch_lifespan( + host: str, + tls_verify: bool, + request_timeout_seconds: int, +) -> AsyncIterator[AsyncElasticsearch]: + """Open the shared client and close it on shutdown. + + Construction opens no connection, so a wrong host surfaces on the first search, not here. + """ + client = AsyncElasticsearch( + host, + verify_certs=tls_verify, + request_timeout=request_timeout_seconds, + max_retries=MAX_RETRIES, + retry_on_timeout=True, + ) + logger.info("Elasticsearch client initialized for %s", host) + try: + yield client + finally: + await client.close() diff --git a/src/mcpdiffusion/lifespan/insee.py b/src/mcpdiffusion/lifespan/insee.py new file mode 100644 index 0000000..411ae5f --- /dev/null +++ b/src/mcpdiffusion/lifespan/insee.py @@ -0,0 +1,49 @@ +"""What the insee.fr tools need at runtime.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from elasticsearch import AsyncElasticsearch +from httpx import AsyncClient, Timeout + +from ..services.insee.document_service import InseeDocumentService +from ..services.insee.index_service import InseeIndexService + +logger = logging.getLogger(__name__) + +# insee.fr serves different markup to unknown agents, so the scraper has to look like a browser. +USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" + + +@asynccontextmanager +async def insee_lifespan( + elasticsearch_client: AsyncElasticsearch, + base_url: str, + request_timeout_seconds: int, + connect_timeout_seconds: int, + publications_index: str, +) -> AsyncIterator[dict[str, Any]]: + """Build the insee.fr services and close the scraping client on shutdown.""" + http_client = AsyncClient( + base_url=base_url, + headers={"User-Agent": USER_AGENT}, + timeout=Timeout( + request_timeout_seconds, + connect=connect_timeout_seconds, + ), + ) + logger.info("insee.fr client initialized for %s", base_url) + try: + yield { + "insee_index_service": InseeIndexService( + elasticsearch_client=elasticsearch_client, + publications_index=publications_index, + ), + "insee_document_service": InseeDocumentService(http_client=http_client), + } + finally: + await http_client.aclose() diff --git a/src/mcpdiffusion/lifespan/melodi.py b/src/mcpdiffusion/lifespan/melodi.py new file mode 100644 index 0000000..fb2765f --- /dev/null +++ b/src/mcpdiffusion/lifespan/melodi.py @@ -0,0 +1,51 @@ +"""What the Melodi tools need at runtime.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from elasticsearch import AsyncElasticsearch +from httpx import AsyncClient, Timeout + +from ..services.melodi.api_service import MelodiApiService +from ..services.melodi.index_service import MelodiIndexService + +logger = logging.getLogger(__name__) + +# The API has no browser requirement, so it gets an honest identity. +USER_AGENT = "McpDiffusion/0.1" + + +@asynccontextmanager +async def melodi_lifespan( + elasticsearch_client: AsyncElasticsearch, + data_base_url: str, + request_timeout_seconds: int, + connect_timeout_seconds: int, + datasets_index: str, + columns_index: str, +) -> AsyncIterator[dict[str, Any]]: + """Build the Melodi services and close the API client on shutdown.""" + http_client = AsyncClient( + base_url=data_base_url, + headers={"User-Agent": USER_AGENT}, + timeout=Timeout( + request_timeout_seconds, + connect=connect_timeout_seconds, + ), + ) + logger.info("MELODI client initialized for %s", data_base_url) + try: + yield { + "melodi_index_service": MelodiIndexService( + elasticsearch_client=elasticsearch_client, + datasets_index=datasets_index, + columns_index=columns_index, + ), + "melodi_api_service": MelodiApiService(http_client=http_client), + } + finally: + await http_client.aclose() diff --git a/src/mcpdiffusion/lifespan/rmes.py b/src/mcpdiffusion/lifespan/rmes.py new file mode 100644 index 0000000..f8371da --- /dev/null +++ b/src/mcpdiffusion/lifespan/rmes.py @@ -0,0 +1,45 @@ +"""What the RMES tools need at runtime.""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from httpx import AsyncClient + +from ..services.rmes.graph_store_service import RmesGraphStoreService + +logger = logging.getLogger(__name__) + +USER_AGENT = "MCP-RMeS/2.0" + + +@asynccontextmanager +async def rmes_lifespan( + sparql_endpoint_url: str, + graph_base_uri: str, + graph_listing_timeout_seconds: float, + graph_listing_max_rows: int, + graph_cache_ttl_seconds: float, +) -> AsyncIterator[dict[str, Any]]: + """Build the RMES service and close its client on shutdown. + + RMES passes its own timeout per query, so this client sets none. + """ + http_client = AsyncClient(headers={"User-Agent": USER_AGENT}) + logger.info("SPARQL client initialized") + try: + yield { + "rmes_graph_store_service": RmesGraphStoreService( + http_client=http_client, + sparql_endpoint_url=sparql_endpoint_url, + graph_base_uri=graph_base_uri, + graph_listing_timeout_seconds=graph_listing_timeout_seconds, + graph_listing_max_rows=graph_listing_max_rows, + graph_cache_ttl_seconds=graph_cache_ttl_seconds, + ), + } + finally: + await http_client.aclose() diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 6d64d1d..fff8134 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -32,26 +32,7 @@ # Only AppToolError messages reach the caller; anything else is a bug and is replaced # by a generic message. mask_error_details=True, - # Fixme: it takes too many arguments, we can maybe pass settings here directly - lifespan=build_lifespan( - es_host=settings.es_host, - es_tls_verify=settings.es_tls_verify, - es_request_timeout_seconds=settings.es_request_timeout_seconds, - insee_base_url=settings.insee_base_url, - insee_request_timeout_seconds=settings.insee_request_timeout_seconds, - insee_connect_timeout_seconds=settings.insee_connect_timeout_seconds, - melodi_data_base_url=settings.melodi_data_base_url, - melodi_request_timeout_seconds=settings.melodi_request_timeout_seconds, - melodi_connect_timeout_seconds=settings.melodi_connect_timeout_seconds, - melodi_datasets_index=settings.es_index_melodi_datasets, - melodi_columns_index=settings.es_index_melodi_columns, - insee_publications_index=settings.es_index_publications, - rmes_sparql_endpoint_url=settings.rmes_sparql_endpoint_url, - rmes_graph_base_uri=settings.rmes_graph_base_uri, - rmes_graph_listing_timeout_seconds=settings.rmes_graph_listing_timeout_seconds, - rmes_graph_listing_max_rows=settings.rmes_graph_listing_max_rows, - rmes_graph_cache_ttl_seconds=settings.rmes_graph_cache_ttl_seconds, - ), + lifespan=build_lifespan(settings), ) register_tools(mcp, settings) diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 641b798..93c5f44 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -1,5 +1,6 @@ """Every value this server can be configured with.""" +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -19,7 +20,9 @@ class Settings(BaseSettings): enable_rmes_tools: bool = True # Elasticsearch ---------------------------------------------------------------------------------------------------- - es_host: str + # Only the insee.fr and Melodi tools search Elasticsearch; rmes runs without it, so the + # host is genuinely absent rather than empty when they are disabled. + es_host: str | None = None es_index_publications: str = "produit" es_index_melodi_datasets: str = "melodi_datasets" es_index_melodi_columns: str = "melodi_columns" @@ -51,6 +54,17 @@ class Settings(BaseSettings): # Logging ---------------------------------------------------------------------------------------------------------- log_level: str = "INFO" + @model_validator(mode="after") + def require_elasticsearch_when_it_is_searched(self) -> "Settings": + """Fail at startup rather than on the first search that needs a host.""" + if self.es_host is None and (self.enable_inseefr_tools or self.enable_melodi_tools): + raise ValueError( + "ES_HOST is required because the insee.fr or Melodi tools are enabled. " + "Set it, or disable those families with ENABLE_INSEEFR_TOOLS=false and " + "ENABLE_MELODI_TOOLS=false." + ) + return self + model_config = SettingsConfigDict( env_file=".env", extra="ignore", From bf8bca37fc6e9c89ffbd990e2bba84cd0f470948 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Mon, 7 Sep 2026 01:20:00 +0200 Subject: [PATCH 27/55] refactor(tools): give each source its own dependency module `dependencies.py` held the providers for all three sources, so adding a source meant editing a file every other source already depends on. It becomes a package with one module per source, matching how `data/`, `services/`, `tools/` and `lifespan/` already group: dependencies/insee.py dependencies/melodi.py dependencies/rmes.py Nothing is re-exported from the package `__init__`: a tool imports from its own source's module, so a new source adds a file rather than editing a shared one. --- src/mcpdiffusion/dependencies.py | 50 ------------------- src/mcpdiffusion/dependencies/__init__.py | 6 +++ src/mcpdiffusion/dependencies/insee.py | 20 ++++++++ src/mcpdiffusion/dependencies/melodi.py | 17 +++++++ src/mcpdiffusion/dependencies/rmes.py | 11 ++++ .../tools/insee/get_document_tool.py | 2 +- .../tools/insee/search_chiffrecle_tool.py | 2 +- .../tools/insee/search_conjoncture_tool.py | 2 +- .../tools/insee/search_documents_tool.py | 2 +- .../tools/melodi/get_observations_tool.py | 2 +- .../tools/melodi/search_datasets_tool.py | 2 +- .../tools/melodi/search_modalities_tool.py | 2 +- .../tools/rmes/describe_resource_tool.py | 2 +- .../tools/rmes/run_sparql_tool.py | 2 +- .../tools/rmes/search_graphs_tool.py | 2 +- 15 files changed, 64 insertions(+), 60 deletions(-) delete mode 100644 src/mcpdiffusion/dependencies.py create mode 100644 src/mcpdiffusion/dependencies/__init__.py create mode 100644 src/mcpdiffusion/dependencies/insee.py create mode 100644 src/mcpdiffusion/dependencies/melodi.py create mode 100644 src/mcpdiffusion/dependencies/rmes.py diff --git a/src/mcpdiffusion/dependencies.py b/src/mcpdiffusion/dependencies.py deleted file mode 100644 index d2d5530..0000000 --- a/src/mcpdiffusion/dependencies.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Everything a tool can ask FastMCP to inject. - -A tool declares what it needs in its signature; FastMCP resolves it per request and hides the -parameter from the tool schema, so the LLM never sees it. - -Every tool takes its service through `Depends(...)`, which reads the context itself and appears -as a parameter default. -""" - -from fastmcp import Context -from fastmcp.dependencies import CurrentContext - -from .services.insee.document_service import InseeDocumentService -from .services.insee.index_service import InseeIndexService -from .services.melodi.api_service import MelodiApiService -from .services.melodi.index_service import MelodiIndexService -from .services.rmes.graph_store_service import RmesGraphStoreService - -# Fixme: these dependency functions do not provide proper typing which is a pity -- the lifespan -# context is an untyped mapping, so every return annotation below is asserted, never checked. - - -# ---------------------------------------------------------------------------------------------------------------------- -# Injected services ---------------------------------------------------------------------------------------------------- -# ---------------------------------------------------------------------------------------------------------------------- - - -def get_insee_index_service(ctx: Context = CurrentContext()) -> InseeIndexService: - """Return the insee.fr Elasticsearch service built at startup.""" - return ctx.lifespan_context["insee_index_service"] - - -def get_insee_document_service(ctx: Context = CurrentContext()) -> InseeDocumentService: - """Return the insee.fr document scraping service built at startup.""" - return ctx.lifespan_context["insee_document_service"] - - -def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: - """Return the Melodi Elasticsearch service built at startup.""" - return ctx.lifespan_context["melodi_index_service"] - - -def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: - """Return the Melodi REST API service built at startup.""" - return ctx.lifespan_context["melodi_api_service"] - - -def get_rmes_graph_store_service(ctx: Context = CurrentContext()) -> RmesGraphStoreService: - """Return the RMES graph store service built at startup.""" - return ctx.lifespan_context["rmes_graph_store_service"] diff --git a/src/mcpdiffusion/dependencies/__init__.py b/src/mcpdiffusion/dependencies/__init__.py new file mode 100644 index 0000000..72401ca --- /dev/null +++ b/src/mcpdiffusion/dependencies/__init__.py @@ -0,0 +1,6 @@ +"""What a tool can ask FastMCP to inject, one module per source. + +A tool declares what it needs in its signature; FastMCP resolves it per request and hides the +parameter from the tool schema, so the LLM never sees it. Nothing is re-exported here: a tool +imports from its own source's module, so adding a source adds a file rather than editing one. +""" diff --git a/src/mcpdiffusion/dependencies/insee.py b/src/mcpdiffusion/dependencies/insee.py new file mode 100644 index 0000000..dbd2cde --- /dev/null +++ b/src/mcpdiffusion/dependencies/insee.py @@ -0,0 +1,20 @@ +"""The insee.fr services a tool can be handed.""" + +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from ..services.insee.document_service import InseeDocumentService +from ..services.insee.index_service import InseeIndexService + +# Fixme: these dependency functions do not provide proper typing which is a pity -- the lifespan +# context is an untyped mapping, so every return annotation below is asserted, never checked. + + +def get_insee_index_service(ctx: Context = CurrentContext()) -> InseeIndexService: + """Return the insee.fr Elasticsearch service built at startup.""" + return ctx.lifespan_context["insee_index_service"] + + +def get_insee_document_service(ctx: Context = CurrentContext()) -> InseeDocumentService: + """Return the insee.fr document scraping service built at startup.""" + return ctx.lifespan_context["insee_document_service"] diff --git a/src/mcpdiffusion/dependencies/melodi.py b/src/mcpdiffusion/dependencies/melodi.py new file mode 100644 index 0000000..9c8bc40 --- /dev/null +++ b/src/mcpdiffusion/dependencies/melodi.py @@ -0,0 +1,17 @@ +"""The Melodi services a tool can be handed.""" + +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from ..services.melodi.api_service import MelodiApiService +from ..services.melodi.index_service import MelodiIndexService + + +def get_melodi_index_service(ctx: Context = CurrentContext()) -> MelodiIndexService: + """Return the Melodi Elasticsearch service built at startup.""" + return ctx.lifespan_context["melodi_index_service"] + + +def get_melodi_api_service(ctx: Context = CurrentContext()) -> MelodiApiService: + """Return the Melodi REST API service built at startup.""" + return ctx.lifespan_context["melodi_api_service"] diff --git a/src/mcpdiffusion/dependencies/rmes.py b/src/mcpdiffusion/dependencies/rmes.py new file mode 100644 index 0000000..16e6de6 --- /dev/null +++ b/src/mcpdiffusion/dependencies/rmes.py @@ -0,0 +1,11 @@ +"""The RMES service a tool can be handed.""" + +from fastmcp import Context +from fastmcp.dependencies import CurrentContext + +from ..services.rmes.graph_store_service import RmesGraphStoreService + + +def get_rmes_graph_store_service(ctx: Context = CurrentContext()) -> RmesGraphStoreService: + """Return the RMES graph store service built at startup.""" + return ctx.lifespan_context["rmes_graph_store_service"] diff --git a/src/mcpdiffusion/tools/insee/get_document_tool.py b/src/mcpdiffusion/tools/insee/get_document_tool.py index 882fb39..88a5e3b 100644 --- a/src/mcpdiffusion/tools/insee/get_document_tool.py +++ b/src/mcpdiffusion/tools/insee/get_document_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_insee_document_service +from ...dependencies.insee import get_insee_document_service from ...models.insee import ( DocumentContentOutput, DocumentUrls, diff --git a/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py b/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py index 043fd0e..c76df5b 100644 --- a/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py +++ b/src/mcpdiffusion/tools/insee/search_chiffrecle_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_insee_index_service +from ...dependencies.insee import get_insee_index_service from ...models.insee import ( DEFAULT_RESULT_COUNT, DocumentSearchOutput, diff --git a/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py b/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py index d552dee..7f270d9 100644 --- a/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py +++ b/src/mcpdiffusion/tools/insee/search_conjoncture_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_insee_index_service +from ...dependencies.insee import get_insee_index_service from ...models.insee import ( DEFAULT_RESULT_COUNT, ConjonctureQuery, diff --git a/src/mcpdiffusion/tools/insee/search_documents_tool.py b/src/mcpdiffusion/tools/insee/search_documents_tool.py index c483394..d079c47 100644 --- a/src/mcpdiffusion/tools/insee/search_documents_tool.py +++ b/src/mcpdiffusion/tools/insee/search_documents_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_insee_index_service +from ...dependencies.insee import get_insee_index_service from ...models.insee import ( DEFAULT_RESULT_COUNT, DocumentSearchOutput, diff --git a/src/mcpdiffusion/tools/melodi/get_observations_tool.py b/src/mcpdiffusion/tools/melodi/get_observations_tool.py index 4d8f56e..b3e02e5 100644 --- a/src/mcpdiffusion/tools/melodi/get_observations_tool.py +++ b/src/mcpdiffusion/tools/melodi/get_observations_tool.py @@ -6,7 +6,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_melodi_api_service +from ...dependencies.melodi import get_melodi_api_service from ...models.melodi import ( ColumnFilters, DatasetId, diff --git a/src/mcpdiffusion/tools/melodi/search_datasets_tool.py b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py index 3602945..f980dd1 100644 --- a/src/mcpdiffusion/tools/melodi/search_datasets_tool.py +++ b/src/mcpdiffusion/tools/melodi/search_datasets_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_melodi_index_service +from ...dependencies.melodi import get_melodi_index_service from ...models.melodi import ( DatasetQuery, DatasetsOutput, diff --git a/src/mcpdiffusion/tools/melodi/search_modalities_tool.py b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py index 8e088de..f79c4bf 100644 --- a/src/mcpdiffusion/tools/melodi/search_modalities_tool.py +++ b/src/mcpdiffusion/tools/melodi/search_modalities_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_melodi_index_service +from ...dependencies.melodi import get_melodi_index_service from ...models.melodi import ( ColumnIds, DatasetId, diff --git a/src/mcpdiffusion/tools/rmes/describe_resource_tool.py b/src/mcpdiffusion/tools/rmes/describe_resource_tool.py index 7f4872d..98df7c8 100644 --- a/src/mcpdiffusion/tools/rmes/describe_resource_tool.py +++ b/src/mcpdiffusion/tools/rmes/describe_resource_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_rmes_graph_store_service +from ...dependencies.rmes import get_rmes_graph_store_service from ...models.rmes import GraphUri, ResourceOutput, ResourceUri from ...services.rmes.graph_store_service import RmesGraphStoreService diff --git a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py index 91ef82d..8fd0070 100644 --- a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py +++ b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_rmes_graph_store_service +from ...dependencies.rmes import get_rmes_graph_store_service from ...errors import AppToolError from ...models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, diff --git a/src/mcpdiffusion/tools/rmes/search_graphs_tool.py b/src/mcpdiffusion/tools/rmes/search_graphs_tool.py index 75bb274..b100b9f 100644 --- a/src/mcpdiffusion/tools/rmes/search_graphs_tool.py +++ b/src/mcpdiffusion/tools/rmes/search_graphs_tool.py @@ -4,7 +4,7 @@ from fastmcp.dependencies import Depends -from ...dependencies import get_rmes_graph_store_service +from ...dependencies.rmes import get_rmes_graph_store_service from ...models.rmes import ( ExpandGraphs, GraphCategory, From 68d149c49447c9ee3d745dd21dc57b5d70247279 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 11:49:58 +0200 Subject: [PATCH 28/55] fix: rate limit each caller instead of all callers together The deployment passed no TRUSTED_PROXY_HOSTS, so uvicorn kept its default of trusting only 127.0.0.1. Behind the Ingress the peer is a cluster IP, never that, so uvicorn ignored the forwarded caller address and reported the Ingress as the client of every request. The rate limiter keys on that address. Every caller therefore shared one bucket: a hundred requests a minute for the whole world combined, and one busy client could lock out the rest, while the code read as though the limit were per caller. Trusting any peer is safe only while nothing but the Ingress can reach the pod. A NetworkPolicy restricting traffic to the Ingress namespace is what makes that true; without one, a pod inside the cluster could reach the server directly and claim any caller address. --- k8s/3_mcp_deploy.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/k8s/3_mcp_deploy.yaml b/k8s/3_mcp_deploy.yaml index 6f52813..50f41e1 100644 --- a/k8s/3_mcp_deploy.yaml +++ b/k8s/3_mcp_deploy.yaml @@ -45,6 +45,12 @@ spec: env: - name: ES_HOST value: "http://elasticsearch-service:9200" + # The Ingress controller reaches the pod from a cluster IP that changes, so the + # forwarded caller address is believed from any peer. Uvicorn otherwise trusts only + # 127.0.0.1, sees every request as coming from the Ingress, and rate limits all + # callers as one. Keep the pod reachable only through the Ingress. + - name: TRUSTED_PROXY_HOSTS + value: '["*"]' # Optional resources block – adjust as needed resources: limits: From 3d212791b9820c4b1400b93e228e6e5eebfd27fe Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 11:50:49 +0200 Subject: [PATCH 29/55] fix(server): reject requests not addressed to this server's hostname The Host and Origin guard was configured in a way that enforced the opposite of what it looked like. `host_origin_protection="auto"` leaves the decision to a heuristic that defers to existing handling for reverse-proxy deployments, while `allowed_hosts` was left at its `["*"]` default, which accepts any Host. The check ran and approved everyone: a request claiming `Host: evil.example.com` reached the session manager untouched. Meanwhile `allowed_origins` was never passed and had no setting, so the Origin half ran on a default nobody chose and could not adjust. A browser client on another origin was refused with no way to permit it. So the rule that was written did nothing and the rule that was not written did the refusing. Protection is now enforced rather than inferred, the Ingress hostname is declared, and the browser origins are a setting: empty today, because no browser client calls this server. A request claiming another hostname is now answered 421, and a cross-origin browser request 403. Local development is unaffected: with nothing set, `ALLOWED_HOSTS` stays `["*"]`. --- .env.example | 3 +++ k8s/3_mcp_deploy.yaml | 7 +++++++ src/mcpdiffusion/server.py | 5 ++++- src/mcpdiffusion/settings.py | 6 +++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 41e1425..da1d3a1 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,10 @@ MCP_HOST=0.0.0.0 MCP_PORT=8000 # JSON list of hosts this server answers to. # "*" accepts any host and is unsafe once the server is publicly reachable. +# The hostnames clients use to reach the server. "*" disables the check; set the real one. ALLOWED_HOSTS=["*"] +# Browser origins allowed to call the server. Empty rejects cross-origin browser requests. +ALLOWED_ORIGINS=[] # JSON list of peers whose X-Forwarded-For header is believed. Set this to your reverse proxy. # Accepts addresses, CIDR networks and literals. Widening it lets any caller forge their own # address, which defeats per-client rate limiting. diff --git a/k8s/3_mcp_deploy.yaml b/k8s/3_mcp_deploy.yaml index 50f41e1..4ef6fb7 100644 --- a/k8s/3_mcp_deploy.yaml +++ b/k8s/3_mcp_deploy.yaml @@ -45,6 +45,13 @@ spec: env: - name: ES_HOST value: "http://elasticsearch-service:9200" + # The hostname the Ingress publishes. Without it ALLOWED_HOSTS defaults to "*", + # which accepts any Host header and makes the check pointless. + - name: ALLOWED_HOSTS + value: '["mcpdiffusion.lab.sspcloud.fr"]' + # No browser client calls this server today, so no cross-origin site is allowed. + - name: ALLOWED_ORIGINS + value: '[]' # The Ingress controller reaches the pod from a cluster IP that changes, so the # forwarded caller address is believed from any peer. Uvicorn otherwise trusts only # 127.0.0.1, sees every request as coming from the Ingress, and rate limits all diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index fff8134..237484a 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -67,9 +67,12 @@ if settings.allowed_hosts == ["*"]: logger.warning("allowed_hosts is ['*']. Set ALLOWED_HOSTS before exposing the server publicly.") +# Enforced rather than "auto": this server is published under a real hostname, so it should +# check the one it was reached by instead of leaving the decision to a heuristic. app = mcp.http_app( - host_origin_protection="auto", + host_origin_protection=True, allowed_hosts=settings.allowed_hosts, + allowed_origins=settings.allowed_origins, ) if __name__ == "__main__": diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 93c5f44..73fae73 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -8,8 +8,12 @@ class Settings(BaseSettings): # HTTP server ------------------------------------------------------------------------------------------------------ mcp_host: str = "0.0.0.0" mcp_port: int = 8000 - # JSON list. "*" accepts any host and is unsafe once the server is publicly reachable. + # JSON list of the hostnames clients use to reach this server, checked against the Host + # header. "*" accepts any host, which disables the check. allowed_hosts: list[str] = ["*"] + # JSON list of browser origins allowed to call the server. Empty rejects every cross-origin + # browser request, which is right until a browser-based client needs in. + allowed_origins: list[str] = [] # Peers whose X-Forwarded-For header is believed; anything else keeps its real socket address. # Accepts addresses, CIDR networks and literals. Widening this lets callers forge their own address. trusted_proxy_hosts: list[str] = ["127.0.0.1"] From 4ae8b78729a97eb57a4506481ca1d19e36bd3de2 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 12:10:03 +0200 Subject: [PATCH 30/55] fix: enforce the error vocabulary instead of describing it `ErrorCode` was a `Literal`, which Python never checks. A typo was accepted in silence and reached the caller as an invented code: `AppToolError("TOTALLY_MADE_UP", ...)` produced `[TOTALLY_MADE_UP] oops` with nothing to catch it. It is a `StrEnum` now, and all twenty-four raises pass a member. A typo is an `AttributeError` at the line that wrote it, and an editor marks it before the code runs. No runtime coercion is added: validating the code inside `__init__` would raise while an `except` block was already handling a failure, replacing the real error with a confusing one. Referencing a member fails earlier and more clearly. `UNKNOWN` becomes `INTERNAL_ERROR`. Every other code says where the fault is -- the caller's input, a backend, the network -- while `UNKNOWN` described our ignorance, which reads as an admission rather than information. The caller learns something useful from `INTERNAL_ERROR`: the fault is in this server, so rephrasing will not help. That code had never been used: the one place needing it wrote `"[UNKNOWN] ..."` as a literal, because it reports a per-URL failure in the result rather than raising. It now builds the prefix from the same vocabulary, so the two cannot drift apart, and that branch is exercised by forcing an unexpected failure rather than assumed to work. The module is `error.py`, matching the rule file that governs it. Message text is unchanged throughout, which matters because the message is the error contract: every insee and Melodi failure message, both live rmes error paths, and the tool contract are identical. --- src/mcpdiffusion/{errors.py => error.py} | 28 ++++++++++++------- .../services/elasticsearch_failures.py | 12 ++++---- .../services/insee/document_service.py | 16 ++++++----- .../services/melodi/api_service.py | 16 +++++------ .../services/rmes/graph_store_service.py | 14 +++++----- .../tools/rmes/run_sparql_tool.py | 4 +-- 6 files changed, 50 insertions(+), 40 deletions(-) rename src/mcpdiffusion/{errors.py => error.py} (55%) diff --git a/src/mcpdiffusion/errors.py b/src/mcpdiffusion/error.py similarity index 55% rename from src/mcpdiffusion/errors.py rename to src/mcpdiffusion/error.py index 1eadb86..ef9dcbd 100644 --- a/src/mcpdiffusion/errors.py +++ b/src/mcpdiffusion/error.py @@ -1,18 +1,26 @@ """The single error type tools and services raise.""" -from typing import Literal +from enum import StrEnum from fastmcp.exceptions import ToolError -ErrorCode = Literal[ - "INVALID_INPUT", - "BACKEND_UNAVAILABLE", - "UPSTREAM_ERROR", - "PARSE_ERROR", - "INVALID_QUERY", - "NOT_FOUND", - "UNKNOWN", -] + +class ErrorCode(StrEnum): + """The closed vocabulary of failures a caller can be told about. + + A member rather than a `Literal`: a literal is only a promise to a type checker, so a typo + reached the caller as an invented code. Referencing a member fails at the typo instead. + """ + + INVALID_INPUT = "INVALID_INPUT" + BACKEND_UNAVAILABLE = "BACKEND_UNAVAILABLE" + UPSTREAM_ERROR = "UPSTREAM_ERROR" + PARSE_ERROR = "PARSE_ERROR" + INVALID_QUERY = "INVALID_QUERY" + NOT_FOUND = "NOT_FOUND" + # A fault in this server rather than in the caller's input or a backend: a bug, logged + # in full server-side and reported to the caller only as ours to fix. + INTERNAL_ERROR = "INTERNAL_ERROR" class AppToolError(ToolError): diff --git a/src/mcpdiffusion/services/elasticsearch_failures.py b/src/mcpdiffusion/services/elasticsearch_failures.py index 14fc440..53b16f8 100644 --- a/src/mcpdiffusion/services/elasticsearch_failures.py +++ b/src/mcpdiffusion/services/elasticsearch_failures.py @@ -16,7 +16,7 @@ from elasticsearch import ApiError, TransportError -from ..errors import AppToolError +from ..error import AppToolError, ErrorCode @asynccontextmanager @@ -34,7 +34,7 @@ async def elasticsearch_failures_as_tool_errors(backend_label: str) -> AsyncIter # the host and port, and that must not leave the process. The full cause, host included, # is in the server log via ErrorHandlingMiddleware. raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"{backend_label} search backend unreachable ({type(exc).__name__}). Verify ES_HOST and try again.", retryable=True, ) @@ -42,26 +42,26 @@ async def elasticsearch_failures_as_tool_errors(backend_label: str) -> AsyncIter status = exc.status_code if status == HTTPStatus.NOT_FOUND: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"The {backend_label} index is missing from Elasticsearch ({exc.error}). " "The index is not loaded on the server, so no query against it can succeed. " "Rephrasing will not help -- report this instead of retrying.", ) if status == HTTPStatus.BAD_REQUEST: raise AppToolError( - "INVALID_QUERY", + ErrorCode.INVALID_QUERY, f"Elasticsearch rejected the {backend_label} search as malformed " f"({exc.error}). This is a defect in the server's query, not in the arguments " "you passed. Report it instead of retrying.", ) if status in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN): raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Elasticsearch refused the {backend_label} search ({exc.error}). The server's " "credentials are missing or insufficient. Report it instead of retrying.", ) raise AppToolError( - "UPSTREAM_ERROR", + ErrorCode.UPSTREAM_ERROR, f"Elasticsearch returned HTTP {status} for the {backend_label} search ({exc.error}).", retryable=status >= HTTPStatus.INTERNAL_SERVER_ERROR, ) diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index 87a44ab..344fb5f 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -11,7 +11,7 @@ from trafilatura import extract from trafilatura.settings import Extractor -from ...errors import AppToolError +from ...error import AppToolError, ErrorCode from ...models.insee import DocumentResult, TableOfContents logger = logging.getLogger(__name__) @@ -155,24 +155,24 @@ async def fetch_html(self, url: str) -> str: return response.text except httpx.TimeoutException as exc: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"insee.fr timed out fetching {target}: {exc}", retryable=True, ) except httpx.HTTPStatusError as exc: if exc.response.status_code == httpx.codes.NOT_FOUND: raise AppToolError( - "NOT_FOUND", + ErrorCode.NOT_FOUND, f"INSEE document not found at {target} (HTTP 404). Verify the URL with `search_insee_documents`.", ) raise AppToolError( - "UPSTREAM_ERROR", + ErrorCode.UPSTREAM_ERROR, f"insee.fr returned HTTP {exc.response.status_code} for {target}.", retryable=exc.response.is_server_error, ) except httpx.HTTPError as exc: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Network error fetching {target}: {exc}", retryable=True, ) @@ -186,7 +186,7 @@ async def fetch_documents( """Fetch and render each URL, reporting per-URL failures rather than aborting the batch.""" if not document_urls: raise AppToolError( - "INVALID_INPUT", + ErrorCode.INVALID_INPUT, "document_urls must contain at least one URL. Use `search_insee_documents` to find URLs first.", ) @@ -231,7 +231,9 @@ async def fetch_documents( results.append( build_failed_document( url=url, - message="[UNKNOWN] Could not fetch this document.", + # Not raised, so the prefix an AppToolError would add is built here, + # from the same vocabulary rather than a hand-written literal. + message=f"[{ErrorCode.INTERNAL_ERROR}] Could not fetch this document.", ) ) diff --git a/src/mcpdiffusion/services/melodi/api_service.py b/src/mcpdiffusion/services/melodi/api_service.py index 1d62182..f748580 100644 --- a/src/mcpdiffusion/services/melodi/api_service.py +++ b/src/mcpdiffusion/services/melodi/api_service.py @@ -6,7 +6,7 @@ import httpx -from ...errors import AppToolError +from ...error import AppToolError, ErrorCode class MelodiApiService: @@ -31,7 +31,7 @@ async def fetch_observations( response.raise_for_status() except httpx.TimeoutException as exc: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Melodi API timed out calling {url}: {exc}. Try again or narrow the query.", retryable=True, ) @@ -43,7 +43,7 @@ async def fetch_observations( # on it would break the moment it is reworded -- so name every remedy instead. if status == httpx.codes.BAD_REQUEST: raise AppToolError( - "INVALID_INPUT", + ErrorCode.INVALID_INPUT, f'Melodi API rejected the query (HTTP 400). Upstream detail: "{body_excerpt}" ' f"Columns/values passed: {column_filters}. " "Confirm the dataset_id with `search_melodi_datasets`, and the column ids " @@ -51,18 +51,18 @@ async def fetch_observations( ) if status == httpx.codes.NOT_FOUND: raise AppToolError( - "NOT_FOUND", + ErrorCode.NOT_FOUND, f"Melodi dataset {dataset_id!r} not found (HTTP 404). " "Check the dataset_id with `search_melodi_datasets`.", ) raise AppToolError( - "UPSTREAM_ERROR", + ErrorCode.UPSTREAM_ERROR, f'Melodi API returned HTTP {status}: "{body_excerpt}"', retryable=exc.response.is_server_error, ) except httpx.HTTPError as exc: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Could not reach Melodi API at {url}: {exc}", retryable=True, ) @@ -71,14 +71,14 @@ async def fetch_observations( payload = response.json() except ValueError as exc: raise AppToolError( - "PARSE_ERROR", + ErrorCode.PARSE_ERROR, f"Melodi API returned non-JSON response: {exc}", ) observations = payload.get("observations") if isinstance(payload, dict) else None if not isinstance(observations, list): raise AppToolError( - "PARSE_ERROR", + ErrorCode.PARSE_ERROR, "Melodi API response did not contain an 'observations' list.", ) return observations diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py index baa271e..365f390 100644 --- a/src/mcpdiffusion/services/rmes/graph_store_service.py +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -14,7 +14,7 @@ import httpx -from ...errors import AppToolError +from ...error import AppToolError, ErrorCode from ...models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, MAX_QUERY_TIMEOUT_SECONDS, @@ -163,7 +163,7 @@ async def execute( query_form = detect_query_form(query) if query_form == UNKNOWN_FORM: raise AppToolError( - "INVALID_QUERY", + ErrorCode.INVALID_QUERY, "Impossible de detecter SELECT / ASK / CONSTRUCT / DESCRIBE dans la requete. " "Verifie la syntaxe SPARQL (pas GraphQL).", ) @@ -181,7 +181,7 @@ async def execute( response.raise_for_status() except httpx.TimeoutException: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Le endpoint RMES n'a pas repondu en moins de {timeout_seconds}s. " "Restreins la requete (ajoute une clause GRAPH precise, reduis le LIMIT, " "evite les scans sans filtre sur tous les graphes).", @@ -192,17 +192,17 @@ async def execute( body = exc.response.text[:2000] if status == httpx.codes.BAD_REQUEST: raise AppToolError( - "INVALID_QUERY", + ErrorCode.INVALID_QUERY, f"Le endpoint RMES a rejete la requete (erreur de syntaxe SPARQL probable) : {body}", ) raise AppToolError( - "UPSTREAM_ERROR", + ErrorCode.UPSTREAM_ERROR, f"Le endpoint RMES a repondu {status} : {body}", retryable=exc.response.is_server_error, ) except httpx.RequestError as exc: raise AppToolError( - "BACKEND_UNAVAILABLE", + ErrorCode.BACKEND_UNAVAILABLE, f"Impossible de contacter l'endpoint RMES ({type(exc).__name__}).", retryable=True, ) @@ -217,7 +217,7 @@ async def execute( payload = response.json() except ValueError as exc: raise AppToolError( - "PARSE_ERROR", + ErrorCode.PARSE_ERROR, f"Le endpoint RMES a renvoye une reponse non-JSON : {exc}", ) diff --git a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py index 8fd0070..7d1c81a 100644 --- a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py +++ b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py @@ -5,7 +5,7 @@ from fastmcp.dependencies import Depends from ...dependencies.rmes import get_rmes_graph_store_service -from ...errors import AppToolError +from ...error import AppToolError, ErrorCode from ...models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, DEFAULT_ROW_LIMIT, @@ -65,7 +65,7 @@ async def run_rmes_sparql( """ if not sparql_query or not sparql_query.strip(): raise AppToolError( - "INVALID_INPUT", + ErrorCode.INVALID_INPUT, "La requete SPARQL est vide. Fournis une requete SELECT, ASK, CONSTRUCT ou DESCRIBE.", ) From 26bdf59220cd4344a1816c51fa15e578666d6d36 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 12:27:36 +0200 Subject: [PATCH 31/55] refactor: enforce keyword arguments at the call site, not in every signature The bare * marker was in only three signatures, and every caller already named its arguments, so it duplicated a convention the layout rules already carry. Ruff FBT003 replaces the one case it genuinely guarded: a bare boolean passed positionally to AppToolError. FBT001 and FBT002 stay off, because a tool's boolean parameter is a named field of its JSON schema. --- .claude/rules/python.md | 5 +++-- pyproject.toml | 3 +++ src/mcpdiffusion/error.py | 1 - src/mcpdiffusion/instructions.py | 1 - src/mcpdiffusion/services/feedback.py | 1 - 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.claude/rules/python.md b/.claude/rules/python.md index aca17da..af5ae77 100644 --- a/.claude/rules/python.md +++ b/.claude/rules/python.md @@ -8,8 +8,9 @@ Default to pure: same arguments in, same value out. - Never read a global inside a function — no `get_settings()`, no module-level client or cache. - Never mutate an argument. Return a new value. - No I/O and no clock reads at import time. -- Take what you need as a parameter, keyword-only unless it is the subject of the call. Pass specific - values, never a whole configuration object. +- Take what you need as a parameter. Pass specific values, never a whole configuration object. +- No bare `*` in a signature. Call sites name their arguments (see Layout), so forcing it in every + declaration only adds noise; `FBT003` catches the positional boolean that actually misreads. ## Typing and syntax diff --git a/pyproject.toml b/pyproject.toml index ea532f3..a6aa823 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ select = [ "I", # isort "B", # flake8-bugbear - catches common bug patterns "UP", # pyupgrade - enforces modern syntax, including `str | None` over `Optional[str]` + # Only the call-site rule of flake8-boolean-trap: a bare `True` in a call reads as nothing. + # FBT001/FBT002 are left off because a tool's boolean parameter is a named field of its schema. + "FBT003", # flake8-boolean-trap ] [tool.ruff.lint.flake8-bugbear] diff --git a/src/mcpdiffusion/error.py b/src/mcpdiffusion/error.py index ef9dcbd..236b3ec 100644 --- a/src/mcpdiffusion/error.py +++ b/src/mcpdiffusion/error.py @@ -38,7 +38,6 @@ def __init__( self, code: ErrorCode, message: str, - *, retryable: bool = False, ) -> None: self.code = code diff --git a/src/mcpdiffusion/instructions.py b/src/mcpdiffusion/instructions.py index fbf00de..a418a1e 100644 --- a/src/mcpdiffusion/instructions.py +++ b/src/mcpdiffusion/instructions.py @@ -170,7 +170,6 @@ def build_instructions( - *, enable_inseefr_tools: bool, enable_melodi_tools: bool, enable_rmes_tools: bool, diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py index c65f6c6..017c108 100644 --- a/src/mcpdiffusion/services/feedback.py +++ b/src/mcpdiffusion/services/feedback.py @@ -32,7 +32,6 @@ def _ensure_feedback_file() -> Path: # Fixme: Also wondering if there is a cap on the author size of the feedback content, # it can make the container write uncontrolled amount of data async def send_feedback_service( - *, author: str, feedback: str, ) -> FeedbackOutput: From f0b7f9433b292bf37d5766c6f28df28eae624c54 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 15:11:38 +0200 Subject: [PATCH 32/55] fix(rmes): stop the sparql tool's limits from governing other queries run_rmes_sparql's schema bounds were applied inside the shared execute path, so the graph listing was silently capped at 60s no matter what RMES_GRAPH_LISTING_TIMEOUT_SECONDS was set to, and describe_rmes_resource drew its own budget from a tool it does not expose. The ceiling now sits in run_rmes_sparql, next to the max_rows clamp, and describe_rmes_resource carries its own timeout and row limit. --- .../services/rmes/graph_store_service.py | 22 +++++++++---------- .../tools/rmes/run_sparql_tool.py | 3 ++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py index 365f390..aee407b 100644 --- a/src/mcpdiffusion/services/rmes/graph_store_service.py +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -15,13 +15,13 @@ import httpx from ...error import AppToolError, ErrorCode -from ...models.rmes import ( - DEFAULT_QUERY_TIMEOUT_SECONDS, - MAX_QUERY_TIMEOUT_SECONDS, - MAX_ROW_LIMIT, - GraphRow, - ResourceProperty, -) +from ...models.rmes import GraphRow, ResourceProperty + +# describe_rmes_resource issues a fixed query the model cannot size, so it carries its own budget. +# These numbers were once the whole module's shared budget. Once the same values also became +# run_rmes_sparql's schema bounds, sharing them let one tool's parameters govern this one. +RESOURCE_QUERY_TIMEOUT_SECONDS = 20.0 +RESOURCE_QUERY_ROW_LIMIT = 2000 # Ask the store for one row per named graph, with how many triples it holds, biggest first. # `?s ?p ?o` matches every triple, so COUNT(*) per ?g is that graph's size. It is the only @@ -120,7 +120,7 @@ def build_resource_query(resource_uri: str, graph_uri: str | None) -> str: GRAPH {graph_clause} {{ ?o ?p <{resource_uri}> }} BIND("incoming" AS ?direction) }} - }} LIMIT {MAX_ROW_LIMIT} + }} LIMIT {RESOURCE_QUERY_ROW_LIMIT} """ @@ -176,7 +176,7 @@ async def execute( self._sparql_endpoint_url, data={"query": effective_query}, headers={"Accept": accept}, - timeout=min(timeout_seconds, MAX_QUERY_TIMEOUT_SECONDS), + timeout=timeout_seconds, ) response.raise_for_status() except httpx.TimeoutException: @@ -267,7 +267,7 @@ async def describe_resource( """Return every triple the endpoint holds about the resource, in either direction.""" response = await self.execute( build_resource_query(resource_uri, graph_uri), - timeout_seconds=DEFAULT_QUERY_TIMEOUT_SECONDS, - max_rows=MAX_ROW_LIMIT, + timeout_seconds=RESOURCE_QUERY_TIMEOUT_SECONDS, + max_rows=RESOURCE_QUERY_ROW_LIMIT, ) return parse_resource_properties(response.bindings or []) diff --git a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py index 7d1c81a..9e2f318 100644 --- a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py +++ b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py @@ -9,6 +9,7 @@ from ...models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, DEFAULT_ROW_LIMIT, + MAX_QUERY_TIMEOUT_SECONDS, MAX_ROW_LIMIT, MaxRows, SparqlOutput, @@ -71,7 +72,7 @@ async def run_rmes_sparql( response = await rmes_graph_store_service.execute( query=sparql_query, - timeout_seconds=timeout_seconds, + timeout_seconds=min(timeout_seconds, MAX_QUERY_TIMEOUT_SECONDS), max_rows=max(1, min(max_rows, MAX_ROW_LIMIT)), ) if response.turtle is not None: From cd37282481d335e31985049ea2d5e3ddf5349731 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 15:11:50 +0200 Subject: [PATCH 33/55] feat(config): make the Elasticsearch retries and document budget configurable ES_MAX_RETRIES and INSEE_DOCUMENT_MAX_MARKDOWN_CHARS were hardcoded constants, though both depend on the deployment: how reliable that Elasticsearch is, and how much context the calling model has. Both reach their clients through the lifespan rather than being read where they are used. --- .env.example | 5 +++++ src/mcpdiffusion/lifespan/__init__.py | 2 ++ src/mcpdiffusion/lifespan/elasticsearch.py | 5 ++--- src/mcpdiffusion/lifespan/insee.py | 6 +++++- .../services/insee/document_service.py | 20 +++++++++++++------ src/mcpdiffusion/settings.py | 6 ++++++ 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index da1d3a1..6da1a78 100644 --- a/.env.example +++ b/.env.example @@ -33,11 +33,16 @@ ES_INDEX_MELODI_COLUMNS=melodi_columns # Only Elasticsearch is configurable here. ES_TLS_VERIFY=true ES_REQUEST_TIMEOUT_SECONDS=30 +# Retries the client makes itself before a search fails. +ES_MAX_RETRIES=2 # INSEE services ------------------------------------------------------------------------------------------------------- INSEE_BASE_URL=https://www.insee.fr INSEE_REQUEST_TIMEOUT_SECONDS=30 INSEE_CONNECT_TIMEOUT_SECONDS=10 +# A rendered publication is truncated past this many characters, so one document cannot fill +# the calling model's context. +INSEE_DOCUMENT_MAX_MARKDOWN_CHARS=30000 MELODI_DATA_BASE_URL=https://api.insee.fr/melodi/data MELODI_REQUEST_TIMEOUT_SECONDS=30 MELODI_CONNECT_TIMEOUT_SECONDS=10 diff --git a/src/mcpdiffusion/lifespan/__init__.py b/src/mcpdiffusion/lifespan/__init__.py index 69af0a6..c176f93 100644 --- a/src/mcpdiffusion/lifespan/__init__.py +++ b/src/mcpdiffusion/lifespan/__init__.py @@ -42,6 +42,7 @@ async def app_lifespan(_server: FastMCP[Any]) -> AsyncIterator[dict[str, Any]]: host=es_host, tls_verify=settings.es_tls_verify, request_timeout_seconds=settings.es_request_timeout_seconds, + max_retries=settings.es_max_retries, ) ) @@ -53,6 +54,7 @@ async def app_lifespan(_server: FastMCP[Any]) -> AsyncIterator[dict[str, Any]]: request_timeout_seconds=settings.insee_request_timeout_seconds, connect_timeout_seconds=settings.insee_connect_timeout_seconds, publications_index=settings.es_index_publications, + document_max_markdown_chars=settings.insee_document_max_markdown_chars, ) ) diff --git a/src/mcpdiffusion/lifespan/elasticsearch.py b/src/mcpdiffusion/lifespan/elasticsearch.py index fd8b8c6..ae65283 100644 --- a/src/mcpdiffusion/lifespan/elasticsearch.py +++ b/src/mcpdiffusion/lifespan/elasticsearch.py @@ -10,14 +10,13 @@ logger = logging.getLogger(__name__) -MAX_RETRIES = 2 - @asynccontextmanager async def elasticsearch_lifespan( host: str, tls_verify: bool, request_timeout_seconds: int, + max_retries: int, ) -> AsyncIterator[AsyncElasticsearch]: """Open the shared client and close it on shutdown. @@ -27,7 +26,7 @@ async def elasticsearch_lifespan( host, verify_certs=tls_verify, request_timeout=request_timeout_seconds, - max_retries=MAX_RETRIES, + max_retries=max_retries, retry_on_timeout=True, ) logger.info("Elasticsearch client initialized for %s", host) diff --git a/src/mcpdiffusion/lifespan/insee.py b/src/mcpdiffusion/lifespan/insee.py index 411ae5f..d75c674 100644 --- a/src/mcpdiffusion/lifespan/insee.py +++ b/src/mcpdiffusion/lifespan/insee.py @@ -26,6 +26,7 @@ async def insee_lifespan( request_timeout_seconds: int, connect_timeout_seconds: int, publications_index: str, + document_max_markdown_chars: int, ) -> AsyncIterator[dict[str, Any]]: """Build the insee.fr services and close the scraping client on shutdown.""" http_client = AsyncClient( @@ -43,7 +44,10 @@ async def insee_lifespan( elasticsearch_client=elasticsearch_client, publications_index=publications_index, ), - "insee_document_service": InseeDocumentService(http_client=http_client), + "insee_document_service": InseeDocumentService( + http_client=http_client, + max_markdown_chars=document_max_markdown_chars, + ), } finally: await http_client.aclose() diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index 344fb5f..d6c01fe 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -20,13 +20,12 @@ output_format="markdown", links=True, formatting=True, - # Fixme: this URL might belong in the settings + # A metadata label, not an address to call: trafilatura records it, but our markdown comes out + # byte-identical whatever it is set to. source="insee.fr", with_metadata=True, ) -MAX_MARKDOWN_CHARS = 30_000 - TRUNCATION_MARKER = """ @@ -110,7 +109,7 @@ def group_table_of_contents(entries: TableOfContentsEntries) -> TableOfContents: return dict(by_category) -def truncate_markdown(text: str, limit: int = MAX_MARKDOWN_CHARS) -> tuple[str, bool]: +def truncate_markdown(text: str, limit: int) -> tuple[str, bool]: """Keep the head and tail of an over-long document, marking where the middle was dropped.""" if len(text) <= limit: return text, False @@ -142,8 +141,13 @@ def build_failed_document(url: str, message: str) -> DocumentResult: class InseeDocumentService: """Fetches insee.fr publication pages and renders them as markdown.""" - def __init__(self, http_client: httpx.AsyncClient) -> None: + def __init__( + self, + http_client: httpx.AsyncClient, + max_markdown_chars: int, + ) -> None: self._http_client = http_client + self._max_markdown_chars = max_markdown_chars async def fetch_html(self, url: str) -> str: """Return the raw HTML of one publication page.""" @@ -197,7 +201,11 @@ async def fetch_documents( try: html = await self.fetch_html(url) markdown = extract(html, options=TRAFILATURA_OPTIONS) or "" - markdown, truncated = truncate_markdown(markdown) if truncate_content else (markdown, False) + markdown, truncated = ( + truncate_markdown(markdown, limit=self._max_markdown_chars) + if truncate_content + else (markdown, False) + ) table_of_contents: TableOfContents | None = None if include_table_of_contents: diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 73fae73..7c4163f 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -33,11 +33,17 @@ class Settings(BaseSettings): # Elasticsearch is often internal with a self-signed certificate. es_tls_verify: bool = True es_request_timeout_seconds: int = 30 + # Retries the client makes itself before a search fails. Raising it hides brief outages; + # lowering it surfaces them sooner. + es_max_retries: int = 2 # INSEE services --------------------------------------------------------------------------------------------------- insee_base_url: str = "https://www.insee.fr" insee_request_timeout_seconds: int = 30 insee_connect_timeout_seconds: int = 10 + # A rendered publication is truncated past this many characters, so one document cannot + # fill the calling model's context. Tune it to the context budget of the client in use. + insee_document_max_markdown_chars: int = 30_000 melodi_data_base_url: str = "https://api.insee.fr/melodi/data" melodi_request_timeout_seconds: int = 30 melodi_connect_timeout_seconds: int = 10 From 174ccc861a74173ee5369b3576315dd76166836a Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 15:12:33 +0200 Subject: [PATCH 34/55] refactor: give melodi and rmes the same user agent format The two clients announced themselves as McpDiffusion/0.1 and MCP-RMeS/2.0: two product names and two versions, neither matching pyproject.toml. Both now send McpDiffusion/0.1.0. insee.fr keeps its browser string, which it needs to be served the real markup. --- src/mcpdiffusion/lifespan/melodi.py | 4 ++-- src/mcpdiffusion/lifespan/rmes.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mcpdiffusion/lifespan/melodi.py b/src/mcpdiffusion/lifespan/melodi.py index fb2765f..8c4dd76 100644 --- a/src/mcpdiffusion/lifespan/melodi.py +++ b/src/mcpdiffusion/lifespan/melodi.py @@ -15,8 +15,8 @@ logger = logging.getLogger(__name__) -# The API has no browser requirement, so it gets an honest identity. -USER_AGENT = "McpDiffusion/0.1" +# Honest identity -- these APIs need no browser spoofing. Hardcoded: bump with pyproject.toml maybe. +USER_AGENT = "McpDiffusion/0.1.0" @asynccontextmanager diff --git a/src/mcpdiffusion/lifespan/rmes.py b/src/mcpdiffusion/lifespan/rmes.py index f8371da..8983255 100644 --- a/src/mcpdiffusion/lifespan/rmes.py +++ b/src/mcpdiffusion/lifespan/rmes.py @@ -13,7 +13,8 @@ logger = logging.getLogger(__name__) -USER_AGENT = "MCP-RMeS/2.0" +# Honest identity -- these APIs need no browser spoofing. Hardcoded: bump with pyproject.toml maybe. +USER_AGENT = "McpDiffusion/0.1.0" @asynccontextmanager From 9e125ac651ec19bb23de6595f52dd173842bc53e Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 15:13:15 +0200 Subject: [PATCH 35/55] docs(models): record why the schema bounds are not settings Both files carried a Fixme claiming these values belong in settings. They do not: they bound the published tool schema, so an env-driven value would advertise a different contract per deployment, and they are read at import time, before any Settings instance exists. The banner said "Constants", which explained nothing about why they sit beside the schema they shape. --- src/mcpdiffusion/models/insee.py | 6 ++++-- src/mcpdiffusion/models/rmes.py | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 5639fb8..7aceb43 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -8,10 +8,12 @@ from pydantic import BaseModel, Field # ---------------------------------------------------------------------------------------------------------------------- -# Constants ------------------------------------------------------------------------------------------------------------ +# Schema bounds -------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- -# Fixme: a lot of values in here belongs in settings +# Deliberately not settings: these bound the tool's published schema, so an env-driven value would +# advertise a different contract per deployment under the same tool name. They are also read at +# import time, before any Settings instance exists. DEFAULT_RESULT_COUNT = 10 MAX_RESULT_COUNT = 20 diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index 4664f46..2aa79df 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -10,10 +10,13 @@ from ..data.rmes.graph_categories import CATEGORY_DEFINITIONS, FALLBACK_CATEGORY_DEFINITION # ---------------------------------------------------------------------------------------------------------------------- -# Constants ------------------------------------------------------------------------------------------------------------ +# Schema bounds -------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- -# Fixme: a lot of values in here belongs in settings +# Deliberately not settings: these bound the tool's published schema, so an env-driven value would +# advertise a different contract per deployment under the same tool name. They are also read at +# import time, before any Settings instance exists. The budgets an operator does tune are the +# RMES_GRAPH_LISTING_* settings, which the graph store service takes as constructor arguments. DEFAULT_QUERY_TIMEOUT_SECONDS = 20.0 MAX_QUERY_TIMEOUT_SECONDS = 60.0 DEFAULT_ROW_LIMIT = 200 From 4d814e64fa6bf522926c5c22327e0f7aee5201e0 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 15:13:36 +0200 Subject: [PATCH 36/55] docs(core): hyphenate retry-ability in the error rules --- .claude/rules/error.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/rules/error.md b/.claude/rules/error.md index 9535c09..f6b1348 100644 --- a/.claude/rules/error.md +++ b/.claude/rules/error.md @@ -10,7 +10,7 @@ error contract — consistency means a consistent message, produced in one place - `core/errors.py` owns every error type. Nothing else defines an error enum, model or vocabulary. - Error types subclass `ToolError` — its message always reaches the client. Anything else is an internal fault and must not leak. -- Code and retryability are attributes on the exception, never formatted into the message. +- Code and retry-ability are attributes on the exception, never formatted into the message. ## Raising From 310f19c33553aac5c50443a3be343ae3d9e25beb Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 16:07:35 +0200 Subject: [PATCH 37/55] fix: restore the feedback tool, dropped when the tool list was restructured send_feedback was registered until d00e9a7 removed the else branch that carried it. It was the only tool registered exclusively there, so it went unnoticed while SKILL.md kept telling clients to call it. It now records to the server log instead of appending to a file inside the container, which was lost on every restart, blocked the event loop, and was tracked by git despite holding user-submitted content. Client text is JSON-encoded into the message so an embedded newline cannot forge a second log line, and both fields are length-bounded in the schema. Registration is gated on ENABLE_FEEDBACK_TOOL, matching the three data sources, and the handshake instructions name the tool only when it is enabled. feedback.md stays on disk but is no longer tracked. --- .claude/rules/logging.md | 10 ++- .env.example | 2 + .gitignore | 4 ++ src/mcpdiffusion/feedback/feedback.md | 12 ---- src/mcpdiffusion/instructions.py | 20 +++++- src/mcpdiffusion/models/feedback.py | 12 ++++ src/mcpdiffusion/server.py | 1 + src/mcpdiffusion/services/feedback.py | 74 +++++++------------- src/mcpdiffusion/settings.py | 2 + src/mcpdiffusion/tools/__init__.py | 18 ++--- src/mcpdiffusion/tools/feedback_send.py | 26 ------- src/mcpdiffusion/tools/send_feedback_tool.py | 24 +++++++ 12 files changed, 108 insertions(+), 97 deletions(-) delete mode 100644 src/mcpdiffusion/feedback/feedback.md delete mode 100644 src/mcpdiffusion/tools/feedback_send.py create mode 100644 src/mcpdiffusion/tools/send_feedback_tool.py diff --git a/.claude/rules/logging.md b/.claude/rules/logging.md index 4348c18..52e8370 100644 --- a/.claude/rules/logging.md +++ b/.claude/rules/logging.md @@ -28,12 +28,20 @@ Never send the same message to both. `logger` with a redacting filter is in place. - Register logging middleware last, so it records execution after the rest of the chain has run. +## Deliberate exception: feedback + +`send_feedback` records a client's report at `info` on the server log. That is a widening of the +channel -- it is not an incident, and nobody is on call for it -- and it is the point: a file inside +the container is lost on the next restart, while the log already reaches the operators. Client text +is JSON-encoded into the message so an embedded newline cannot forge a second log line. + ## Who logs what - **Middleware logs failures, not services.** `ErrorHandlingMiddleware` already catches, logs and converts every exception. Code that logs before raising records the same failure twice. - A service logs only what the exception cannot carry, and never at `error` level. -- Never log credentials, tokens or request bodies. +- Never log credentials, tokens or request bodies. The single exception is `send_feedback`, whose + body is the record itself -- see "Deliberate exception" above. ## Configuration diff --git a/.env.example b/.env.example index 6da1a78..7214e9c 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,8 @@ TRUSTED_PROXY_HOSTS=["127.0.0.1"] ENABLE_INSEEFR_TOOLS=true ENABLE_MELODI_TOOLS=true ENABLE_RMES_TOOLS=true +# Lets a client report a broken tool. Records to the server log; needs no backend. +ENABLE_FEEDBACK_TOOL=true # Elasticsearch -------------------------------------------------------------------------------------------------------- # Inside Docker use the service name; on the host use localhost. diff --git a/.gitignore b/.gitignore index 1f1d4ea..b07af4d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ mcp_* .idea .claude/settings.local.json TODO.md + +# User-submitted feedback. Kept on disk, never versioned -- the tool now records to the +# server log, so nothing writes here any more. +src/mcpdiffusion/feedback/ diff --git a/src/mcpdiffusion/feedback/feedback.md b/src/mcpdiffusion/feedback/feedback.md deleted file mode 100644 index b9144e8..0000000 --- a/src/mcpdiffusion/feedback/feedback.md +++ /dev/null @@ -1,12 +0,0 @@ -# Feedback Log - -This file collects feedback from users and the assistant about MCP tools, server behavior, and suggestions for improvement. Each entry is timestamped and formatted as Markdown for easy review. - ---- - -## 2026-08-04T13:12:08 — mirlon - -hello from mcp inspector - ---- - diff --git a/src/mcpdiffusion/instructions.py b/src/mcpdiffusion/instructions.py index a418a1e..14c0db3 100644 --- a/src/mcpdiffusion/instructions.py +++ b/src/mcpdiffusion/instructions.py @@ -169,12 +169,28 @@ """ +FEEDBACK_SECTION = """ + ## FEEDBACK + + ### `send_feedback` + + WHEN TO USE + - A tool failed, returned an empty result you have good reason to think is wrong, or its description led you to + the wrong call. Say which tool and what you expected. + + WHEN NOT TO USE + - To answer the person you are talking to. It reaches the server maintainers, not them. + - To keep notes for yourself, or to acknowledge a call that worked. +""" + + def build_instructions( enable_inseefr_tools: bool, enable_melodi_tools: bool, enable_rmes_tools: bool, + enable_feedback_tool: bool, ) -> str: - """Assemble the guidance for the tool families this deployment actually registers.""" + """Assemble the guidance for the tools this deployment actually registers.""" sections = [OVERVIEW, GLOBAL_RULES] if enable_inseefr_tools: sections.append(INSEE_SECTION) @@ -182,4 +198,6 @@ def build_instructions( sections.append(MELODI_SECTION) if enable_rmes_tools: sections.append(RMES_SECTION) + if enable_feedback_tool: + sections.append(FEEDBACK_SECTION) return "\n\n".join(dedent(section).strip() for section in sections) diff --git a/src/mcpdiffusion/models/feedback.py b/src/mcpdiffusion/models/feedback.py index 440f90c..a55c6f6 100644 --- a/src/mcpdiffusion/models/feedback.py +++ b/src/mcpdiffusion/models/feedback.py @@ -7,6 +7,16 @@ from pydantic import BaseModel, Field +# ---------------------------------------------------------------------------------------------------------------------- +# Schema bounds -------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------------- + +# Both fields are client-supplied and land in the server log, so they are bounded here rather +# than trusted. Pydantic rejects an over-long value before any of it is recorded. +MAX_AUTHOR_CHARS = 100 +MAX_FEEDBACK_CHARS = 10_000 + + # ---------------------------------------------------------------------------------------------------------------------- # Tool parameters ------------------------------------------------------------------------------------------------------ # ---------------------------------------------------------------------------------------------------------------------- @@ -15,6 +25,7 @@ str, Field( description="Identifier for the feedback author (e.g., user name, role, or session ID).", + max_length=MAX_AUTHOR_CHARS, examples=[ "alice", "data_analyst", @@ -31,6 +42,7 @@ "(which tool, what happened), expected vs actual behavior, and proposed solutions " "if applicable. Write as if filing a GitHub issue." ), + max_length=MAX_FEEDBACK_CHARS, examples=[ "## Bug Report\n\n**Tool:** search_melodi_datasets\n\n**Issue:** No results returned " "for 'prix du pain' even though dataset DS_PRIX exists.\n\n**Expected:** Should find " diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 237484a..2f11d82 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -28,6 +28,7 @@ enable_inseefr_tools=settings.enable_inseefr_tools, enable_melodi_tools=settings.enable_melodi_tools, enable_rmes_tools=settings.enable_rmes_tools, + enable_feedback_tool=settings.enable_feedback_tool, ), # Only AppToolError messages reach the caller; anything else is a bug and is replaced # by a generic message. diff --git a/src/mcpdiffusion/services/feedback.py b/src/mcpdiffusion/services/feedback.py index 017c108..8d05af5 100644 --- a/src/mcpdiffusion/services/feedback.py +++ b/src/mcpdiffusion/services/feedback.py @@ -2,58 +2,36 @@ from __future__ import annotations -from datetime import datetime -from pathlib import Path +import json +import logging +from datetime import UTC, datetime from ..models.feedback import FeedbackOutput -# Fixme: this is extremely hacky, and feedback gets tied to the running instance -_FEEDBACK_DIR = Path(__file__).resolve().parent.parent / "feedback" -_FEEDBACK_FILE = _FEEDBACK_DIR / "feedback.md" - - -# Fixme: this function writes files within the running container, this is a major side effect -# Fixme: also check where this code is invoked, because if in the event loop, it is blocking -# Fixme: those kind of checks belong at the app startup, for example in a lifespan function -def _ensure_feedback_file() -> Path: - _FEEDBACK_DIR.mkdir(parents=True, exist_ok=True) - if not _FEEDBACK_FILE.exists(): - _FEEDBACK_FILE.write_text( - "# Feedback Log\n\n" - "This file collects feedback from users and the assistant about MCP tools, " - "server behavior, and suggestions for improvement. Each entry is timestamped " - "and formatted as Markdown for easy review.\n\n---\n\n", - encoding="utf-8", - ) - return _FEEDBACK_FILE - - -# Fixme: I am wondering where the author come from, because if sent by the client, this can be messed up -# Fixme: Also wondering if there is a cap on the author size of the feedback content, -# it can make the container write uncontrolled amount of data -async def send_feedback_service( - author: str, - feedback: str, -) -> FeedbackOutput: - feedback_path = _ensure_feedback_file() - # Fixme: there is no timezone here, while at some in the code we consider timezones - # it seems a bit inconsistent - recorded_at = datetime.now() - - # Fixme: prefer more readable multiline strings - entry = ( - # Fixme: again, people can insert anything and forge data into the feedback file - # just hope this is not ultimately fed to an LLM - # Fixme: a big flaw is that feedback.md is versioned, so user feedback might be fed into git - # Fixme: beware the data is lost on each restart - f"## {recorded_at.isoformat(timespec='seconds')} — {author}\n\n{feedback}\n\n---\n\n" +logger = logging.getLogger(__name__) + + +def record_feedback(author: str, feedback: str) -> FeedbackOutput: + """Record one feedback entry in the server log and confirm it to the caller. + + The log is the sink on purpose. A file written inside the container is lost on the next + restart and reaches nobody; the log already goes wherever the operators are looking. + + Both fields come from the client, so they are JSON-encoded into the message. That keeps the + entry on a single line and stops a crafted newline from forging a second one. The same values + go in `extra` for aggregators that read structured fields rather than the rendered message. + """ + recorded_at = datetime.now(UTC) + logger.info( + "Feedback received: author=%s body=%s", + json.dumps(author), + json.dumps(feedback), + extra={ + "feedback_author": author, + "feedback_body": feedback, + "feedback_chars_count": len(feedback), + }, ) - - # Fixme: this call is blocking the event loop - # consider using aiofiles instead - with feedback_path.open("a", encoding="utf-8") as f: - f.write(entry) - return FeedbackOutput( message="Feedback recorded successfully.", timestamp=recorded_at, diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 7c4163f..1ce8221 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -22,6 +22,8 @@ class Settings(BaseSettings): enable_inseefr_tools: bool = True enable_melodi_tools: bool = True enable_rmes_tools: bool = True + # Reporting only: it records to the server log and needs no backend. + enable_feedback_tool: bool = True # Elasticsearch ---------------------------------------------------------------------------------------------------- # Only the insee.fr and Melodi tools search Elasticsearch; rmes runs without it, so the diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index bba1e99..d7abda4 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -1,11 +1,11 @@ """Tool registration entrypoint. -The only place that knows about the MCP server. A family is registered only when its flag is on: -an unregistered tool is the one kind of "disabled" the protocol guarantees, unlike tag or -visibility filtering, which a later call can undo. +The only place that knows about the MCP server. Each group -- the three data sources, and the +feedback tool -- is registered only when its flag is on: an unregistered tool is the one kind of +"disabled" the protocol guarantees, unlike tag or visibility filtering, which a later call can undo. -Every tool is a plain function that takes the service it needs through `Depends`, so none of them -carries a registration wrapper. +Every tool is a plain function, so none of them carries a registration wrapper. Those that need a +service take it through `Depends`; `send_feedback` needs none. """ from __future__ import annotations @@ -13,10 +13,6 @@ from fastmcp import FastMCP from ..settings import Settings - -# Imported but never registered: send_feedback is not exposed. Decide whether to wire it up or -# drop it, then remove this import or the noqa. -from .feedback_send import register_send_feedback # noqa: F401 from .insee.get_document_tool import get_insee_document from .insee.get_homepage_tool import get_insee_homepage from .insee.search_chiffrecle_tool import search_insee_chiffrecle @@ -28,6 +24,7 @@ from .rmes.describe_resource_tool import describe_rmes_resource from .rmes.run_sparql_tool import run_rmes_sparql from .rmes.search_graphs_tool import search_rmes_graphs +from .send_feedback_tool import send_feedback def register_tools(mcp: FastMCP, settings: Settings) -> None: @@ -48,3 +45,6 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: mcp.add_tool(search_rmes_graphs) mcp.add_tool(describe_rmes_resource) mcp.add_tool(run_rmes_sparql) + + if settings.enable_feedback_tool: + mcp.add_tool(send_feedback) diff --git a/src/mcpdiffusion/tools/feedback_send.py b/src/mcpdiffusion/tools/feedback_send.py deleted file mode 100644 index 24cffe3..0000000 --- a/src/mcpdiffusion/tools/feedback_send.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Tool: send_feedback -- thin registration layer.""" - -from __future__ import annotations - -from fastmcp import FastMCP - -from ..models.feedback import Author, Feedback, FeedbackOutput -from ..services.feedback import send_feedback_service - - -# Fixme: I already stated clients can send anything as author and feedback -def register_send_feedback(mcp: FastMCP) -> None: - @mcp.tool - async def send_feedback( - author: Author, - feedback: Feedback, - ) -> FeedbackOutput: - """Submit structured feedback about the MCP tools, server behavior, or user experience. This - tool appends a timestamped Markdown entry to the feedback log for administrator review. - - Returns a confirmation carrying the timestamp under which the feedback was recorded. - """ - return await send_feedback_service( - author=author, - feedback=feedback, - ) diff --git a/src/mcpdiffusion/tools/send_feedback_tool.py b/src/mcpdiffusion/tools/send_feedback_tool.py new file mode 100644 index 0000000..da2dc61 --- /dev/null +++ b/src/mcpdiffusion/tools/send_feedback_tool.py @@ -0,0 +1,24 @@ +"""Tool: send_feedback.""" + +from __future__ import annotations + +from ..models.feedback import Author, Feedback, FeedbackOutput +from ..services.feedback import record_feedback + + +def send_feedback( + author: Author, + feedback: Feedback, +) -> FeedbackOutput: + """Report a problem or a suggestion about this server's tools to the people who maintain it. + + Use it when a tool failed, returned an empty result you have good reason to think is wrong, or + carried a description that led you to the wrong call. The entry reaches the server operators, + not the person you are talking to, so it is not a way to answer them. + + Returns a confirmation carrying the timestamp under which the feedback was recorded. + """ + return record_feedback( + author=author, + feedback=feedback, + ) From 4d68c1d42a672feeacf70535f12081cbc3c296dc Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 17:27:30 +0200 Subject: [PATCH 38/55] refactor: group the error contract into its own package error.py and the Elasticsearch failure translator sat at the package root and in services/ respectively, though both belong to one concern and are used by every layer. errors/ now holds them, with a membership rule the root never had: does it define or produce an AppToolError. The translator also gains a home it lacked. It was never a service -- it orchestrates nothing and holds no client -- so services/ only ever housed it for want of anywhere better. --- src/mcpdiffusion/errors/__init__.py | 12 ++++++++++++ .../elasticsearch_tool_error_handler.py} | 4 ++-- src/mcpdiffusion/{ => errors}/error.py | 0 src/mcpdiffusion/services/insee/document_service.py | 2 +- src/mcpdiffusion/services/insee/index_service.py | 4 ++-- src/mcpdiffusion/services/melodi/api_service.py | 2 +- src/mcpdiffusion/services/melodi/index_service.py | 6 +++--- .../services/rmes/graph_store_service.py | 2 +- src/mcpdiffusion/tools/rmes/run_sparql_tool.py | 2 +- 9 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 src/mcpdiffusion/errors/__init__.py rename src/mcpdiffusion/{services/elasticsearch_failures.py => errors/elasticsearch_tool_error_handler.py} (95%) rename src/mcpdiffusion/{ => errors}/error.py (100%) diff --git a/src/mcpdiffusion/errors/__init__.py b/src/mcpdiffusion/errors/__init__.py new file mode 100644 index 0000000..a74479a --- /dev/null +++ b/src/mcpdiffusion/errors/__init__.py @@ -0,0 +1,12 @@ +"""The error contract: the single type tools and services raise, and what produces it. + +Re-exported here so callers name the concern rather than the file: `from ...errors import +AppToolError`. The per-backend translators are imported from their own modules. +""" + +from .error import AppToolError, ErrorCode + +__all__ = [ + "AppToolError", + "ErrorCode", +] diff --git a/src/mcpdiffusion/services/elasticsearch_failures.py b/src/mcpdiffusion/errors/elasticsearch_tool_error_handler.py similarity index 95% rename from src/mcpdiffusion/services/elasticsearch_failures.py rename to src/mcpdiffusion/errors/elasticsearch_tool_error_handler.py index 53b16f8..74ed7df 100644 --- a/src/mcpdiffusion/services/elasticsearch_failures.py +++ b/src/mcpdiffusion/errors/elasticsearch_tool_error_handler.py @@ -16,11 +16,11 @@ from elasticsearch import ApiError, TransportError -from ..error import AppToolError, ErrorCode +from .error import AppToolError, ErrorCode @asynccontextmanager -async def elasticsearch_failures_as_tool_errors(backend_label: str) -> AsyncIterator[None]: +async def elasticsearch_tool_error_handler(backend_label: str) -> AsyncIterator[None]: """Translate a failed Elasticsearch search into an `AppToolError` naming the backend. Wraps the `await` rather than performing it, so it fits both the DSL and the raw client. diff --git a/src/mcpdiffusion/error.py b/src/mcpdiffusion/errors/error.py similarity index 100% rename from src/mcpdiffusion/error.py rename to src/mcpdiffusion/errors/error.py diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index d6c01fe..6a0d229 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -11,7 +11,7 @@ from trafilatura import extract from trafilatura.settings import Extractor -from ...error import AppToolError, ErrorCode +from ...errors import AppToolError, ErrorCode from ...models.insee import DocumentResult, TableOfContents logger = logging.getLogger(__name__) diff --git a/src/mcpdiffusion/services/insee/index_service.py b/src/mcpdiffusion/services/insee/index_service.py index d64b116..c39d7f9 100644 --- a/src/mcpdiffusion/services/insee/index_service.py +++ b/src/mcpdiffusion/services/insee/index_service.py @@ -20,8 +20,8 @@ from ...data.insee.geography import DICT_GEO from ...data.insee.themes import DICT_THEME_CONJ, KEYS_THEME_NIV1 +from ...errors.elasticsearch_tool_error_handler import elasticsearch_tool_error_handler from ...models.insee import DocumentHit -from ..elasticsearch_failures import elasticsearch_failures_as_tool_errors RAPIDES_COLLECTION = "Informations rapides" CHIFFRES_CLES_CATEGORY = "Chiffres-clés" @@ -327,7 +327,7 @@ async def _run( ) -> list[DocumentHit]: """Bind the search to the client and index, execute it, and map the hits.""" bound = search.using(self._elasticsearch_client).index(self._publications_index) - async with elasticsearch_failures_as_tool_errors(backend_label): + async with elasticsearch_tool_error_handler(backend_label): response = await bound.execute() return parse_document_hits(response) diff --git a/src/mcpdiffusion/services/melodi/api_service.py b/src/mcpdiffusion/services/melodi/api_service.py index f748580..fd4e2d2 100644 --- a/src/mcpdiffusion/services/melodi/api_service.py +++ b/src/mcpdiffusion/services/melodi/api_service.py @@ -6,7 +6,7 @@ import httpx -from ...error import AppToolError, ErrorCode +from ...errors import AppToolError, ErrorCode class MelodiApiService: diff --git a/src/mcpdiffusion/services/melodi/index_service.py b/src/mcpdiffusion/services/melodi/index_service.py index 9859d2e..9c62062 100644 --- a/src/mcpdiffusion/services/melodi/index_service.py +++ b/src/mcpdiffusion/services/melodi/index_service.py @@ -12,13 +12,13 @@ from elasticsearch.dsl.response import Hit from elasticsearch.dsl.utils import AttrList +from ...errors.elasticsearch_tool_error_handler import elasticsearch_tool_error_handler from ...models.melodi import ( ColumnResult, DatasetDescription, DatasetSearchResult, Modality, ) -from ..elasticsearch_failures import elasticsearch_failures_as_tool_errors # The column query asks for a fixed page of columns and narrows within them via inner_hits. COLUMN_SEARCH_SIZE = 20 @@ -194,7 +194,7 @@ async def search_datasets( .using(self._elasticsearch_client) .index(self._datasets_index) ) - async with elasticsearch_failures_as_tool_errors("Melodi datasets"): + async with elasticsearch_tool_error_handler("Melodi datasets"): response = await search.execute() results: list[DatasetSearchResult] = [] @@ -227,7 +227,7 @@ async def search_columns( .using(self._elasticsearch_client) .index(self._columns_index) ) - async with elasticsearch_failures_as_tool_errors("Melodi columns"): + async with elasticsearch_tool_error_handler("Melodi columns"): response = await search.execute() results: list[ColumnResult] = [] diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py index aee407b..d8cc928 100644 --- a/src/mcpdiffusion/services/rmes/graph_store_service.py +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -14,7 +14,7 @@ import httpx -from ...error import AppToolError, ErrorCode +from ...errors import AppToolError, ErrorCode from ...models.rmes import GraphRow, ResourceProperty # describe_rmes_resource issues a fixed query the model cannot size, so it carries its own budget. diff --git a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py index 9e2f318..4d4b74a 100644 --- a/src/mcpdiffusion/tools/rmes/run_sparql_tool.py +++ b/src/mcpdiffusion/tools/rmes/run_sparql_tool.py @@ -5,7 +5,7 @@ from fastmcp.dependencies import Depends from ...dependencies.rmes import get_rmes_graph_store_service -from ...error import AppToolError, ErrorCode +from ...errors import AppToolError, ErrorCode from ...models.rmes import ( DEFAULT_QUERY_TIMEOUT_SECONDS, DEFAULT_ROW_LIMIT, From e4db8f668d7bf32a7a57d40131124af3c6ec6720 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 17:27:42 +0200 Subject: [PATCH 39/55] refactor: move the rate-limit key resolver out of the package root rate_limiting.py was named after the feature that consumes the function rather than what the function does: it resolves the caller's address and knows nothing about windows or buckets. It is now utils/client_host.py, named for its subject, and the docstring says why the middleware context argument goes unused. The package root is left holding only what server.py needs to boot. --- src/mcpdiffusion/server.py | 2 +- src/mcpdiffusion/utils/__init__.py | 0 src/mcpdiffusion/{rate_limiting.py => utils/client_host.py} | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 src/mcpdiffusion/utils/__init__.py rename src/mcpdiffusion/{rate_limiting.py => utils/client_host.py} (82%) diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index 2f11d82..faa9219 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -12,9 +12,9 @@ from .instructions import build_instructions from .lifespan import build_lifespan from .logging import build_logging_config, configure_logging -from .rate_limiting import resolve_client_host from .settings import load_settings from .tools import register_tools +from .utils.client_host import resolve_client_host settings = load_settings() configure_logging(settings.log_level) diff --git a/src/mcpdiffusion/utils/__init__.py b/src/mcpdiffusion/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mcpdiffusion/rate_limiting.py b/src/mcpdiffusion/utils/client_host.py similarity index 82% rename from src/mcpdiffusion/rate_limiting.py rename to src/mcpdiffusion/utils/client_host.py index c8710f5..7903797 100644 --- a/src/mcpdiffusion/rate_limiting.py +++ b/src/mcpdiffusion/utils/client_host.py @@ -14,7 +14,8 @@ def resolve_client_host(_context: MiddlewareContext) -> str: """Rate-limit key. Only as trustworthy as `trusted_proxy_hosts`: widen that and a caller can forge it. All callers without a resolvable host share one bucket, so a transport that never carries an HTTP - request would rate-limit every client together. + request would rate-limit every client together. The middleware context is unused -- it is part of + the `get_client_id` signature, not something this resolver needs. """ try: client = get_http_request().client From c18d62ac0b66e3395997ab563c66621db289e3a4 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 17:32:15 +0200 Subject: [PATCH 40/55] refactor(config): name the insee.fr flag like the rest of the code enable_inseefr_tools was the only identifier using "inseefr". Five directories, all five tool names and every other setting already say "insee", so the flag was the outlier rather than the convention. A comment now carries what the longer name was reaching for: INSEE is the institution and owns all three sources, while this flag is only the website. ENABLE_INSEEFR_TOOLS becomes ENABLE_INSEE_TOOLS. No manifest sets it, so nothing breaks silently, but it belongs in the release notes. --- .env.example | 5 +++-- src/mcpdiffusion/instructions.py | 4 ++-- src/mcpdiffusion/lifespan/__init__.py | 4 ++-- src/mcpdiffusion/server.py | 2 +- src/mcpdiffusion/settings.py | 7 ++++--- src/mcpdiffusion/tools/__init__.py | 2 +- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 7214e9c..43fa920 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,8 @@ ALLOWED_ORIGINS=[] TRUSTED_PROXY_HOSTS=["127.0.0.1"] # Tool selection ------------------------------------------------------------------------------------------------------- -ENABLE_INSEEFR_TOOLS=true +# insee.fr, the website. MELODI and RMES are INSEE sources too; this is only the site. +ENABLE_INSEE_TOOLS=true ENABLE_MELODI_TOOLS=true ENABLE_RMES_TOOLS=true # Lets a client report a broken tool. Records to the server log; needs no backend. @@ -27,7 +28,7 @@ ENABLE_FEEDBACK_TOOL=true # Elasticsearch -------------------------------------------------------------------------------------------------------- # Inside Docker use the service name; on the host use localhost. -# Required only when ENABLE_INSEEFR_TOOLS or ENABLE_MELODI_TOOLS is true. +# Required only when ENABLE_INSEE_TOOLS or ENABLE_MELODI_TOOLS is true. ES_HOST=http://localhost:9200 ES_INDEX_PUBLICATIONS=produit ES_INDEX_MELODI_DATASETS=melodi_datasets diff --git a/src/mcpdiffusion/instructions.py b/src/mcpdiffusion/instructions.py index 14c0db3..41608d3 100644 --- a/src/mcpdiffusion/instructions.py +++ b/src/mcpdiffusion/instructions.py @@ -185,14 +185,14 @@ def build_instructions( - enable_inseefr_tools: bool, + enable_insee_tools: bool, enable_melodi_tools: bool, enable_rmes_tools: bool, enable_feedback_tool: bool, ) -> str: """Assemble the guidance for the tools this deployment actually registers.""" sections = [OVERVIEW, GLOBAL_RULES] - if enable_inseefr_tools: + if enable_insee_tools: sections.append(INSEE_SECTION) if enable_melodi_tools: sections.append(MELODI_SECTION) diff --git a/src/mcpdiffusion/lifespan/__init__.py b/src/mcpdiffusion/lifespan/__init__.py index c176f93..91a1ab0 100644 --- a/src/mcpdiffusion/lifespan/__init__.py +++ b/src/mcpdiffusion/lifespan/__init__.py @@ -31,7 +31,7 @@ async def app_lifespan(_server: FastMCP[Any]) -> AsyncIterator[dict[str, Any]]: # insee.fr and Melodi search the same index, so they share one client. The settings # validator guarantees a host whenever either is enabled, which is what makes `es_host` # non-None here and lets the client take a plain `str`. - es_host = settings.es_host if (settings.enable_inseefr_tools or settings.enable_melodi_tools) else None + es_host = settings.es_host if (settings.enable_insee_tools or settings.enable_melodi_tools) else None async with AsyncExitStack() as stack: context: dict[str, Any] = {} @@ -46,7 +46,7 @@ async def app_lifespan(_server: FastMCP[Any]) -> AsyncIterator[dict[str, Any]]: ) ) - if settings.enable_inseefr_tools: + if settings.enable_insee_tools: context |= await stack.enter_async_context( insee_lifespan( elasticsearch_client=elasticsearch_client, diff --git a/src/mcpdiffusion/server.py b/src/mcpdiffusion/server.py index faa9219..8f504b3 100644 --- a/src/mcpdiffusion/server.py +++ b/src/mcpdiffusion/server.py @@ -25,7 +25,7 @@ # Routing guidance, delivered in the handshake so it reaches the caller without relying on a # separate file being loaded. Built from the enabled families so it never names a missing tool. instructions=build_instructions( - enable_inseefr_tools=settings.enable_inseefr_tools, + enable_insee_tools=settings.enable_insee_tools, enable_melodi_tools=settings.enable_melodi_tools, enable_rmes_tools=settings.enable_rmes_tools, enable_feedback_tool=settings.enable_feedback_tool, diff --git a/src/mcpdiffusion/settings.py b/src/mcpdiffusion/settings.py index 1ce8221..0fa0ed6 100644 --- a/src/mcpdiffusion/settings.py +++ b/src/mcpdiffusion/settings.py @@ -19,7 +19,8 @@ class Settings(BaseSettings): trusted_proxy_hosts: list[str] = ["127.0.0.1"] # Tool selection --------------------------------------------------------------------------------------------------- - enable_inseefr_tools: bool = True + # insee.fr, the website. MELODI and RMES are INSEE sources too; this flag is only the site. + enable_insee_tools: bool = True enable_melodi_tools: bool = True enable_rmes_tools: bool = True # Reporting only: it records to the server log and needs no backend. @@ -69,10 +70,10 @@ class Settings(BaseSettings): @model_validator(mode="after") def require_elasticsearch_when_it_is_searched(self) -> "Settings": """Fail at startup rather than on the first search that needs a host.""" - if self.es_host is None and (self.enable_inseefr_tools or self.enable_melodi_tools): + if self.es_host is None and (self.enable_insee_tools or self.enable_melodi_tools): raise ValueError( "ES_HOST is required because the insee.fr or Melodi tools are enabled. " - "Set it, or disable those families with ENABLE_INSEEFR_TOOLS=false and " + "Set it, or disable those families with ENABLE_INSEE_TOOLS=false and " "ENABLE_MELODI_TOOLS=false." ) return self diff --git a/src/mcpdiffusion/tools/__init__.py b/src/mcpdiffusion/tools/__init__.py index d7abda4..e63a873 100644 --- a/src/mcpdiffusion/tools/__init__.py +++ b/src/mcpdiffusion/tools/__init__.py @@ -29,7 +29,7 @@ def register_tools(mcp: FastMCP, settings: Settings) -> None: """Register the enabled tools, handing each the settings it needs.""" - if settings.enable_inseefr_tools: + if settings.enable_insee_tools: mcp.add_tool(search_insee_documents) mcp.add_tool(get_insee_homepage) mcp.add_tool(get_insee_document) From 22e92a2361c6dc2ef7a0dea6dbd7977161a53528 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 17:49:04 +0200 Subject: [PATCH 41/55] fix(tools): tell the model the tool list beats the routing hints The guidance sections point at each other -- the Melodi entries route to search_insee_documents, the insee.fr ones to run_rmes_sparql -- so a deployment with a family disabled was telling the model to call tools that were never registered. A global rule now says the tool list is what exists, rather than trying to keep prose and configuration in step. The insee.fr bullet that routed to two sources in one sentence is split, one per source. --- src/mcpdiffusion/instructions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mcpdiffusion/instructions.py b/src/mcpdiffusion/instructions.py index 41608d3..c941581 100644 --- a/src/mcpdiffusion/instructions.py +++ b/src/mcpdiffusion/instructions.py @@ -24,6 +24,8 @@ discovery call first. - The data is French. Search with French keywords and rich synonyms. - An empty result is a valid answer, not a failure. It usually means the filters were too narrow. + - A routing hint may name a tool from another source. Only the tools in your tool list exist here; if a hint + names one you do not have, ignore it and use what you have. """ # language=Markdown @@ -44,8 +46,8 @@ - Cas simples : 'Quelle est la population de X ?', 'Taux de chomage en 2024 ?', 'Inflation en juillet 2026 ?' WHEN NOT TO USE - - Analyses detaillees, impacts/contexte, tendances complexes, donnees produit granulaires historiques - -> `search_melodi_datasets` ou `search_insee_documents` selon le contexte. + - Analyses detaillees, impacts/contexte, tendances complexes -> `search_insee_documents`. + - Donnees produit granulaires historiques -> `search_melodi_datasets`. ### `search_insee_documents` From 5fb6451ff20867c6d5dc2fd8d6aad449b4cc2a4d Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:20:48 +0200 Subject: [PATCH 42/55] fix(insee): cap how many documents one call may fetch get_insee_document took an unbounded list, and each entry costs a fetch of insee.fr plus a full extraction. A caller could ask for hundreds in one call and hold the request open while insee.fr absorbed the load. The bound sits in the schema rather than in a runtime check, so the model is told the limit instead of discovering it through an error. --- src/mcpdiffusion/models/insee.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 7aceb43..9a06fcd 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -16,6 +16,9 @@ # import time, before any Settings instance exists. DEFAULT_RESULT_COUNT = 10 MAX_RESULT_COUNT = 20 +# Each URL costs one fetch and one extraction, so a long list is a slow call and a load on +# insee.fr. Bounded here rather than checked in the service. +MAX_DOCUMENT_URLS = 10 # ---------------------------------------------------------------------------------------------------------------------- @@ -160,6 +163,7 @@ class ThemeConjonctureChoice(StrEnum): list[str], Field( description=("List of relative URLs to retrieve (e.g. '/fr/statistiques/4277658?sommaire=4318291')."), + max_length=MAX_DOCUMENT_URLS, examples=[ ["/fr/statistiques/4277658?sommaire=4318291"], ], From a2d48fbd2fea252d66f60f1f38525bb777f25cae Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:21:05 +0200 Subject: [PATCH 43/55] refactor(insee): let the data declare the schema instead of copying it ThemeChoice and ThemeConjonctureChoice listed by hand the same labels that the tables in data/insee/themes.py already hold, so a theme added to one was invisible to the other. Both are now derived from those tables, following what rmes already does with GraphCategoryChoice. The enum values are unchanged; only their order follows the tables now. The curated indicator entries gain a TypedDict naming their three keys. They stay plain dicts, because data/ holds plain data and the consuming layer is what turns it into typed objects. --- src/mcpdiffusion/data/insee/indicators.py | 14 ++++++- src/mcpdiffusion/models/insee.py | 49 ++++++++++------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/mcpdiffusion/data/insee/indicators.py b/src/mcpdiffusion/data/insee/indicators.py index 2da8d4c..7520658 100644 --- a/src/mcpdiffusion/data/insee/indicators.py +++ b/src/mcpdiffusion/data/insee/indicators.py @@ -1,5 +1,16 @@ """Curated INSEE key indicators (homepage data).""" +from typing import TypedDict + + +class KeyIndicatorEntry(TypedDict): + """One curated figure. The keys are the source data's, hence French.""" + + cle: str + alias: str + valeur: str + + # Business rule: these figures are frozen literals — nothing refreshes them, so the server reports whatever # was true when this file was last edited, while each sentence asserts its own date. Whether to fetch # insee.fr live, derive them from the Elasticsearch index, or keep a curated list with a visible @@ -8,8 +19,7 @@ # The tool description used to promise `mainIndicators` with a per-indicator link to pass to # `get_insee_document`, plus `lastArticles` and `keyGraphics`. None of that was ever produced. It is # recorded here because it says what the tool was meant to be, and is worth raising in that decision. -# Fixme: i feel this list can be typed, or at least the objects within -KEY_INDICATORS = [ +KEY_INDICATORS: list[KeyIndicatorEntry] = [ { "cle": "estimation de population France", "alias": "", diff --git a/src/mcpdiffusion/models/insee.py b/src/mcpdiffusion/models/insee.py index 9a06fcd..6af887d 100644 --- a/src/mcpdiffusion/models/insee.py +++ b/src/mcpdiffusion/models/insee.py @@ -2,11 +2,14 @@ from __future__ import annotations +import re from enum import StrEnum from typing import Annotated, Literal from pydantic import BaseModel, Field +from ..data.insee.themes import DICT_THEME_CONJ, KEYS_THEME_NIV1 + # ---------------------------------------------------------------------------------------------------------------------- # Schema bounds -------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------- @@ -26,20 +29,20 @@ # ---------------------------------------------------------------------------------------------------------------------- -# Fixme: a lot of static data from this file seems derived from the one in the 'data' package -# this could be merged / refactored / better exploited -class ThemeChoice(StrEnum): - ALL = "ALL" - METHODES = "Methodes" - DEMOGRAPHIE = "Demographie" - REVENUS = "Revenus - Pouvoir d'achat - Consommation" - CONDITIONS = "Conditions de vie - Societe" - TRAVAIL = "Marche du travail - Salaires" - ECONOMIE = "Economie - Conjoncture - Comptes nationaux" - DD = "Developpement durable - Environnement" - ENTREPRISES = "Entreprises" - SECTEURS = "Secteurs d'activite" - TERRITOIRES = "Territoires, villes et quartiers" +def build_enum_member_name(label: str) -> str: + """Turn a theme label into a usable member name. Only `ALL` is ever referenced by name.""" + return re.sub(r"\W+", "_", label).strip("_").upper() + + +# Derived from the data tables, so a theme cannot be offered to the caller without being searchable, +# nor searchable without being offered. "ALL" is not a theme: it means "do not filter". +ThemeChoice = StrEnum( + "ThemeChoice", + { + "ALL": "ALL", + **{build_enum_member_name(theme): theme for theme in KEYS_THEME_NIV1}, + }, +) class GeoLevelChoice(StrEnum): @@ -51,20 +54,10 @@ class GeoLevelChoice(StrEnum): FRANCE = "FRANCE" -class ThemeConjonctureChoice(StrEnum): - INDUSTRY = "Industrial production and activity" - BUILDING = "Construction and building sector" - HOUSING = "Housing and real estate" - RETAIL = "Retail, wholesale and services" - BUSINESS = "Business demographics and confidence" - EMPLOYMENT = "Employment, unemployment and labour market" - WAGES = "Wages and labour costs" - PUBLIC_SECTOR = "Public sector employment and pay" - CONSUMPTION = "Households, consumption and health" - PRICES = "Inflation and producer prices" - ACCOUNTING = "National accounts and public finance" - TRANSPORT = "Transport and tourism" - FINANCE = "Business financing" +ThemeConjonctureChoice = StrEnum( + "ThemeConjonctureChoice", + {build_enum_member_name(theme): theme for theme in DICT_THEME_CONJ}, +) # ---------------------------------------------------------------------------------------------------------------------- From 6d830f764113351add71241dc774bffdb556cad6 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:33:37 +0200 Subject: [PATCH 44/55] fix(rmes): reject URIs that could break out of the SPARQL query describe_rmes_resource interpolates the resource and graph URIs into `<...>`, so a URI carrying `>` closed the brackets and the rest of the string ran as query text. Both parameters are now checked against the IRI grammar, which forbids those characters anyway, so nothing legitimate is refused. Not an escalation -- run_rmes_sparql already accepts arbitrary SPARQL from the same caller. What it fixes is the diagnosis: a malformed URI now fails naming the parameter, instead of coming back as a syntax error from RMES. --- src/mcpdiffusion/models/rmes.py | 8 ++++++++ src/mcpdiffusion/services/rmes/graph_store_service.py | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/mcpdiffusion/models/rmes.py b/src/mcpdiffusion/models/rmes.py index 2aa79df..083ded6 100644 --- a/src/mcpdiffusion/models/rmes.py +++ b/src/mcpdiffusion/models/rmes.py @@ -79,10 +79,17 @@ # --- describe_rmes_resource --- +# Both URIs below are interpolated into `<...>` in a SPARQL query. The SPARQL grammar already +# forbids these characters inside an IRI, so rejecting them costs no legitimate value and stops a +# crafted URI from closing the brackets and continuing the query. The caller gets a schema error +# naming the parameter instead of a syntax error from RMES. +IRI_PATTERN = r'^[^<>"{}|^`\\\x00-\x20]+$' + ResourceUri = Annotated[ str, Field( description="URI complete de la ressource RDF a decrire.", + pattern=IRI_PATTERN, examples=[ "http://id.insee.fr/codes/naf2025/section/A", ], @@ -96,6 +103,7 @@ "URI d'un graphe nomme pour restreindre la recherche. Sans cette valeur (None par defaut), " "la recherche se fait sur tous les graphes (plus lent)." ), + pattern=IRI_PATTERN, ), ] diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py index d8cc928..befd9de 100644 --- a/src/mcpdiffusion/services/rmes/graph_store_service.py +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -108,8 +108,9 @@ def build_resource_query(resource_uri: str, graph_uri: str | None) -> str: """Ask for every triple where the resource appears, in either direction.""" graph_clause = f"<{graph_uri}>" if graph_uri else "?g" graph_values = f"VALUES ?g {{ <{graph_uri}> }}" if graph_uri else "" - # Fixme: the query is built using string interpolation - # just check whether injection can cause problems here + # Interpolated, not parameterised: SPARQL has no bind parameters for IRIs. Safe because both + # URIs are `pattern`-checked in models/rmes.py against the IRI grammar, so neither can carry + # the `>` that would close the brackets and let the rest run as query text. return f""" SELECT ?g ?direction ?p ?o WHERE {{ {graph_values} From f64436160fe6838cc05b704fe07e78dfd44b98dd Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:33:37 +0200 Subject: [PATCH 45/55] docs(insee): drop the Fixme asking for a cap that now exists MAX_DOCUMENT_URLS bounds the list in the schema, so the note asking for it was stale. The one below it opened with "on top of that" and lost its antecedent, and claimed the sequential fetching blocks the event loop -- awaiting in a loop is serial, not blocking. Reworded to say what is actually wrong with it. --- src/mcpdiffusion/services/insee/document_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index 6a0d229..0cd0791 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -195,8 +195,7 @@ async def fetch_documents( ) results: list[DocumentResult] = [] - # Fixme: there should be a cap in the number of URLs provided to avoid overloading the server - # Fixme: on top of that, the fetching is done sequentially, impacting the event loop + # Fixme: the URLs are fetched one after another, so the call takes the sum of their times for url in document_urls: try: html = await self.fetch_html(url) From 24acd77f483e5fab7ed05b4662ca033ad6ca2872 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:40:22 +0200 Subject: [PATCH 46/55] fix(rmes): cap a query whose only LIMIT belongs to a subquery The check looked for LIMIT anywhere in the query, so one inside a subquery -- or the word inside a string literal -- counted as the caller's own and no cap was added. The outer query then returned as many rows as the endpoint would give. A LIMIT that bounds the whole query is the last thing in it, with only OFFSET allowed to follow, so the check is anchored to the tail. This bounds what comes back, not what RMES computes: a subquery with no limit of its own is still evaluated in full upstream. The per-query timeout stays the only guard on that. --- src/mcpdiffusion/services/rmes/graph_store_service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mcpdiffusion/services/rmes/graph_store_service.py b/src/mcpdiffusion/services/rmes/graph_store_service.py index befd9de..b2c1e72 100644 --- a/src/mcpdiffusion/services/rmes/graph_store_service.py +++ b/src/mcpdiffusion/services/rmes/graph_store_service.py @@ -32,7 +32,10 @@ STRIP_PREFIX_PATTERN = re.compile(r"(?i)^\s*(PREFIX|BASE)\b.*$", re.MULTILINE) QUERY_FORM_PATTERN = re.compile(r"(?i)\b(SELECT|ASK|CONSTRUCT|DESCRIBE)\b") -LIMIT_PATTERN = re.compile(r"(?i)\bLIMIT\s+\d+\b") +# A LIMIT that bounds the whole query is the last thing in it -- OFFSET may follow or precede it, +# but nothing else does. Matching LIMIT anywhere counted one belonging to a subquery, or the word +# sitting in a string literal, and left the outer query unbounded. +TRAILING_LIMIT_PATTERN = re.compile(r"(?i)\bLIMIT\s+\d+\b(?:\s+OFFSET\s+\d+)?\s*;?\s*$") JSON_RESULT_FORMS = ("SELECT", "ASK") LIMITABLE_FORMS = ("SELECT", "CONSTRUCT") @@ -66,9 +69,7 @@ def ensure_row_limit(query: str, query_form: str, max_rows: int) -> tuple[str, b """Append a LIMIT when the caller supplied none, so an open query cannot flood the response.""" if query_form not in LIMITABLE_FORMS: return query, False - # Fixme: this is a particular case, but if there is inner queries with the word limit, - # nothing prevents outer queries from not being bound - if LIMIT_PATTERN.search(query): + if TRAILING_LIMIT_PATTERN.search(query.rstrip()): return query, False return query.rstrip().rstrip(";") + f"\nLIMIT {max_rows}", True From b5fa91d24dd00c0e23138e3c76c5ead1d68de3a8 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 18:48:19 +0200 Subject: [PATCH 47/55] fix(melodi): stop one malformed observation failing the whole year filter An observation whose `dimensions` was null crashed the filter: the `{}` fallback only applies when the key is absent, not when its value is null. That raised an AttributeError, which is not a ToolError, so the caller lost the entire batch to a generic internal error over a single row. Reading the year moves into its own function, total for every shape of missing period. The open question -- whether TIME_PERIOD is always a string, which we do not control -- stays marked, with a note that coercing it would hide a change in the upstream format rather than surface it. --- .../tools/melodi/get_observations_tool.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/mcpdiffusion/tools/melodi/get_observations_tool.py b/src/mcpdiffusion/tools/melodi/get_observations_tool.py index b3e02e5..612c688 100644 --- a/src/mcpdiffusion/tools/melodi/get_observations_tool.py +++ b/src/mcpdiffusion/tools/melodi/get_observations_tool.py @@ -17,6 +17,23 @@ from ...services.melodi.api_service import MelodiApiService +def read_observation_year(observation: dict[str, Any]) -> str: + """The year an observation covers, taken from the head of its TIME_PERIOD ("2023-01" -> "2023"). + + Returns "" when the observation carries no period at all, so a year filter drops it rather than + failing the whole call. + """ + dimensions = observation.get("dimensions") or {} + time_period = dimensions.get("TIME_PERIOD") + if time_period is None: + return "" + # Fixme: TIME_PERIOD is assumed to be a string. Every dataset checked returns one, but we do not + # control the format and have not seen them all, so a non-string raises here. Coercing with + # str() was considered and rejected: it would match nothing silently, turning a surprise in the + # upstream format into a wrong answer instead of a visible failure. + return time_period.split("-")[0] + + def keep_requested_years( observations: list[dict[str, Any]], years: list[int], @@ -28,12 +45,7 @@ def keep_requested_years( if not years: return observations requested_years = {str(year) for year in years} - return [ - observation - for observation in observations - # Fixme: 'TIME_PERIOD' could be sanitized - if (observation.get("dimensions", {}).get("TIME_PERIOD", "").split("-")[0]) in requested_years - ] + return [observation for observation in observations if read_observation_year(observation) in requested_years] async def get_melodi_observations( From abfbbd4394cbfb4e9db65f24502ff44a6082d241 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 19:01:30 +0200 Subject: [PATCH 48/55] perf(insee): fetch the requested documents concurrently A batch took the sum of its URLs: each was fetched only once the previous one had been rendered. They now go out together, bounded by MAX_DOCUMENT_URLS on the schema rather than a second limit of its own. Rendering one URL moves into its own method, which returns a failure as a result instead of raising, so one bad URL still costs the caller nothing but that entry and the order still matches the URLs given. --- .../services/insee/document_service.py | 108 ++++++++++-------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index 0cd0791..b0b8de1 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import logging from collections import defaultdict from urllib.parse import urljoin, urlparse @@ -194,54 +195,67 @@ async def fetch_documents( "document_urls must contain at least one URL. Use `search_insee_documents` to find URLs first.", ) - results: list[DocumentResult] = [] - # Fixme: the URLs are fetched one after another, so the call takes the sum of their times - for url in document_urls: - try: - html = await self.fetch_html(url) - markdown = extract(html, options=TRAFILATURA_OPTIONS) or "" - markdown, truncated = ( - truncate_markdown(markdown, limit=self._max_markdown_chars) - if truncate_content - else (markdown, False) - ) - - table_of_contents: TableOfContents | None = None - if include_table_of_contents: - entries = parse_table_of_contents( - html=html, - base_url=str(self._http_client.base_url), - ) - table_of_contents = group_table_of_contents(entries) if entries else None - - results.append( - DocumentResult( - id=url, - status="success", - markdown_content=markdown, - sommaire=table_of_contents, - truncated=truncated, - error=None, - ) - ) - except AppToolError as exc: - # A typed failure is written for the caller, so it is safe to pass on. - results.append( - build_failed_document( + # Bounded by MAX_DOCUMENT_URLS on the tool schema, so this fans out to at most that many + # requests. gather keeps the results in the order the URLs were given. + return list( + await asyncio.gather( + *( + self.fetch_document( url=url, - message=str(exc), - ) - ) - except Exception: - # Anything else is a bug: log it here, tell the caller only that this URL failed. - logger.exception("Unexpected failure fetching %s", url) - results.append( - build_failed_document( - url=url, - # Not raised, so the prefix an AppToolError would add is built here, - # from the same vocabulary rather than a hand-written literal. - message=f"[{ErrorCode.INTERNAL_ERROR}] Could not fetch this document.", + include_table_of_contents=include_table_of_contents, + truncate_content=truncate_content, ) + for url in document_urls ) + ) + ) + + async def fetch_document( + self, + url: str, + include_table_of_contents: bool, + truncate_content: bool, + ) -> DocumentResult: + """Render one URL, returning its failure as a result rather than raising. + + Every failure is reported in the same shape as a success, so one bad URL never costs the + caller the rest of the batch. + """ + try: + html = await self.fetch_html(url) + markdown = extract(html, options=TRAFILATURA_OPTIONS) or "" + markdown, truncated = ( + truncate_markdown(markdown, limit=self._max_markdown_chars) if truncate_content else (markdown, False) + ) - return results + table_of_contents: TableOfContents | None = None + if include_table_of_contents: + entries = parse_table_of_contents( + html=html, + base_url=str(self._http_client.base_url), + ) + table_of_contents = group_table_of_contents(entries) if entries else None + + return DocumentResult( + id=url, + status="success", + markdown_content=markdown, + sommaire=table_of_contents, + truncated=truncated, + error=None, + ) + except AppToolError as exc: + # A typed failure is written for the caller, so it is safe to pass on. + return build_failed_document( + url=url, + message=str(exc), + ) + except Exception: + # Anything else is a bug: log it here, tell the caller only that this URL failed. + logger.exception("Unexpected failure fetching %s", url) + return build_failed_document( + url=url, + # Not raised, so the prefix an AppToolError would add is built here, + # from the same vocabulary rather than a hand-written literal. + message=f"[{ErrorCode.INTERNAL_ERROR}] Could not fetch this document.", + ) From 7a2e9cd2e8898f1726f80ad615566366529986d5 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 19:01:46 +0200 Subject: [PATCH 49/55] fix(insee): stop document rendering blocking every other request Turning a page into markdown costs 80-1000 ms of CPU, and it ran on the event loop. During a batch the server served nothing else: a probe scheduled every 10 ms got zero turns until the whole batch finished. Rendering and table-of-contents parsing now run in a worker thread. The batch itself costs about 7% more wall-clock; in exchange the loop stays responsive, with a worst observed stall of 56 ms. Sharing TRAFILATURA_OPTIONS across threads was checked first: extract() never writes to it, and concurrent runs return byte-identical output to serial ones. --- src/mcpdiffusion/services/insee/document_service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mcpdiffusion/services/insee/document_service.py b/src/mcpdiffusion/services/insee/document_service.py index b0b8de1..182e8ba 100644 --- a/src/mcpdiffusion/services/insee/document_service.py +++ b/src/mcpdiffusion/services/insee/document_service.py @@ -223,14 +223,18 @@ async def fetch_document( """ try: html = await self.fetch_html(url) - markdown = extract(html, options=TRAFILATURA_OPTIONS) or "" + # Rendering a page costs 80-1000 ms of CPU. Left on the event loop it stalls every other + # request in flight, not just this one, so it runs in a worker thread. Sharing + # TRAFILATURA_OPTIONS across threads is safe: extract() only reads it. + markdown = await asyncio.to_thread(extract, html, options=TRAFILATURA_OPTIONS) or "" markdown, truncated = ( truncate_markdown(markdown, limit=self._max_markdown_chars) if truncate_content else (markdown, False) ) table_of_contents: TableOfContents | None = None if include_table_of_contents: - entries = parse_table_of_contents( + entries = await asyncio.to_thread( + parse_table_of_contents, html=html, base_url=str(self._http_client.base_url), ) From fbf14553f06f43990f8b03f71436835b40c732fb Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Tue, 8 Sep 2026 19:04:22 +0200 Subject: [PATCH 50/55] docs(tools): drop the typing note from one of three identical modules The return annotations are correct; what the note described -- that `ctx.lifespan_context` is an untyped mapping, so they are asserted rather than checked -- follows from the FastMCP lifespan contract and holds for the Melodi and RMES modules just the same. Only insee.py said so, which made the three read as though one of them were different. --- src/mcpdiffusion/dependencies/insee.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/mcpdiffusion/dependencies/insee.py b/src/mcpdiffusion/dependencies/insee.py index dbd2cde..2e34b80 100644 --- a/src/mcpdiffusion/dependencies/insee.py +++ b/src/mcpdiffusion/dependencies/insee.py @@ -6,9 +6,6 @@ from ..services.insee.document_service import InseeDocumentService from ..services.insee.index_service import InseeIndexService -# Fixme: these dependency functions do not provide proper typing which is a pity -- the lifespan -# context is an untyped mapping, so every return annotation below is asserted, never checked. - def get_insee_index_service(ctx: Context = CurrentContext()) -> InseeIndexService: """Return the insee.fr Elasticsearch service built at startup.""" From a8d45742e855093534adfbfc8c60162fc6bff37c Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Wed, 9 Sep 2026 01:49:39 +0200 Subject: [PATCH 51/55] docs(insee): mark the theme ids as a question for the data owners Two notes sat on this table. Whether static reference data belongs in the source tree was settled when data/ was reorganised by source, so that one goes. The other is real but not ours to answer: the ids are insee.fr's own, transcribed by hand, and nothing in this repo can check them. A wrong id searches the wrong theme without failing, which makes it a Business rule -- preserved, flagged, left to whoever owns the site's taxonomy. --- src/mcpdiffusion/data/insee/themes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mcpdiffusion/data/insee/themes.py b/src/mcpdiffusion/data/insee/themes.py index a43534b..5220628 100644 --- a/src/mcpdiffusion/data/insee/themes.py +++ b/src/mcpdiffusion/data/insee/themes.py @@ -1,8 +1,8 @@ """INSEE theme mappings and conjoncture sub-themes.""" -# Fixme: Is that the right place for this kind of data? In the source code? -# Maybe a JSON file or a database is a better place -# Fixme: This seems like mapping themes to IDs manually, this is fragile if so... +# Business rule: these ids are insee.fr's own, transcribed by hand, and nothing here can verify +# them. A wrong id silently searches the wrong theme rather than failing, so only whoever owns +# the site's taxonomy can confirm them or point at a feed to derive them from. Preserved as is. KEYS_THEME_NIV1 = { "Demographie": 0, "Conditions de vie - Societe": 6, From e35d9263fe884c2f28bf0335b8b0449e1c40dc10 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Wed, 9 Sep 2026 01:49:39 +0200 Subject: [PATCH 52/55] docs(melodi): record why the years are filtered here and not by the API "I wonder whether the API supports filtering" is now answered by testing it. It does filter, but `TIME_PERIOD=2025` matches only periods starting on that date: on DS_DECES_MORTALITE_SERIES, which holds annual and monthly rows together, it returns the yearly row and January while skipping August entirely. Moving the filter upstream would look like an obvious speed-up and would silently drop most of a monthly dataset, so fetching everything is deliberate rather than pending. The note on the value's type shrinks to the one thing that is not obvious from the code: str() is absent on purpose, because a coerced value would match nothing instead of failing. --- .../tools/melodi/get_observations_tool.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/mcpdiffusion/tools/melodi/get_observations_tool.py b/src/mcpdiffusion/tools/melodi/get_observations_tool.py index 612c688..be54c0c 100644 --- a/src/mcpdiffusion/tools/melodi/get_observations_tool.py +++ b/src/mcpdiffusion/tools/melodi/get_observations_tool.py @@ -27,10 +27,7 @@ def read_observation_year(observation: dict[str, Any]) -> str: time_period = dimensions.get("TIME_PERIOD") if time_period is None: return "" - # Fixme: TIME_PERIOD is assumed to be a string. Every dataset checked returns one, but we do not - # control the format and have not seen them all, so a non-string raises here. Coercing with - # str() was considered and rejected: it would match nothing silently, turning a surprise in the - # upstream format into a wrong answer instead of a visible failure. + # Deliberately not coerced with str(): a non-string should raise, not quietly match nothing. return time_period.split("-")[0] @@ -62,8 +59,10 @@ async def get_melodi_observations( list means no rows matched; a structured error means the upstream API failed or the inputs were invalid. """ - # Fixme: it seems we retrieve all the observations data and filter next - # I wonder whether the API supports filtering + # Business rule: fetching everything and filtering years here is deliberate. The API's own + # filter matches only periods starting on that date, so `TIME_PERIOD=2025` returns the + # yearly row and January but not August. Filtering upstream would silently drop most of a + # monthly dataset. Verified on DS_DECES_MORTALITE_SERIES, which holds both. observations = await melodi_api_service.fetch_observations( dataset_id=dataset_id, column_filters=column_filters, From 9508aa3ff7dc255c7a45bd79a8f3654fa22f0fac Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Wed, 9 Sep 2026 12:44:54 +0200 Subject: [PATCH 53/55] build(docker): harden the image build The build context carried the whole working tree, .venv and .git included, to the daemon on every build; a .dockerignore now excludes it. uv was pulled from the :latest tag, so the tool could change between two builds of the same commit. `uv sync` used --frozen, which installs a stale lock without complaint: a dependency added to pyproject.toml but never locked would be missing from the image and surface as an ImportError at runtime. --locked fails the build instead. The image also gains a healthcheck, so compose can wait for the server rather than guess, OCI labels tracing it back to this repository, and the venv on PATH so `python` means the right one when exec-ing into a container. Cache mounts keep uv's downloads between builds without storing them in the image. --- .dockerignore | 31 ++++++++++++++++++++++++++++ Dockerfile | 57 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6a42324 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Everything the image does not need. Without this the whole working tree -- .venv +# included -- is uploaded to the daemon on every build. + +# Virtual environments and caches +.venv/ +__pycache__/ +*.py[cod] +.ruff_cache/ +.pytest_cache/ +.mypy_cache/ + +# Version control and CI +.git/ +.github/ +.gitignore + +# Local configuration and secrets +*.env +!.env.example +.idea/ +.claude/ +.mcp.json + +# Not part of the runtime +tests/ +docs/ +k8s/ +*.md +Dockerfile* +docker-compose*.yml +.pre-commit-config.yaml diff --git a/Dockerfile b/Dockerfile index 5dffef0..a119d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,58 @@ -# ---- Build stage ---- +# Multi-stage: the builder installs the dependencies, the runtime keeps only the result. The uv +# binary and the download caches never reach the image that ships. + +# ---- Build stage ---------------------------------------------------------------------------------- FROM python:3.12-slim AS builder +# UV_LINK_MODE: uv hard-links from its cache into the venv, and hard links cannot cross filesystems. +# The cache mount below is a different mount, so uv would warn on every build. Copy instead. ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + UV_LINK_MODE=copy WORKDIR /app -# Install uv -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +# Pinned rather than :latest -- a build stage that changes under you is not reproducible. +COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /uvx /bin/ -# Copy dependency definition first for layer caching +# Dependencies before source: a one-line code edit must not invalidate the layer that installs them. COPY pyproject.toml uv.lock /app/ -# Install dependencies (no dev, no editable install) -RUN uv sync --no-dev --no-install-project --frozen +# The cache mount survives between builds without being stored in the image. +# --locked fails when uv.lock is not current for pyproject.toml. Plain `uv sync` would rewrite the +# lock and install versions nobody tested; --frozen would use a stale lock and silently omit a +# dependency that was added but never locked. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --no-dev --no-install-project --locked COPY src/ /app/src/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --no-dev --locked -# Install the project itself -RUN uv sync --no-dev --frozen - -# ---- Runtime stage ---- +# ---- Runtime stage -------------------------------------------------------------------------------- FROM python:3.12-slim +# CI passes the real version; `dev` is honest for a local build. +ARG APP_VERSION=dev + +LABEL org.opencontainers.image.title="McpDiffusion" \ + org.opencontainers.image.description="MCP server exposing INSEE public data to LLM clients" \ + org.opencontainers.image.source="https://github.com/InseeFrLab/McpDiffusion" \ + org.opencontainers.image.version="${APP_VERSION}" \ + org.opencontainers.image.licenses="Apache-2.0" + +# PATH: `python` means the venv's python everywhere -- CMD, HEALTHCHECK, and docker exec. ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + PATH="/app/.venv/bin:${PATH}" WORKDIR /app -# Utilisateur non privilegie (UID/GID fixes pour la coherence des volumes) +# Fixed UID/GID so a bind-mounted file keeps the same owner on the host. RUN groupadd --gid 1000 app \ && useradd --uid 1000 --gid 1000 --create-home --shell /usr/sbin/nologin app -# Copy only the virtual environment and source from the build stage +# --chown during the copy, never a later chown: that would duplicate every file into a new layer. COPY --from=builder --chown=app:app /app/.venv /app/.venv COPY --from=builder --chown=app:app /app/src /app/src @@ -40,7 +60,10 @@ USER app EXPOSE 8000 -# Default ES_HOST points at the Docker-compose service name; override when -# running the image standalone. +# A TCP connect, not an HTTP request: the MCP endpoint answers 400 without a session handshake, so an +# HTTP check would fail on a healthy server. The start period covers the lifespan opening its clients. +HEALTHCHECK --interval=15s --timeout=3s --start-period=20s --retries=3 \ + CMD python -c "import os, socket; socket.create_connection(('127.0.0.1', int(os.environ.get('MCP_PORT', 8000))), 2).close()" -CMD ["/app/.venv/bin/python", "-m", "mcpdiffusion.server"] +# Exec form: the process runs as PID 1 and receives SIGTERM, so shutdown is clean. +CMD ["python", "-m", "mcpdiffusion.server"] From 20e8456169e9c0b10462eb5c570aa3d8067e62bb Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Wed, 9 Sep 2026 12:45:11 +0200 Subject: [PATCH 54/55] build(docker): run the whole stack with one command Compose could not start the project. It referenced an image nobody built, ran no Elasticsearch though the insee.fr and Melodi tools require one, expected a network created by hand, and read an env file that does not exist. Anyone cloning the repo had no way to run it. It now builds the server, starts Elasticsearch with the INSEE indexes, restores the snapshot and waits for each step to be healthy before the next. The restore skips itself when the data is already there, so a restart costs seconds rather than a minute. Elasticsearch is built from the published amd64 image rather than run from it: the snapshot is copied into the official multi-arch base, so it runs native on arm64 and amd64 with no emulation. Setting path.repo there also removes the dependency on configuration baked into an image we do not own. ES_HOST defaults to the bundled service but yields to one set in .env, which is what lets an existing cluster be used with `up mcpdiffusion inspector`. The example env file no longer ships a value that would be wrong inside a container. --- .env.example | 15 ++++-- CLAUDE.md | 6 +-- Dockerfile.elasticsearch | 24 +++++++++ docker-compose-dev.yaml | 37 -------------- docker-compose.yml | 105 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 45 deletions(-) create mode 100644 Dockerfile.elasticsearch delete mode 100644 docker-compose-dev.yaml create mode 100644 docker-compose.yml diff --git a/.env.example b/.env.example index 43fa920..2ebc5b6 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,11 @@ +# Copy this file to .env before running anything: `cp .env.example .env`. Both the server and +# docker compose read it from there. +# # Every variable this server reads. ES_HOST is required unless the insee.fr and Melodi tools are # both disabled; the values shown are the defaults. # -# This file is resolved from the current working directory, not from this package. Docker and -# Kubernetes inject real environment variables instead and never read it. +# It is resolved from the current working directory, not from this package. docker compose reads +# it too; Kubernetes does not, and injects real environment variables instead. # HTTP server ---------------------------------------------------------------------------------------------------------- MCP_HOST=0.0.0.0 @@ -27,9 +30,11 @@ ENABLE_RMES_TOOLS=true ENABLE_FEEDBACK_TOOL=true # Elasticsearch -------------------------------------------------------------------------------------------------------- -# Inside Docker use the service name; on the host use localhost. -# Required only when ENABLE_INSEE_TOOLS or ENABLE_MELODI_TOOLS is true. -ES_HOST=http://localhost:9200 +# Required when ENABLE_INSEE_TOOLS or ENABLE_MELODI_TOOLS is true. +# Leave this commented out for `docker compose up`: the stack runs Elasticsearch and points the +# server at it. Set it to run the server outside Docker, or to aim the stack at a cluster you +# already have -- in which case start only `mcpdiffusion inspector` and skip the local one. +#ES_HOST=http://localhost:9200 ES_INDEX_PUBLICATIONS=produit ES_INDEX_MELODI_DATASETS=melodi_datasets ES_INDEX_MELODI_COLUMNS=melodi_columns diff --git a/CLAUDE.md b/CLAUDE.md index 25fd978..3a3da64 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,8 +66,8 @@ uv sync # install (dev deps included) uv run python -m mcpdiffusion.server # run the server locally, needs ES_HOST uv run pytest -q # test suite — do not trust it, see Hard rules -docker build -t mcp-insee . -docker compose -f docker-compose-dev.yaml up # server + MCP Inspector (needs the `elastic` network) +cp .env.example .env # once, before the first compose run +docker compose up --build # Elasticsearch + data + server + MCP Inspector ``` ## Hard rules @@ -87,7 +87,7 @@ docker compose -f docker-compose-dev.yaml up # server + MCP Inspector (needs th env file is the only description of what this server can be configured with — how the values actually reach the process (shell, compose `env_file`, k8s `env:`) does not change that. - **Keep every place that names a setting in sync**: the example env file, the env file - `docker-compose-dev.yaml` expects, and the `env:` block in `k8s/`. A variable set in a manifest that + `docker-compose.yml` expects, and the `env:` block in `k8s/`. A variable set in a manifest that `Settings` no longer reads is a bug, not leftovers. Touching `k8s/` for this is expected — it is the exception to the rule below. - Ask before adding a dependency, a new tool, or a new data source. diff --git a/Dockerfile.elasticsearch b/Dockerfile.elasticsearch new file mode 100644 index 0000000..63de65b --- /dev/null +++ b/Dockerfile.elasticsearch @@ -0,0 +1,24 @@ +# Elasticsearch carrying the INSEE snapshot, built for whichever architecture you are on. +# +# mirlon382/mcp_diffusion:db is published for linux/amd64 only. It is used here purely as a file +# source: COPY --from reads its layers and never executes it, so no emulation is involved. What +# actually runs is the official Elasticsearch image, which is multi-arch and therefore native on +# arm64 and amd64 alike. +# +# The snapshot is a backup of the three indexes, not the indexes themselves. Elasticsearch starts +# empty and only loads it once told to -- see the es-restore service in docker-compose.yml. + +FROM --platform=linux/amd64 mirlon382/mcp_diffusion:db AS snapshot + +FROM docker.elastic.co/elasticsearch/elasticsearch:9.3.2 + +# Declared once and reused below. It cannot be read back out of `path.repo`: the dot makes +# `$path.repo` expand as `${path}` followed by the text `.repo`, which would silently be wrong. +ARG BACKUPS_DIR=/usr/share/elasticsearch/backups + +# Where the restore step looks for the backup. Set here rather than in compose so the image is +# usable on its own, and so nothing depends on configuration baked into someone else's image. +ENV path.repo=${BACKUPS_DIR} + +# Elasticsearch runs as the `elasticsearch` user and cannot read root-owned files. +COPY --from=snapshot --chown=elasticsearch:root ${BACKUPS_DIR} ${BACKUPS_DIR} diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml deleted file mode 100644 index dd73c57..0000000 --- a/docker-compose-dev.yaml +++ /dev/null @@ -1,37 +0,0 @@ -services: - mcp-inspector: - image: ghcr.io/modelcontextprotocol/inspector:latest - container_name: mcp-inspector - ports: - - "6274:6274" # Web UI, à ouvrir dans le navigateur - - "6277:6277" # Serveur proxy interne de l'inspector (client MCP + API) - - #volumes: - # Conserve la liste des serveurs / config entre deux `docker compose up` - # - inspector-config:/home/node/.mcp-inspector - depends_on: - - mcpdiffusion - networks: - - elastic - - mcpdiffusion: - image: localhost/mcpdiffusion:1.0 - container_name: mcp-diffusion - ports: - - "8000:8000" - env_file: - - mcp-diffusion.env - #volumes: - # - ./feedback:/app/mcpdiffusion/feedback - networks: - - elastic - - - -networks: - elastic: - external: true #need to exist oc podman network create elastic - - -volumes: - inspector-config: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8f0e0d0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +# Local development stack: Elasticsearch with the INSEE data, the MCP server, and the Inspector. +# +# cp .env.example .env # once +# docker compose up --build # first run pulls ~1.3 GB of index data +# +# Then open the Inspector at http://localhost:6274 and connect it to: +# +# http://mcpdiffusion:8000/mcp (transport: Streamable HTTP) +# +# To work against an Elasticsearch you already have, put ES_HOST in your .env and start only the +# server, skipping the two Elasticsearch containers: +# +# docker compose up --build mcpdiffusion inspector + +services: + # Elasticsearch built from Dockerfile.elasticsearch, so it runs native on arm64 and amd64. + elasticsearch: + build: + context: . + dockerfile: Dockerfile.elasticsearch + image: mcpdiffusion-elasticsearch:local + environment: + discovery.type: single-node + xpack.security.enabled: "false" + ES_JAVA_OPTS: -Xms2g -Xmx2g + ports: + - "9200:9200" + healthcheck: + # A green or yellow cluster is ready to serve; a single node is yellow by design, since + # replicas have nowhere to go. + test: ["CMD-SHELL", "curl -sf 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=1s' || exit 1"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + + # Loads the snapshot into Elasticsearch, then exits. Runs on every `up`; restoring an index that + # is already there is refused by Elasticsearch, which the script treats as success. + es-restore: + image: curlimages/curl:8.5.0 + depends_on: + elasticsearch: + condition: service_healthy + restart: "no" + entrypoint: ["sh", "-c"] + command: + - | + set -eu + ES=http://elasticsearch:9200 + + echo "Registering the snapshot repository..." + curl -sf -X PUT "$$ES/_snapshot/insee_snapshots" \ + -H 'Content-Type: application/json' \ + -d '{"type":"fs","settings":{"location":"/usr/share/elasticsearch/backups"}}' > /dev/null + + SNAPSHOT=$$(curl -sf "$$ES/_snapshot/insee_snapshots/_all" \ + | sed -n 's/.*"snapshot":"\([^"]*\)".*/\1/p' | head -n1) + if [ -z "$$SNAPSHOT" ]; then + echo "No snapshot found in the repository." >&2 + exit 1 + fi + + if curl -sf "$$ES/produit" > /dev/null 2>&1; then + echo "Indexes already present, nothing to restore." + exit 0 + fi + + echo "Restoring snapshot $$SNAPSHOT..." + curl -sf -X POST "$$ES/_snapshot/insee_snapshots/$$SNAPSHOT/_restore?wait_for_completion=true" \ + -H 'Content-Type: application/json' -d '{}' > /dev/null + echo "Restore complete." + + mcpdiffusion: + build: + context: . + dockerfile: Dockerfile + image: mcpdiffusion:local + depends_on: + es-restore: + condition: service_completed_successfully + env_file: + - .env + environment: + # Listen on every interface: the container's loopback is not reachable from the host. + MCP_HOST: 0.0.0.0 + # Defaults to the service above, but an ES_HOST in your .env wins -- which is what lets you + # point at an existing Elasticsearch and skip these containers entirely. + ES_HOST: ${ES_HOST:-http://elasticsearch:9200} + ports: + - "8000:8000" + + # Official MCP debugging UI: lists the server's tools and calls them by hand, no LLM involved. + inspector: + image: ghcr.io/modelcontextprotocol/inspector:latest + depends_on: + mcpdiffusion: + condition: service_healthy + ports: + - "6274:6274" # web UI + - "6277:6277" # the proxy it talks to the server through + +volumes: + elasticsearch-data: From 5d497c8d2956ce16cabe34286a1026d768dc2986 Mon Sep 17 00:00:00 2001 From: Mamadou Diallo-Ext Date: Wed, 9 Sep 2026 12:53:32 +0200 Subject: [PATCH 55/55] docs: report what the refactor changed and what it left open Fifty-five commits are hard to review as a list. This groups them by what they address -- architecture, FastMCP adoption, bugs, security, configuration, performance, build -- and states the before and after for each. It also records what was deliberately not done: the eight questions left for whoever owns the data semantics, and the four known gaps, so neither reads later as something that was missed. --- docs/report.md | 191 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 docs/report.md diff --git a/docs/report.md b/docs/report.md new file mode 100644 index 0000000..a29e3d1 --- /dev/null +++ b/docs/report.md @@ -0,0 +1,191 @@ +# Refactor report — `feat/refacto-clean-archi-2` + +What changed between `main` and this branch, and why. + +**55 commits · 107 files · +6880 / −3897** + +| | before | after | +|---|---|---| +| `# Fixme:` markers | 131 (peak, after the code review) | **0** | +| `# Business rule:` markers | 0 | 8 — questions only the data owners can answer | +| Configuration variables | 7 | 31, all typed and documented | +| Python modules | 23, mostly flat | 58, grouped by responsibility | +| `docker compose up` | did not run at all | works from a clean clone | +| Linter | none | ruff, clean | + +--- + +## 1. Architecture + +The package was a flat `helpers/` drawer plus a `tools/` directory of prefixed files. Every module now +says what it holds. + +**Before** +``` +helpers/{es,es_search,rmes,schemas,logging}.py +middleware.py +tools/{insee_*,melodi_*,rmes_*,extras_send_feedback}.py +``` + +**After** +``` +tools// declare the MCP tools; wire, never compute +services// orchestration and business logic +models/ tool schemas and result types +data// static reference tables +lifespan/ builds the shared clients at startup +dependencies/ what a tool can be handed +errors/ the error contract and its translators +utils/, settings.py, instructions.py, logging.py, server.py +``` + +Key moves: + +- **`helpers/` dismantled** — each file went to a module named for its subject (`0c639b8`). +- **`core/`, `infra/`, `config/` deleted** as junk drawers; their contents live at the package root or in + a named package. +- **Static data grouped by source** (`b431717`) — `data/insee/`, `data/rmes/`. +- **One lifespan and one dependency module per source** (`bf8bca3`), so adding a source adds a file rather + than editing one. +- **The error contract became a package** (`4d68c1d`) — `errors/` holds the type and the Elasticsearch + translator, which had been sitting in `services/` despite orchestrating nothing. +- **The package root earned a membership rule**: it holds only what `server.py` needs to boot. + +## 2. FastMCP 4 adoption + +The version had been bumped without adopting the APIs. + +- **Dependency injection** replaced service-locator lookups in every tool (`efa39aa`, `8c71e83`, `b7d1e14`). + Tools declare what they need; `Depends` resolves it per request and hides it from the LLM schema. +- **Registration wrappers removed** — every tool is a plain function added with `mcp.add_tool`. +- **Built-in rate limiting** replaced a hand-rolled middleware, and the `limits` dependency went with it. +- **Built-in host protection** replaced hand-wired `TrustedHostMiddleware` (`3d21279`). +- **`mask_error_details=True`** so unexpected exceptions stop leaking their message to clients. +- **Tool descriptions moved into docstrings**, which FastMCP parses for both the tool and its parameters. + +## 3. Bugs fixed + +The substantive ones, each reproduced before being fixed. + +### Errors that never reached the caller + +- **Elasticsearch `ApiError` escaped the failure boundary entirely** in both insee and melodi — a missing + index, a 400, a 401/403 or a 5xx surfaced as an internal error with no guidance. `ApiError` does not + subclass `TransportError`; catching one never caught the other (`974cc5a`). +- **`search_insee_chiffrecle` reported the wrong backend** in its failure message — a copy-paste. +- **MELODI answers HTTP 400 for an unknown dataset, column and modality alike**; the message only ever + suggested checking modality codes. All three bodies were confirmed live, and the message now names every + remedy. + +### Silent data loss + +- **A query whose only `LIMIT` sat in a subquery went unbounded** (`24acd77`). The check matched `LIMIT` + anywhere, so a subquery's own limit — or the word inside a string literal — counted as the caller's. +- **One malformed observation failed the whole batch** (`b5fa91d`). An observation with a null `dimensions` + raised `AttributeError`, which is not a `ToolError`, so the caller lost every row over one. +- **The handshake instructions named tools that were not registered** (`22e92a2`). Sections cross-reference + each other, so disabling a family left the model being told to call tools that did not exist. +- **`send_feedback` had been silently dropped** (`310f19c`) — it was registered until a commit removed the + `else` branch that carried it. It was the only tool registered exclusively there, so nobody noticed, while + `SKILL.md` kept instructing clients to call it. + +### Concurrency and lifecycle + +- **A thundering-herd race on the RMES graph cache** — a module global with no lock. Replaced by instance + state with an `asyncio.Lock` and a double freshness check; verified 10 concurrent callers produce one + execution. +- **The server could not start in rmes-only mode** (`4a28e5b`) — `ES_HOST` was required unconditionally, + though only insee and melodi search Elasticsearch. +- **One tool's limits governed another's queries** (`f0b7f94`) — `run_rmes_sparql`'s schema bound was applied + inside the shared execute path, silently capping the graph listing at 60s regardless of its own documented + setting, and `describe_rmes_resource` drew its budget from a tool it does not expose. + +## 4. Security + +- **Every caller shared one rate-limit bucket** (`68d149c`). Behind a Kubernetes ingress, `TRUSTED_PROXY_HOSTS` + defaulted to `127.0.0.1`, so uvicorn ignored `X-Forwarded-For` and every client looked like the ingress. +- **The host guard approved everyone** (`3d21279`) — `allowed_hosts=["*"]` with `host_origin_protection="auto"`, + while `allowed_origins` was doing the rejecting and was not configurable. +- **The Elasticsearch host leaked to clients** in exception messages under `max_retries=2`, which is the + production setting. Only the exception type is quoted now; the full cause stays in the server log. +- **SPARQL injection through a resource URI** (`6d830f7`). `describe_rmes_resource` interpolated the URI into + `<...>`; a `>` closed the brackets and the rest ran as query text. Both parameters are now checked against + the IRI grammar, which forbids those characters anyway. +- **Unbounded input** — no cap on how many documents one call could request (`5fb6451`), and no length bound + on feedback. Both are schema bounds now, so the model is told the limit rather than discovering it. + +## 5. Error handling + +- **A single error type and a closed vocabulary** (`4ae8b78`). `ErrorCode` was a `Literal` — a promise to a + type checker that nothing enforced, so `AppToolError('TOTALLY_MADE_UP', ...)` was accepted silently. It is + a `StrEnum` now; a typo fails at the reference. +- **`UNKNOWN` became `INTERNAL_ERROR`** — the former advertised poor error handling rather than naming a fault. +- **Translation happens once, at the boundary owning the dependency**, in `errors/`. +- **Error messages are the contract**: every one names the backend, the failure, and the next step. Message + text was diffed byte-for-byte across the refactor so the contract did not drift. + +## 6. Configuration + +- **7 variables became 31**, all typed in `settings.py`, all documented in `.env.example`, and verified in + sync both directions. +- **No magic numbers left in the code paths that matter** — timeouts, retries, index names, budgets and + limits are settings. +- **Settings fail at startup, not on first use** — a validator rejects a configuration that cannot work. +- **Schema bounds stayed out of settings deliberately** (`9e125ac`). They bound the published tool schema, so + an env-driven value would advertise a different contract per deployment. +- **Renames pending release notes**: `ES_INDEX_PRODUITS` → `ES_INDEX_PUBLICATIONS`, + `RMES_ENDPOINT` → `RMES_SPARQL_ENDPOINT_URL`, `ENABLE_INSEEFR_TOOLS` → `ENABLE_INSEE_TOOLS`. + +## 7. Performance + +- **Documents are fetched concurrently** (`abfbbd4`) — a batch took the sum of its URLs; five 0.3s fetches + went from 1.50s to 0.32s. +- **Rendering moved off the event loop** (`7a2e9cd`). Turning a page into markdown costs 80–1000 ms of CPU, + and it ran on the loop: during a batch the server served *nothing else* — a probe scheduled every 10 ms got + zero turns. Now a worker thread; the batch costs ~7% more, the server stays responsive. + +## 8. Build and tooling + +- **ruff adopted** for linting and formatting (`6030832`), with `FBT003` guarding the call-site convention + that replaced keyword-only markers (`26bdf59`). +- **The image build hardened** (`9508aa3`) — a `.dockerignore` (the build context had been shipping `.venv` + and `.git`), uv pinned instead of `:latest`, `--locked` instead of `--frozen` so a stale lock fails the + build rather than silently omitting a dependency, plus a healthcheck, OCI labels and cache mounts. +- **The whole stack runs with one command** (`20e8456`). Compose previously referenced an image nobody built, + ran no Elasticsearch though the tools require one, expected a hand-created network, and read an env file + that does not exist. It now builds the server, starts Elasticsearch with the INSEE indexes, restores the + snapshot, and waits for each step before the next. +- **Elasticsearch runs native on arm64 and amd64** — the published image is amd64-only, so the snapshot is + copied into the official multi-arch base rather than the image being run under emulation. + +## 9. Conventions + +- **Tools, parameters and schemas named after what they are** (`cdf44df`) — a breaking rename, done once. +- **English throughout the code**; French remains only where it is data (insee.fr CSS classes, RMES messages). +- **A convention file per concern** under `.claude/rules/` — python, errors, logging, git — updated whenever a + decision contradicted them, so the harness and the code agree. +- **Attribution forbidden in commit messages**, and existing trailers stripped from history (`396ea7e`). + +--- + +## Deliberately left open + +Not oversights. Each is recorded in the code where it matters. + +**8 `# Business rule:` markers** — questions only whoever owns the search and data semantics can answer: + +- Homepage indicators are frozen literals; nothing refreshes them. +- insee.fr theme ids are transcribed by hand and nothing here can verify them. +- An unrecognised geo level, theme or subtheme drops its filter silently and broadens the search. +- MELODI observations are fetched whole and filtered locally: the API's own year filter matches only periods + *starting* on that date, so it returns the annual row and January but not August. Filtering upstream would + silently lose most of a monthly dataset — verified on `DS_DECES_MORTALITE_SERIES`. + +**Known gaps** + +- `k8s/3_mcp_deploy.yaml` declares no readiness or liveness probe. Kubernetes ignores Docker's `HEALTHCHECK`, + so the pod is considered ready before the lifespan has opened its clients. +- No Elasticsearch credentials are supported — only host, TLS verification, timeouts and retries. Fine for the + current deployment; a secured cluster would need a settings addition. +- `README.md` and `SKILL.md` describe the pre-refactor code and are scheduled for regeneration. +- The test suite does not collect; it is auto-generated, unreviewed, and slated for a single pass of its own.