Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions tests/test_raw_run_format_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

from pathlib import Path

import pytest

from toolkit.raw._fetch_utils import _format_args
Expand All @@ -16,6 +18,22 @@ def test_format_args_simple_year_substitution(self) -> None:
result = _format_args(args, 2023)
assert result["url"] == "https://example.com/data2023.csv"

def test_format_args_path_year_substitution(self) -> None:
"""local_file path normalized to Path still substitutes {year}.

Regression: _normalize_paths converts args.path to a PosixPath; the
{year} placeholder must be replaced on Path values too.
"""
args = {"path": Path("/repo/_local/seed/dati/{year}/ETA_{year}.CSV")}
result = _format_args(args, 2024)
assert result["path"] == Path("/repo/_local/seed/dati/2024/ETA_2024.CSV")

def test_format_args_path_without_year_untouched(self) -> None:
"""Path without {year} is returned unchanged (same type)."""
args = {"path": Path("/repo/anagrafica/_data/CompartoContratto.CSV")}
result = _format_args(args, 2024)
assert result["path"] == Path("/repo/anagrafica/_data/CompartoContratto.CSV")

def test_format_args_no_url_suffix_by_year(self) -> None:
"""Without url_suffix_by_year, output is unchanged."""
args = {"url": "https://example.com/data{year}.csv", "other": "value"}
Expand Down
17 changes: 14 additions & 3 deletions toolkit/raw/_fetch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

from toolkit.core.exceptions import DownloadError
Expand All @@ -22,9 +23,17 @@


def _format_args(args: dict, year: int) -> dict:
formatted = {}
formatted: dict[str, Any] = {}
for k, v in (args or {}).items():
if isinstance(v, str) and "{year}" in v:
if isinstance(v, Path):
# _normalize_paths converte i path relativi in PosixPath: il
# placeholder {year} va sostituito anche su valori Path (es.
# raw.sources[].args.path per local_file).
if "{year}" in str(v):
formatted[k] = Path(str(v).replace("{year}", str(year)))
else:
formatted[k] = v
elif isinstance(v, str) and "{year}" in v:
# replace instead of str.format to avoid conflicts with SPARQL {} braces
formatted[k] = v.replace("{year}", str(year))
else:
Expand All @@ -35,7 +44,9 @@ def _format_args(args: dict, year: int) -> dict:
if isinstance(suffix_map, dict):
suffix = suffix_map.get(year, "")
if isinstance(suffix, str):
formatted["url"] = formatted["url"] + suffix
# url può essere Path (local_file): normalizza a str per il suffix
base_url = str(formatted["url"])
formatted["url"] = base_url + suffix
# Remove url_suffix_by_year from output — internal config, not for consumers
formatted.pop("url_suffix_by_year", None)
return formatted
Expand Down
Loading