From a96286b20fb761cda277732651c213fadcd644a9 Mon Sep 17 00:00:00 2001 From: Mike Odnis Date: Fri, 25 Sep 2026 04:53:31 -0400 Subject: [PATCH 1/2] fix(changelog): verify versions against registries before publishing them build_changelog.py opens with the claim that version tables "can never drift from what was actually published", but it derives every version from a git tag and never checks whether a package by that name and version exists. Tags are not proof of publication, so the changelog has been advertising releases that were never pushed. CodeRabbit caught two on #127: resq-mcp 0.1.0 PyPI has no such version (earliest is 0.3.1) .NET 0.2.0 NuGet has no such version (earliest is 0.3.0) Checking every derived version against its registry turns up far more. Seven Rust crate names in the tables do not exist on crates.io at all: resq-bin-explorer resq-cleanup resq-deploy-cli resq-flamegraph resq-health-checker resq-log-viewer resq-perf-monitor plus resq-deploy, resq-health and resq-logs (404), and resq-mcp 1.3.4 (latest is 1.2.2). Those rows told readers to install crates that `cargo add` cannot resolve. Each derived (package, version) is now checked against npm, PyPI, crates.io or NuGet and dropped if the registry says it was never published, with the reason logged to stderr. Two deliberate safety properties: - Unreachable registry fails OPEN. A network error returns None and the entry is kept, so a transient outage cannot silently erase real releases from published history. Only a definitive 404 drops a row. - vcpkg is exempt rather than dropped: it is a port collection with no version registry to query, so there is nothing to verify against. Verification is skipped automatically under --releases-file, which exists for offline runs, and can be disabled with --no-verify-registry. Verified: the predicate returns False for resq-mcp 0.1.0 and .NET 0.2.0 and True for resq-mcp 0.3.1, .NET 0.4.2 and @resq-systems/ui 0.42.0; every dropped crate name was confirmed 404 on crates.io by hand while resq-bin, resq-clean, resq-flame and resq-perf return 200 and are kept. Regenerated output re-runs clean under --check. This supersedes #127, whose generated content includes the two invalid entries above. Note for follow-up: the hand-written rename table in automation/changelog-notes.md maps resq-deploy-cli -> resq-deploy and similar, but neither side of three of those pairs was ever published. That text is outside the AUTOGEN region and is not touched here. Co-Authored-By: Claude Opus 5 (1M context) --- changelog.mdx | 50 ++++++++++++------ scripts/build_changelog.py | 105 ++++++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 19 deletions(-) diff --git a/changelog.mdx b/changelog.mdx index 24d0a20b..b97a2609 100644 --- a/changelog.mdx +++ b/changelog.mdx @@ -16,6 +16,29 @@ in `automation/changelog-notes.md`. {/* CHANGELOG:AUTOGEN:START — edit scripts/build_changelog.py or automation/changelog-notes.md, not this region */} + + ## TypeScript + + | Package | Version | + |---------|---------| + | `@resq-systems/map` | 4.0.0 | + | `@resq-systems/email-templates` | 1.2.0 | + | `@resq-systems/ui` | 0.42.0 | + | `@resq-systems/constants` | 0.7.0 | + | `@resq-systems/nav` | 0.2.0 | + + ## Rust + + | Package | Version | + |---------|---------| + | `resq-cli` | 0.4.7 | + | `resq-bin`, `resq-clean`, `resq-flame` | 0.1.26 | + | `resq-perf` | 0.1.22 | + | `resq-tui` | 0.1.18 | + | `resq-dsa` | 0.1.4 | + | `resq-ai` | 0.1.3 | + + ## TypeScript @@ -25,8 +48,9 @@ in `automation/changelog-notes.md`. | `@resq-systems/security` | 2.1.0 | | `@resq-systems/analytics`, `@resq-systems/http` | 2.0.2 | | `@resq-systems/map` | 2.0.0 | + | `@resq-systems/email-templates` | 1.1.0 | | `@resq-systems/ui` | 0.41.1 | - | `@resq-systems/helpers` | 0.6.0 | + | `@resq-systems/constants`, `@resq-systems/helpers` | 0.6.0 | | `@resq-systems/rate-limiting` | 0.4.3 | | `@resq-systems/telemetry` | 0.3.0 | | `@resq-systems/types` | 0.2.0 | @@ -35,9 +59,9 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| - | `resq-cli` | 0.4.0 | - | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.19 | - | `resq-tui` | 0.1.11 | + | `resq-cli` | 0.4.1 | + | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.20 | + | `resq-tui` | 0.1.12 | | `resq-ai` | 0.1.1 | @@ -71,7 +95,7 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| - | `resq-bin`, `resq-clean`, `resq-deploy`, `resq-flame`, `resq-health`, `resq-logs`, `resq-perf` | 0.1.18 | + | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.18 | | `resq-tui` | 0.1.10 | @@ -80,7 +104,7 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| - | `resq-bin`, `resq-clean`, `resq-deploy`, `resq-flame`, `resq-health`, `resq-logs`, `resq-perf` | 0.1.17 | + | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.17 | | `resq-tui` | 0.1.9 | @@ -96,12 +120,12 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| - | `resq-bin`, `resq-clean`, `resq-deploy`, `resq-flame`, `resq-health`, `resq-logs`, `resq-perf` | 0.1.16 | + | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.16 | | `resq-tui` | 0.1.8 | | `resq-dsa` | 0.1.3 | - + **Rust CLI tools renamed.** The seven CLI tools were renamed on 3 April 2026. The old crate names stopped at `0.1.9` and are no longer published; the new names continue the version series. @@ -122,19 +146,12 @@ in `automation/changelog-notes.md`. |---------|---------| | `@resq-sw/logger`, `@resq-sw/rate-limiting` | 0.1.1 | - ## Python - - | Package | Version | - |---------|---------| - | `resq-mcp` | 1.3.4 | - ## Rust | Package | Version | |---------|---------| | `resq-cli` | 0.2.6 | - | `resq-bin`, `resq-clean`, `resq-deploy`, `resq-flame`, `resq-health`, `resq-logs`, `resq-perf` | 0.1.15 | - | `resq-bin-explorer`, `resq-cleanup`, `resq-deploy-cli`, `resq-flamegraph`, `resq-health-checker`, `resq-log-viewer`, `resq-perf-monitor` | 0.1.10 | + | `resq-bin`, `resq-clean`, `resq-flame`, `resq-perf` | 0.1.15 | | `resq-tui` | 0.1.7 | | `resq-dsa` | 0.1.2 | | `resq-ai` | 0.1.0 | @@ -158,7 +175,6 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| | `resq-cli` | 0.2.3 | - | `resq-bin-explorer`, `resq-cleanup`, `resq-deploy-cli`, `resq-flamegraph`, `resq-health-checker`, `resq-log-viewer`, `resq-perf-monitor` | 0.1.8 | | `resq-tui` | 0.1.4 | | `resq-dsa` | 0.1.1 | diff --git a/scripts/build_changelog.py b/scripts/build_changelog.py index b64703f0..8ad9d3fb 100644 --- a/scripts/build_changelog.py +++ b/scripts/build_changelog.py @@ -25,6 +25,9 @@ import re import subprocess import sys +import urllib.error +import urllib.parse +import urllib.request # repo -> (sidebar label/tag, how tags encode the package name, single-pkg name) # "scoped" -> @scope/name@version (npm) @@ -40,6 +43,24 @@ } ECOSYSTEM_ORDER = ["npm", "pypi", "crates", "dotnet-sdk", "vcpkg"] +# repo -> (registry, package to query for "single"-mode repos) +# +# Tags are not proof of publication. A repo-level tag like v0.1.0 in a +# "single"-mode repo parses as a package release even when no such package +# version was ever pushed, which is how resq-mcp 0.1.0 and .NET 0.2.0 reached +# the changelog: neither exists (PyPI starts at 0.3.1, NuGet at 0.3.0). Every +# derived version is checked against the registry before it is rendered. +# +# vcpkg is a port collection with no version registry to query, so it is +# exempt rather than silently dropped. +REGISTRY = { + "npm": ("npm", None), + "pypi": ("pypi", "resq-mcp"), + "crates": ("crates", None), + "dotnet-sdk": ("nuget", "ResQ.Core"), + "vcpkg": (None, None), +} + # Scope renames: a package under the old scope is dropped from a month's table # only when its counterpart under the new scope shipped that same month. This # hides the abandoned-scope tail (e.g. @resq-sw/http's final release in July, @@ -139,7 +160,78 @@ def load_releases(releases_file: str | None) -> list[dict]: return out -def latest_by_month(releases: list[dict]) -> dict[str, dict[str, dict[str, str]]]: +_PUBLISHED: dict[tuple[str, str], set[str] | None] = {} + + +def _fetch(url: str) -> dict | None: + """Return parsed JSON, {} for 404 (absent), or None if the lookup failed.""" + req = urllib.request.Request(url, headers={"User-Agent": "resq-docs-changelog"}) + try: + with urllib.request.urlopen(req, timeout=20) as r: + return json.loads(r.read().decode("utf-8")) + except urllib.error.HTTPError as e: + if e.code == 404: + return {} + print(f"registry lookup failed ({e.code}): {url}", file=sys.stderr) + return None + except Exception as e: # network/DNS/timeout + print(f"registry lookup failed ({e}): {url}", file=sys.stderr) + return None + + +def published_versions(registry: str, pkg: str) -> set[str] | None: + """Versions actually published for pkg, or None if the registry was unreachable.""" + key = (registry, pkg) + if key in _PUBLISHED: + return _PUBLISHED[key] + if registry == "npm": + d = _fetch(f"https://registry.npmjs.org/{urllib.parse.quote(pkg, safe='@')}") + out = set((d or {}).get("versions", {})) if d is not None else None + elif registry == "pypi": + d = _fetch(f"https://pypi.org/pypi/{urllib.parse.quote(pkg)}/json") + out = set((d or {}).get("releases", {})) if d is not None else None + elif registry == "crates": + d = _fetch(f"https://crates.io/api/v1/crates/{urllib.parse.quote(pkg)}") + out = {v["num"] for v in (d or {}).get("versions", [])} if d is not None else None + elif registry == "nuget": + d = _fetch( + f"https://api.nuget.org/v3-flatcontainer/{urllib.parse.quote(pkg.lower())}/index.json" + ) + out = set((d or {}).get("versions", [])) if d is not None else None + else: + out = None + _PUBLISHED[key] = out + return out + + +def is_published(repo: str, pkg: str | None, ver: str) -> bool: + """True unless the registry positively says this version was never published. + + Unreachable registries and registry-less ecosystems fail open: a transient + network error must not silently erase real releases from the changelog. + """ + registry, probe = REGISTRY.get(repo, (None, None)) + if registry is None: + return True + target = pkg or probe + if not target: + return True + known = published_versions(registry, target) + if known is None: + return True + if ver in known: + return True + print( + f"dropping unpublished {repo} release: {target} {ver} " + f"(not in {registry})", + file=sys.stderr, + ) + return False + + +def latest_by_month( + releases: list[dict], verify: bool = True +) -> dict[str, dict[str, dict[str, str]]]: """month 'YYYY-MM' -> repo -> package -> highest version released that month.""" tree: dict[str, dict[str, dict[str, str]]] = {} for rel in releases: @@ -150,6 +242,8 @@ def latest_by_month(releases: list[dict]) -> dict[str, dict[str, dict[str, str]] if not parsed: continue pkg, ver = parsed + if verify and not is_published(repo, pkg, ver): + continue month = rel["published_at"][:7] by_pkg = tree.setdefault(month, {}).setdefault(repo, {}) if pkg not in by_pkg or version_key(ver) > version_key(by_pkg[pkg]): @@ -254,11 +348,18 @@ def main() -> int: parser.add_argument( "--check", action="store_true", help="exit 1 if changelog.mdx is not up to date" ) + parser.add_argument( + "--no-verify-registry", + action="store_true", + help="skip checking each version against its package registry " + "(implied by --releases-file, which is for offline runs)", + ) args = parser.parse_args() root = args.root.resolve() releases = load_releases(args.releases_file) - tree = latest_by_month(releases) + verify = not (args.no_verify_registry or args.releases_file) + tree = latest_by_month(releases, verify=verify) notes = load_notes(root) region = render(tree, notes) From ed9bf7b6d44c06f5a0d4bb2df50f50fe24d329a5 Mon Sep 17 00:00:00 2001 From: WomB0ComB0 Date: Tue, 29 Sep 2026 19:08:42 -0400 Subject: [PATCH 2/2] fix(changelog): confirm NuGet versions directly before dropping + normalize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit is_published dropped a release whenever its version was missing from the NuGet flat-container index, but that index normalizes versions and can lag a just-published version during propagation (NuGet/NuGetGallery#3455) — so a valid release could be erased on a cosmetic mismatch or a transient miss. Now: match on the NuGet-normalized form (lowercase, drop zero revision/leading zeros), and if still absent HEAD the exact .nupkg to confirm; only a positive 404 drops it, anything inconclusive fails open (consistent with the unreachable-registry policy). Regenerated changelog.mdx (adds @resq-systems/analytics 3.1.0). --- changelog.mdx | 1 + scripts/build_changelog.py | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/changelog.mdx b/changelog.mdx index b97a2609..72d4a464 100644 --- a/changelog.mdx +++ b/changelog.mdx @@ -22,6 +22,7 @@ in `automation/changelog-notes.md`. | Package | Version | |---------|---------| | `@resq-systems/map` | 4.0.0 | + | `@resq-systems/analytics` | 3.1.0 | | `@resq-systems/email-templates` | 1.2.0 | | `@resq-systems/ui` | 0.42.0 | | `@resq-systems/constants` | 0.7.0 | diff --git a/scripts/build_changelog.py b/scripts/build_changelog.py index 8ad9d3fb..57db070d 100644 --- a/scripts/build_changelog.py +++ b/scripts/build_changelog.py @@ -204,6 +204,42 @@ def published_versions(registry: str, pkg: str) -> set[str] | None: return out +def _normalize_nuget_version(v: str) -> str: + """NuGet normalizes versions: lowercased, leading zeros stripped, and a zero-valued + fourth (revision) segment dropped. Comparing in this form keeps a tag like `0.6.0.0` + matching the index's `0.6.0`, so a purely cosmetic difference never looks 'unpublished'. + """ + v = v.strip().lower() + core, _, pre = v.partition("-") + parts = core.split(".") + while len(parts) > 3 and parts[-1] in ("0", ""): + parts.pop() + parts = [str(int(p)) if p.isdigit() else p for p in parts] + core = ".".join(parts) + return f"{core}-{pre}" if pre else core + + +def _nuget_version_exists(pkg: str, ver: str) -> bool | None: + """Confirm one NuGet version directly. The flat-container index can lag behind a + just-published version during propagation (NuGet/NuGetGallery#3455), so an index-only + miss is not proof of absence. Returns True (present), False (a 404 confirms absent), or + None (inconclusive -> the caller should fail open). + """ + p = urllib.parse.quote(pkg.lower()) + v = urllib.parse.quote(_normalize_nuget_version(ver)) + url = f"https://api.nuget.org/v3-flatcontainer/{p}/{v}/{p}.{v}.nupkg" + req = urllib.request.Request( + url, method="HEAD", headers={"User-Agent": "resq-docs-changelog"} + ) + try: + with urllib.request.urlopen(req, timeout=20): + return True + except urllib.error.HTTPError as e: + return False if e.code == 404 else None + except Exception: # network/DNS/timeout + return None + + def is_published(repo: str, pkg: str | None, ver: str) -> bool: """True unless the registry positively says this version was never published. @@ -221,6 +257,17 @@ def is_published(repo: str, pkg: str | None, ver: str) -> bool: return True if ver in known: return True + if registry == "nuget": + # NuGet normalizes versions, and its flat-container index can lag behind a + # just-published version during propagation, so neither a formatting + # difference nor an index-only miss proves the release never shipped. Match + # on the normalized form, then, if still absent, confirm the exact version + # directly before dropping it. + norm = _normalize_nuget_version(ver) + if any(_normalize_nuget_version(k) == norm for k in known): + return True + if _nuget_version_exists(target, ver) is not False: + return True print( f"dropping unpublished {repo} release: {target} {ver} " f"(not in {registry})",