From 738caa0bcb5f3cd9ff30be299f48e56a34e8b947 Mon Sep 17 00:00:00 2001 From: Tako Schotanus Date: Wed, 12 Aug 2026 12:35:03 +0200 Subject: [PATCH 1/2] chore: 403s are now also considered pruneable --- .../jdkdb/scraper/DefaultDownloadManager.java | 17 +++++++++-------- .../jbang/jdkdb/util/HttpStatusException.java | 16 ++++++++++++++++ .../java/dev/jbang/jdkdb/util/HttpUtils.java | 13 ------------- 3 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java diff --git a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java index d6cb932..8fd0e4c 100644 --- a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java +++ b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java @@ -3,6 +3,7 @@ import dev.jbang.jdkdb.model.JdkMetadata; import dev.jbang.jdkdb.util.ArchiveUtils; import dev.jbang.jdkdb.util.HashUtils; +import dev.jbang.jdkdb.util.HttpStatusException; import dev.jbang.jdkdb.util.HttpUtils; import dev.jbang.jdkdb.util.MetadataUtils; import java.io.*; @@ -332,10 +333,10 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE try { httpUtils.downloadFile(url, tempFile); } catch (IOException e) { - if (unlistedSince.isPresent() && isHttp404(e)) { + if (unlistedSince.isPresent() && isHttpStatus(e, 403, 404)) { task.downloadLogger() .warn( - "Download returned 404 for unlisted package {} (unlisted_since={}). " + "Download returned 40X for unlisted package {} (unlisted_since={}). " + "This package is most likely not available anymore and is a candidate for pruning.", filename, unlistedSince.get()); @@ -418,14 +419,14 @@ private Optional findUnlistedSince(JdkMetadata metadata) { return Optional.of(value); } - private boolean isHttp404(IOException exception) { - Throwable current = exception; - while (current != null) { - String message = current.getMessage(); - if (message != null && message.contains("HTTP status: 404")) { + private boolean isHttpStatus(IOException e, int... statusCodes) { + if (!(e instanceof HttpStatusException hse)) { + return false; + } + for (int code : statusCodes) { + if (hse.getStatusCode() == code) { return true; } - current = current.getCause(); } return false; } diff --git a/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java b/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java new file mode 100644 index 0000000..378fbdb --- /dev/null +++ b/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java @@ -0,0 +1,16 @@ +package dev.jbang.jdkdb.util; + +import java.io.IOException; + +public class HttpStatusException extends IOException { + private final int statusCode; + + public HttpStatusException(int statusCode, String message) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java b/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java index 42321e1..362f173 100644 --- a/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java +++ b/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java @@ -156,16 +156,3 @@ private T retry(IOSupplier operation) throws IOException, InterruptedExce throw lastException; } } - -class HttpStatusException extends IOException { - private final int statusCode; - - public HttpStatusException(int statusCode, String message) { - super(message); - this.statusCode = statusCode; - } - - public int getStatusCode() { - return statusCode; - } -} From 6f3846e5115e9252aaccd529ecefc5a145d41747 Mon Sep 17 00:00:00 2001 From: Tako Schotanus Date: Wed, 12 Aug 2026 13:16:44 +0200 Subject: [PATCH 2/2] feat: download now marks missing files --- .../java/dev/jbang/jdkdb/DownloadCommand.java | 8 +++- .../dev/jbang/jdkdb/model/JdkMetadata.java | 13 ++++++ .../jdkdb/scraper/DefaultDownloadManager.java | 45 ++++++++++++++----- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/main/java/dev/jbang/jdkdb/DownloadCommand.java b/src/main/java/dev/jbang/jdkdb/DownloadCommand.java index 2b35d90..be86a26 100644 --- a/src/main/java/dev/jbang/jdkdb/DownloadCommand.java +++ b/src/main/java/dev/jbang/jdkdb/DownloadCommand.java @@ -95,6 +95,11 @@ public class DownloadCommand implements Callable { description = "Randomize the order of downloads instead of processing files in order") private boolean randomize; + @Option( + names = {"--mark-missing"}, + description = "Mark files as missing_since when they return 403/404 during download") + private boolean markMissing; + @Override public Integer call() throws Exception { GitHubUtils.setupGitHubToken(); @@ -136,7 +141,8 @@ public Integer call() throws Exception { var threadCount = maxThreads > 0 ? maxThreads : Runtime.getRuntime().availableProcessors(); DownloadManager downloadManager = statsOnly ? new NoOpDownloadManager(fileTypeFilter) - : new DefaultDownloadManager(threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter); + : new DefaultDownloadManager( + threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter, markMissing); downloadManager.start(); if (fileTypeFilter != null) { logger.info("File type filter enabled: {}", fileTypeFilter); diff --git a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java index 0ec15a3..fa5a156 100644 --- a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java +++ b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java @@ -142,6 +142,10 @@ public enum DistroChecksumType { @JsonInclude(JsonInclude.Include.NON_NULL) private String unlistedSince; + @JsonProperty("missing_since") + @JsonInclude(JsonInclude.Include.NON_NULL) + private String missingSince; + @JsonIgnore private transient Path metadataFile; @@ -365,6 +369,15 @@ public JdkMetadata setUnlistedSince(String unlistedSince) { return this; } + public String getMissingSince() { + return missingSince; + } + + public JdkMetadata setMissingSince(String missingSince) { + this.missingSince = missingSince; + return this; + } + public Path metadataFile() { if (metadataFile == null && filename != null) { return Path.of(filename + ".json"); diff --git a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java index 8fd0e4c..68a4018 100644 --- a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java +++ b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java @@ -39,6 +39,7 @@ public class DefaultDownloadManager implements DownloadManager { private final ConcurrentHashMap submittedPerDistro; private final ConcurrentHashMap completedPerDistro; private final ConcurrentHashMap failedPerDistro; + private final boolean markMissing; private static final Logger logger = LoggerFactory.getLogger(DefaultDownloadManager.class); /** @@ -50,6 +51,7 @@ public class DefaultDownloadManager implements DownloadManager { * @param maxDownloadsPerHost Maximum number of concurrent downloads per host (default: 3) * @param limitTotal Maximum number of total downloads to accept (-1 for unlimited) * @param fileTypeFilter Set of file types to accept (null to accept all) + * @param markMissing Whether to mark files as missing_since when they return 403/404 */ public DefaultDownloadManager( int threadCount, @@ -57,7 +59,8 @@ public DefaultDownloadManager( Path checksumDir, int maxDownloadsPerHost, int limitTotal, - Set fileTypeFilter) { + Set fileTypeFilter, + boolean markMissing) { this.downloadQueue = new LinkedBlockingQueue<>(); this.executorService = Executors.newFixedThreadPool(threadCount); this.httpUtils = new HttpUtils(); @@ -75,6 +78,7 @@ public DefaultDownloadManager( this.submittedPerDistro = new ConcurrentHashMap<>(); this.completedPerDistro = new ConcurrentHashMap<>(); this.failedPerDistro = new ConcurrentHashMap<>(); + this.markMissing = markMissing; } /** @@ -333,13 +337,21 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE try { httpUtils.downloadFile(url, tempFile); } catch (IOException e) { - if (unlistedSince.isPresent() && isHttpStatus(e, 403, 404)) { - task.downloadLogger() - .warn( - "Download returned 40X for unlisted package {} (unlisted_since={}). " - + "This package is most likely not available anymore and is a candidate for pruning.", - filename, - unlistedSince.get()); + if (isHttpStatus(e, 403, 404)) { + if (unlistedSince.isPresent()) { + task.downloadLogger() + .warn( + "Download returned 40X for unlisted package {} (unlisted_since={}). " + + "This package is most likely not available anymore and is a candidate for pruning.", + filename, + unlistedSince.get()); + } + if (markMissing && metadata.getMissingSince() == null) { + String today = java.time.LocalDate.now().toString(); + metadata.setMissingSince(today); + task.downloadLogger().warn("Marking {} as missing_since={}", filename, today); + saveMetadata(task, metadata); + } } throw e; } @@ -364,6 +376,10 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE // Update metadata with download results DownloadResult result = new DownloadResult(md5, sha1, sha256, sha512, size); metadata.download(result); + if (markMissing && metadata.getMissingSince() != null) { + task.downloadLogger().info("Clearing missing_since for {}", filename); + metadata.setMissingSince(null); + } // Extract and parse release info from archive try { @@ -384,10 +400,7 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE } // Save metadata file - Path distroMetadataDir = metadataDir.resolve(task.distro); - Files.createDirectories(distroMetadataDir); - Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile()); - MetadataUtils.saveMetadataFile(metadataFile, metadata); + Path metadataFile = saveMetadata(task, metadata); // Apply the original file timestamp to the metadata file try { @@ -404,6 +417,14 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE } } + private Path saveMetadata(DownloadTask task, JdkMetadata metadata) throws IOException { + Path distroMetadataDir = metadataDir.resolve(task.distro); + Files.createDirectories(distroMetadataDir); + Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile()); + MetadataUtils.saveMetadataFile(metadataFile, metadata); + return metadataFile; + } + /** Save checksum to file */ private void saveChecksumFile(Path checksumDir, String filename, String algorithm, String checksum) throws IOException {