diff --git a/src/main/java/dev/jbang/jdkdb/DownloadCommand.java b/src/main/java/dev/jbang/jdkdb/DownloadCommand.java index 2b35d90..be86a26 100644 --- a/src/main/java/dev/jbang/jdkdb/DownloadCommand.java +++ b/src/main/java/dev/jbang/jdkdb/DownloadCommand.java @@ -95,6 +95,11 @@ public class DownloadCommand implements Callable { description = "Randomize the order of downloads instead of processing files in order") private boolean randomize; + @Option( + names = {"--mark-missing"}, + description = "Mark files as missing_since when they return 403/404 during download") + private boolean markMissing; + @Override public Integer call() throws Exception { GitHubUtils.setupGitHubToken(); @@ -136,7 +141,8 @@ public Integer call() throws Exception { var threadCount = maxThreads > 0 ? maxThreads : Runtime.getRuntime().availableProcessors(); DownloadManager downloadManager = statsOnly ? new NoOpDownloadManager(fileTypeFilter) - : new DefaultDownloadManager(threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter); + : new DefaultDownloadManager( + threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter, markMissing); downloadManager.start(); if (fileTypeFilter != null) { logger.info("File type filter enabled: {}", fileTypeFilter); diff --git a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java index 0ec15a3..fa5a156 100644 --- a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java +++ b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java @@ -142,6 +142,10 @@ public enum DistroChecksumType { @JsonInclude(JsonInclude.Include.NON_NULL) private String unlistedSince; + @JsonProperty("missing_since") + @JsonInclude(JsonInclude.Include.NON_NULL) + private String missingSince; + @JsonIgnore private transient Path metadataFile; @@ -365,6 +369,15 @@ public JdkMetadata setUnlistedSince(String unlistedSince) { return this; } + public String getMissingSince() { + return missingSince; + } + + public JdkMetadata setMissingSince(String missingSince) { + this.missingSince = missingSince; + return this; + } + public Path metadataFile() { if (metadataFile == null && filename != null) { return Path.of(filename + ".json"); diff --git a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java index d6cb932..68a4018 100644 --- a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java +++ b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java @@ -3,6 +3,7 @@ import dev.jbang.jdkdb.model.JdkMetadata; import dev.jbang.jdkdb.util.ArchiveUtils; import dev.jbang.jdkdb.util.HashUtils; +import dev.jbang.jdkdb.util.HttpStatusException; import dev.jbang.jdkdb.util.HttpUtils; import dev.jbang.jdkdb.util.MetadataUtils; import java.io.*; @@ -38,6 +39,7 @@ public class DefaultDownloadManager implements DownloadManager { private final ConcurrentHashMap submittedPerDistro; private final ConcurrentHashMap completedPerDistro; private final ConcurrentHashMap failedPerDistro; + private final boolean markMissing; private static final Logger logger = LoggerFactory.getLogger(DefaultDownloadManager.class); /** @@ -49,6 +51,7 @@ public class DefaultDownloadManager implements DownloadManager { * @param maxDownloadsPerHost Maximum number of concurrent downloads per host (default: 3) * @param limitTotal Maximum number of total downloads to accept (-1 for unlimited) * @param fileTypeFilter Set of file types to accept (null to accept all) + * @param markMissing Whether to mark files as missing_since when they return 403/404 */ public DefaultDownloadManager( int threadCount, @@ -56,7 +59,8 @@ public DefaultDownloadManager( Path checksumDir, int maxDownloadsPerHost, int limitTotal, - Set fileTypeFilter) { + Set fileTypeFilter, + boolean markMissing) { this.downloadQueue = new LinkedBlockingQueue<>(); this.executorService = Executors.newFixedThreadPool(threadCount); this.httpUtils = new HttpUtils(); @@ -74,6 +78,7 @@ public DefaultDownloadManager( this.submittedPerDistro = new ConcurrentHashMap<>(); this.completedPerDistro = new ConcurrentHashMap<>(); this.failedPerDistro = new ConcurrentHashMap<>(); + this.markMissing = markMissing; } /** @@ -332,13 +337,21 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE try { httpUtils.downloadFile(url, tempFile); } catch (IOException e) { - if (unlistedSince.isPresent() && isHttp404(e)) { - task.downloadLogger() - .warn( - "Download returned 404 for unlisted package {} (unlisted_since={}). " - + "This package is most likely not available anymore and is a candidate for pruning.", - filename, - unlistedSince.get()); + if (isHttpStatus(e, 403, 404)) { + if (unlistedSince.isPresent()) { + task.downloadLogger() + .warn( + "Download returned 40X for unlisted package {} (unlisted_since={}). " + + "This package is most likely not available anymore and is a candidate for pruning.", + filename, + unlistedSince.get()); + } + if (markMissing && metadata.getMissingSince() == null) { + String today = java.time.LocalDate.now().toString(); + metadata.setMissingSince(today); + task.downloadLogger().warn("Marking {} as missing_since={}", filename, today); + saveMetadata(task, metadata); + } } throw e; } @@ -363,6 +376,10 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE // Update metadata with download results DownloadResult result = new DownloadResult(md5, sha1, sha256, sha512, size); metadata.download(result); + if (markMissing && metadata.getMissingSince() != null) { + task.downloadLogger().info("Clearing missing_since for {}", filename); + metadata.setMissingSince(null); + } // Extract and parse release info from archive try { @@ -383,10 +400,7 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE } // Save metadata file - Path distroMetadataDir = metadataDir.resolve(task.distro); - Files.createDirectories(distroMetadataDir); - Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile()); - MetadataUtils.saveMetadataFile(metadataFile, metadata); + Path metadataFile = saveMetadata(task, metadata); // Apply the original file timestamp to the metadata file try { @@ -403,6 +417,14 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE } } + private Path saveMetadata(DownloadTask task, JdkMetadata metadata) throws IOException { + Path distroMetadataDir = metadataDir.resolve(task.distro); + Files.createDirectories(distroMetadataDir); + Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile()); + MetadataUtils.saveMetadataFile(metadataFile, metadata); + return metadataFile; + } + /** Save checksum to file */ private void saveChecksumFile(Path checksumDir, String filename, String algorithm, String checksum) throws IOException { @@ -418,14 +440,14 @@ private Optional findUnlistedSince(JdkMetadata metadata) { return Optional.of(value); } - private boolean isHttp404(IOException exception) { - Throwable current = exception; - while (current != null) { - String message = current.getMessage(); - if (message != null && message.contains("HTTP status: 404")) { + private boolean isHttpStatus(IOException e, int... statusCodes) { + if (!(e instanceof HttpStatusException hse)) { + return false; + } + for (int code : statusCodes) { + if (hse.getStatusCode() == code) { return true; } - current = current.getCause(); } return false; } diff --git a/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java b/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java new file mode 100644 index 0000000..378fbdb --- /dev/null +++ b/src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java @@ -0,0 +1,16 @@ +package dev.jbang.jdkdb.util; + +import java.io.IOException; + +public class HttpStatusException extends IOException { + private final int statusCode; + + public HttpStatusException(int statusCode, String message) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java b/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java index 42321e1..362f173 100644 --- a/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java +++ b/src/main/java/dev/jbang/jdkdb/util/HttpUtils.java @@ -156,16 +156,3 @@ private T retry(IOSupplier operation) throws IOException, InterruptedExce throw lastException; } } - -class HttpStatusException extends IOException { - private final int statusCode; - - public HttpStatusException(int statusCode, String message) { - super(message); - this.statusCode = statusCode; - } - - public int getStatusCode() { - return statusCode; - } -}