feat: download now marks missing files

src/main/java/dev/jbang/jdkdb/DownloadCommand.java

@@ -95,6 +95,11 @@ public class DownloadCommand implements Callable<Integer> {
 			description = "Randomize the order of downloads instead of processing files in order")
 	private boolean randomize;
 
+	@Option(
+			names = {"--mark-missing"},
+			description = "Mark files as missing_since when they return 403/404 during download")
+	private boolean markMissing;
+
 	@Override
 	public Integer call() throws Exception {
 		GitHubUtils.setupGitHubToken();
@@ -136,7 +141,8 @@ public Integer call() throws Exception {
 		var threadCount = maxThreads > 0 ? maxThreads : Runtime.getRuntime().availableProcessors();
 		DownloadManager downloadManager = statsOnly
 				? new NoOpDownloadManager(fileTypeFilter)
-				: new DefaultDownloadManager(threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter);
+				: new DefaultDownloadManager(
+						threadCount, metadataDir, checksumDir, 3, limitTotal, fileTypeFilter, markMissing);
 		downloadManager.start();
 		if (fileTypeFilter != null) {
 			logger.info("File type filter enabled: {}", fileTypeFilter);

src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java

@@ -142,6 +142,10 @@ public enum DistroChecksumType {
 	@JsonInclude(JsonInclude.Include.NON_NULL)
 	private String unlistedSince;
 
+	@JsonProperty("missing_since")
+	@JsonInclude(JsonInclude.Include.NON_NULL)
+	private String missingSince;
+
 	@JsonIgnore
 	private transient Path metadataFile;
 
@@ -365,6 +369,15 @@ public JdkMetadata setUnlistedSince(String unlistedSince) {
 		return this;
 	}
 
+	public String getMissingSince() {
+		return missingSince;
+	}
+
+	public JdkMetadata setMissingSince(String missingSince) {
+		this.missingSince = missingSince;
+		return this;
+	}
+
 	public Path metadataFile() {
 		if (metadataFile == null && filename != null) {
 			return Path.of(filename + ".json");

src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java

@@ -3,6 +3,7 @@
 import dev.jbang.jdkdb.model.JdkMetadata;
 import dev.jbang.jdkdb.util.ArchiveUtils;
 import dev.jbang.jdkdb.util.HashUtils;
+import dev.jbang.jdkdb.util.HttpStatusException;
 import dev.jbang.jdkdb.util.HttpUtils;
 import dev.jbang.jdkdb.util.MetadataUtils;
 import java.io.*;
@@ -38,6 +39,7 @@ public class DefaultDownloadManager implements DownloadManager {
 	private final ConcurrentHashMap<String, AtomicInteger> submittedPerDistro;
 	private final ConcurrentHashMap<String, AtomicInteger> completedPerDistro;
 	private final ConcurrentHashMap<String, AtomicInteger> failedPerDistro;
+	private final boolean markMissing;
 	private static final Logger logger = LoggerFactory.getLogger(DefaultDownloadManager.class);
 
 	/**
@@ -49,14 +51,16 @@ public class DefaultDownloadManager implements DownloadManager {
 	 * @param maxDownloadsPerHost Maximum number of concurrent downloads per host (default: 3)
 	 * @param limitTotal Maximum number of total downloads to accept (-1 for unlimited)
 	 * @param fileTypeFilter Set of file types to accept (null to accept all)
+	 * @param markMissing Whether to mark files as missing_since when they return 403/404
 	 */
 	public DefaultDownloadManager(
 			int threadCount,
 			Path metadataDir,
 			Path checksumDir,
 			int maxDownloadsPerHost,
 			int limitTotal,
-			Set<JdkMetadata.FileType> fileTypeFilter) {
+			Set<JdkMetadata.FileType> fileTypeFilter,
+			boolean markMissing) {
 		this.downloadQueue = new LinkedBlockingQueue<>();
 		this.executorService = Executors.newFixedThreadPool(threadCount);
 		this.httpUtils = new HttpUtils();
@@ -74,6 +78,7 @@ public DefaultDownloadManager(
 		this.submittedPerDistro = new ConcurrentHashMap<>();
 		this.completedPerDistro = new ConcurrentHashMap<>();
 		this.failedPerDistro = new ConcurrentHashMap<>();
+		this.markMissing = markMissing;
 	}
 
 	/**
@@ -332,13 +337,21 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE
 			try {
 				httpUtils.downloadFile(url, tempFile);
 			} catch (IOException e) {
-				if (unlistedSince.isPresent() && isHttp404(e)) {
-					task.downloadLogger()
-							.warn(
-									"Download returned 404 for unlisted package {} (unlisted_since={}). "
-											+ "This package is most likely not available anymore and is a candidate for pruning.",
-									filename,
-									unlistedSince.get());
+				if (isHttpStatus(e, 403, 404)) {
+					if (unlistedSince.isPresent()) {
+						task.downloadLogger()
+								.warn(
+										"Download returned 40X for unlisted package {} (unlisted_since={}). "
+												+ "This package is most likely not available anymore and is a candidate for pruning.",
+										filename,
+										unlistedSince.get());
+					}
+					if (markMissing && metadata.getMissingSince() == null) {
+						String today = java.time.LocalDate.now().toString();
+						metadata.setMissingSince(today);
+						task.downloadLogger().warn("Marking {} as missing_since={}", filename, today);
+						saveMetadata(task, metadata);
+					}
 				}
 				throw e;
 			}
@@ -363,6 +376,10 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE
 			// Update metadata with download results
 			DownloadResult result = new DownloadResult(md5, sha1, sha256, sha512, size);
 			metadata.download(result);
+			if (markMissing && metadata.getMissingSince() != null) {
+				task.downloadLogger().info("Clearing missing_since for {}", filename);
+				metadata.setMissingSince(null);
+			}
 
 			// Extract and parse release info from archive
 			try {
@@ -383,10 +400,7 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE
 			}
 
 			// Save metadata file
-			Path distroMetadataDir = metadataDir.resolve(task.distro);
-			Files.createDirectories(distroMetadataDir);
-			Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile());
-			MetadataUtils.saveMetadataFile(metadataFile, metadata);
+			Path metadataFile = saveMetadata(task, metadata);
 
 			// Apply the original file timestamp to the metadata file
 			try {
@@ -403,6 +417,14 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE
 		}
 	}
 
+	private Path saveMetadata(DownloadTask task, JdkMetadata metadata) throws IOException {
+		Path distroMetadataDir = metadataDir.resolve(task.distro);
+		Files.createDirectories(distroMetadataDir);
+		Path metadataFile = distroMetadataDir.resolve(metadata.metadataFile());
+		MetadataUtils.saveMetadataFile(metadataFile, metadata);
+		return metadataFile;
+	}
+
 	/** Save checksum to file */
 	private void saveChecksumFile(Path checksumDir, String filename, String algorithm, String checksum)
 			throws IOException {
@@ -418,14 +440,14 @@ private Optional<String> findUnlistedSince(JdkMetadata metadata) {
 		return Optional.of(value);
 	}
 
-	private boolean isHttp404(IOException exception) {
-		Throwable current = exception;
-		while (current != null) {
-			String message = current.getMessage();
-			if (message != null && message.contains("HTTP status: 404")) {
+	private boolean isHttpStatus(IOException e, int... statusCodes) {
+		if (!(e instanceof HttpStatusException hse)) {
+			return false;
+		}
+		for (int code : statusCodes) {
+			if (hse.getStatusCode() == code) {
 				return true;
 			}
-			current = current.getCause();
 		}
 		return false;
 	}

src/main/java/dev/jbang/jdkdb/util/HttpStatusException.java

@@ -0,0 +1,16 @@
+package dev.jbang.jdkdb.util;
+
+import java.io.IOException;
+
+public class HttpStatusException extends IOException {
+	private final int statusCode;
+
+	public HttpStatusException(int statusCode, String message) {
+		super(message);
+		this.statusCode = statusCode;
+	}
+
+	public int getStatusCode() {
+		return statusCode;
+	}
+}

src/main/java/dev/jbang/jdkdb/util/HttpUtils.java

@@ -156,16 +156,3 @@ private <T> T retry(IOSupplier<T> operation) throws IOException, InterruptedExce
 		throw lastException;
 	}
 }
-
-class HttpStatusException extends IOException {
-	private final int statusCode;
-
-	public HttpStatusException(int statusCode, String message) {
-		super(message);
-		this.statusCode = statusCode;
-	}
-
-	public int getStatusCode() {
-		return statusCode;
-	}
-}