diff --git a/README.md b/README.md index 0ae9977..1fb4197 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,24 @@ The `index` command can be run any time you think the indices might be out of da And finally the `clean` command can be used to get rid of any invalid or orphaned data. If you run it without any flags it will give an overview of everything that could be cleaned up without actually doing any of the cleaning yet. +### Mark unlisted metadata + +You can ask `update` to mark metadata files that are no longer listed by upstream sources: + +```bash +jbang scraper@jbangdev/jdkdb-scraper update --mark-unlisted +``` + +When enabled, the scraper adds `unlisted_since` (current date, `YYYY-MM-DD`) to metadata files that were not listed in the current scan. + +Safety rules: + +- The field is added, never updated +- The field is removed if the metadata appears again in a future scan +- Marking is done per distro. +- A distro is only marked when all scrapers for that distro ran and all of them completed without errors. +- Index files (`all.json`, `latest.json`) are never marked. + ## Command Line Options ### Main Command @@ -87,101 +105,100 @@ And finally the `clean` command can be used to get rid of any invalid or orphane ```bash Usage: jdkdb-scraper [-hV] [COMMAND] Scrapes JDK metadata from various distros and generates index files --h, --help Show this help message and exit. --V, --version Print version information and exit. + -h, --help Show this help message and exit. + -V, --version Print version information and exit. Commands: -update Scrape JDK metadata from various distros and update metadata files -index Generate all.json files for distro directories by aggregating - individual metadata files -download Download and compute checksums for metadata files that have missing - checksum values -clean Clean up metadata by removing incomplete files and pruning old EA - releases + update Scrape JDK metadata from various distros and update metadata files + index Generate all.json files for distro directories by aggregating + individual metadata files + download Download and compute checksums for metadata files that have missing + checksum values + clean Clean up metadata by removing incomplete files and pruning old EA + releases + convert Convert all json files to latest format ``` ### Update Command ```bash -Usage: jdkdb-scraper update [-hlV] [--from-start] [--no-download] [--no-index] - [-c=] [-x=] - [--exclude=[,...]]... - [--include=[,...]]... +Usage: jdkdb-scraper update [-hlV] [--from-start] [--mark-unlisted] + [-c=] [--limit-progress=] - [--limit-total=] - [-m=] [--max-failures=] + [--limit-total=] [-m=] + [--max-failures=] [-p=] [--skip-ea=] [-t=] - [-s=[,...]]... - + [--exclude=[, + ...]]... + [--include=[, + ...]]... [-s=[, + ...]]... Scrape JDK metadata from various distros and update metadata files Options: -c, --checksum-dir= - Directory to store checksum files (default: db/checksums) + Directory to store checksum files (default: + db/checksums) --exclude=[,...] - Exclude these file types (e.g., msi,exe). These types will - not be downloaded. + Exclude these file types (e.g., msi,exe). These + types will not be downloaded. --from-start Ignore existing metadata files and scrape all items from - the start + the start -h, --help Show this help message and exit. --include=[,...] - Include only these file types (e.g., tar_gz,zip). If - specified, only these types will be downloaded. + Include only these file types (e.g., tar_gz,zip). If + specified, only these types will be downloaded. -l, --list List all available scraper IDs and exit --limit-progress= - Maximum number of metadata items to process per scraper - before aborting (default: unlimited) + Maximum number of metadata items to process per + scraper before aborting (default: unlimited) --limit-total= - Maximum total number of downloads to accept before - stopping (default: unlimited) + Maximum total number of downloads to accept before + stopping (default: unlimited) -m, --metadata-dir= - Directory to store metadata files (default: db/metadata) + Directory to store metadata files (default: + db/metadata) --max-failures= - Maximum number of allowed failures per scraper before - aborting that scraper (default: 10) - --no-download Skip downloading files and only generate metadata (for - testing/dry-run) - --no-index Skip generating index files (for testing/dry-run) + Maximum number of allowed failures per scraper + before aborting that scraper (default: 10) + --mark-unlisted + Mark metadata files that were not listed in the + current full successful distro scan by adding + unlisted_since -s, --scrapers=[,...] - Comma-separated list of scraper IDs to run (if not - specified, all scrapers run) + Comma-separated list of scraper IDs to run (if not + specified, all scrapers run) --skip-ea= - Skip early access (EA) releases older than the specified - duration (e.g., '6m' for 6 months, '1y' for 1 year) - (default: 6m) + Skip early access (EA) releases older than the + specified duration (e.g., '6m' for 6 months, '1y' + for 1 year) (default: 6m) -t, --threads= - Maximum number of parallel scraper threads (default: - number of processors) + Maximum number of parallel scraper threads (default: + number of processors) -V, --version Print version information and exit. --x, --index-dir= - Directory to write generated index files to (default: - db/metadata) ``` ### Index Command ```bash Usage: jdkdb-scraper index [-hV] [--allow-incomplete] [-m=] - [-x=] [-v=[,...]]... - + [-x=] [-v=[, + ...]]... Generate all.json files for distro directories by aggregating individual metadata files - -Options: - --allow-incomplete - Allow incomplete metadata files (missing checksums) to be - included + --allow-incomplete Allow incomplete metadata files (missing checksums) + to be included -h, --help Show this help message and exit. -m, --metadata-dir= - Directory containing metadata files (default: - db/metadata) + Directory containing metadata files (default: + db/metadata) -v, --distros=[,...] - Comma-separated list of distro names to regenerate - all.json for (if not specified, all distros are - processed) + Comma-separated list of distro names to regenerate + all.json for (if not specified, all distros are + processed) -V, --version Print version information and exit. -x, --index-dir= - Directory to write generated index files to (default: - db/metadata) + Directory to write generated index files to + (default: db/metadata) ``` ### Download Command @@ -189,45 +206,42 @@ Options: ```bash Usage: jdkdb-scraper download [-hV] [--randomize] [--stats-only] [-c=] - [--exclude=[,...]]... - [--include=[,...]]... [--limit-progress=] - [--limit-total=] - [-m=] [-t=] - [-v=[,...]]... - + [--limit-total=] [-m=] + [-t=] [--exclude=[, + ...]]... + [--include=[, + ...]]... [-v=[, + ...]]... Download and compute checksums for metadata files that have missing checksum values - -Options: -c, --checksum-dir= - Directory to store checksum files (default: db/checksums) + Directory to store checksum files (default: db/checksums) --exclude=[,...] - Exclude these file types (e.g., msi,exe). These types will - not be downloaded. + Exclude these file types (e.g., msi,exe). These types will + not be downloaded. -h, --help Show this help message and exit. --include=[,...] - Include only these file types (e.g., tar_gz,zip). If - specified, only these types will be downloaded. + Include only these file types (e.g., tar_gz,zip). If + specified, only these types will be downloaded. --limit-progress= - Maximum number of metadata items to process per scraper - before aborting (default: unlimited) + Maximum number of metadata items to process per scraper + before aborting (default: unlimited) --limit-total= - Maximum total number of downloads to accept before - stopping (default: unlimited) + Maximum total number of downloads to accept before + stopping (default: unlimited) -m, --metadata-dir= - Directory containing metadata files (default: - db/metadata) + Directory containing metadata files (default: db/metadata) --randomize Randomize the order of downloads instead of processing - files in order + files in order --stats-only Skip downloading files and only show statistics (for - testing/dry-run) + testing/dry-run) -t, --threads= - Maximum number of parallel download threads (default: - number of processors) + Maximum number of parallel download threads (default: + number of processors) -v, --distros=[,...] - Comma-separated list of distro names to process (if not - specified, all distros are processed) + Comma-separated list of distro names to process (if not + specified, all distros are processed) -V, --version Print version information and exit. ``` @@ -235,35 +249,33 @@ Options: ```bash Usage: jdkdb-scraper clean [-hV] [--dry-run] [--prune-checksums] - [--remove-invalid] [-c=] - [-m=] [--prune-ea=] - [--remove-incomplete=] - + [--remove-invalid] [-c=] + [-m=] [--prune-ea=] + [--remove-incomplete=] Clean up metadata by removing incomplete files and pruning old EA releases - -Options: -c, --checksum-dir= - Directory containing checksum files (default: - db/checksums) + Directory containing checksum files (default: + db/checksums) --dry-run Show statistics without actually deleting files -h, --help Show this help message and exit. -m, --metadata-dir= - Directory containing metadata files (default: - db/metadata) + Directory containing metadata files (default: + db/metadata) --prune-checksums - Remove orphaned checksum files that don't have a matching - metadata file + Remove orphaned checksum files that don't have a + matching metadata file --prune-ea= - Prune EA releases older than specified duration (e.g., - 30d, 3w, 6m, 1y). Duration format: [number][d|w|m|y] + Prune EA releases older than specified duration (e. + g., 30d, 3w, 6m, 1y). Duration format: [number] + [d|w|m|y] --remove-incomplete= - Remove metadata files with incomplete data. Options: - checksums (missing checksums), release-info (missing - release info), all (either missing checksums or release - info) (default: all) + Remove metadata files with incomplete data. + Options: checksums (missing checksums), + release-info (missing release info), all (either + missing checksums or release info) (default: all) --remove-invalid - Remove metadata files that fail validation - (MetadataUtils.isValidMetadata) + Remove metadata files that fail validation + (MetadataUtils.isValidMetadata) -V, --version Print version information and exit. ``` diff --git a/src/main/java/dev/jbang/jdkdb/CleanCommand.java b/src/main/java/dev/jbang/jdkdb/CleanCommand.java index ec03324..4eaf591 100644 --- a/src/main/java/dev/jbang/jdkdb/CleanCommand.java +++ b/src/main/java/dev/jbang/jdkdb/CleanCommand.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; @@ -54,16 +55,32 @@ public enum IncompleteType { description = "Remove metadata files that fail validation (MetadataUtils.isValidMetadata)") private boolean removeInvalid; + @Option( + names = {"--remove-orphaned"}, + description = "Remove orphaned checksum files that don't have a matching metadata file") + private boolean removeOrphanedChecksums; + @Option( names = {"--prune-ea"}, + arity = "0..1", + paramLabel = "SINCE", description = "Prune EA releases older than specified duration (e.g., 30d, 3w, 6m, 1y). Duration format: [number][d|w|m|y]") private String pruneEa; @Option( - names = {"--prune-checksums"}, - description = "Remove orphaned checksum files that don't have a matching metadata file") - private boolean pruneChecksums; + names = {"--prune-unlisted"}, + arity = "0..1", + paramLabel = "SINCE", + description = + "Prune metadata files with unlisted_since up to SINCE. SINCE uses duration format [number][d|w|m|y].") + private String pruneUnlisted; + + @Option( + names = {"-p", "--prune-dir"}, + description = "Directory to move pruned files into (default: db/pruned)", + defaultValue = "db/pruned") + private Path pruneDir; @Option( names = {"--dry-run"}, @@ -75,15 +92,23 @@ public Integer call() throws Exception { logger.info("Java Metadata Scraper - Clean"); logger.info("============================="); logger.info("Metadata directory: {}", metadataDir.toAbsolutePath()); + logger.info("Checksum directory: {}", checksumDir.toAbsolutePath()); + logger.info("Prune directory: {}", pruneDir.toAbsolutePath()); // Apply default values if no options specified - if (removeIncomplete == null && !removeInvalid && pruneEa == null && !pruneChecksums && !dryRun) { + if (removeIncomplete == null + && !removeInvalid + && pruneEa == null + && pruneUnlisted == null + && !removeOrphanedChecksums + && !dryRun) { logger.info("No options specified, using defaults: --remove-incomplete=all --prune-ea=6m --dry-run"); logger.info(""); removeIncomplete = IncompleteType.all; removeInvalid = true; - pruneChecksums = true; + removeOrphanedChecksums = true; pruneEa = "6m"; + pruneUnlisted = "1w"; dryRun = true; } @@ -95,7 +120,8 @@ public Integer call() throws Exception { : "disabled")); logger.info(" Remove invalid: {}", removeInvalid); logger.info(" Prune EA: {}", (pruneEa != null ? pruneEa : "disabled")); - logger.info(" Prune checksums: {}", pruneChecksums); + logger.info(" Prune unlisted: {}", (pruneUnlisted != null ? pruneUnlisted : "disabled")); + logger.info(" Remove orphaned checksums: {}", removeOrphanedChecksums); logger.info(" Dry run: {}", dryRun); logger.info(""); @@ -106,7 +132,7 @@ public Integer call() throws Exception { } // Parse prune-ea duration if specified - Instant pruneThreshold = null; + Instant pruneEaThreshold = null; if (pruneEa != null) { Duration duration = MetadataUtils.parseDuration(pruneEa); if (duration == null) { @@ -114,31 +140,52 @@ public Integer call() throws Exception { logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)"); return 1; } - pruneThreshold = Instant.now().minus(duration); - logger.info("Pruning EA releases older than: {} (before {})", pruneEa, pruneThreshold); + pruneEaThreshold = Instant.now().minus(duration); + logger.info("Pruning EA releases older than: {} (before {})", pruneEa, pruneEaThreshold); logger.info(""); } - // Collect files to delete + Instant pruneUnlistedThreshold = null; + if (pruneUnlisted != null) { + Duration duration = MetadataUtils.parseDuration(pruneUnlisted); + if (duration == null) { + logger.error("Error: Invalid --prune-unlisted SINCE value: {}", pruneUnlisted); + logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)"); + return 1; + } + pruneUnlistedThreshold = Instant.now().minus(duration); + logger.info("Pruning unlisted metadata up to: {} (<= {})", pruneUnlisted, pruneUnlistedThreshold); + logger.info(""); + } + + // Collect files to delete/prune final CleanStats stats = new CleanStats(); final List filesToDelete = new ArrayList<>(); - final Instant finalPruneThreshold = pruneThreshold; + final List filesToPrune = new ArrayList<>(); + final Instant finalPruneEaThreshold = pruneEaThreshold; + final Instant finalPruneUnlistedThreshold = pruneUnlistedThreshold; List metadataList = MetadataUtils.collectAllMetadata(distroDir, 2, true, true); for (JdkMetadata metadata : metadataList) { try { - processMetadataFile(metadata, stats, filesToDelete, finalPruneThreshold); + processMetadataFile( + metadata, + stats, + filesToDelete, + filesToPrune, + finalPruneEaThreshold, + finalPruneUnlistedThreshold); } catch (IOException e) { logger.error("Failed to process {}: {}", metadata.metadataFile().getFileName(), e.getMessage()); stats.errors++; } } - // Prune orphaned checksum files (runs last after metadata cleanup) - if (pruneChecksums) { - logger.info("Pruning orphaned checksum files..."); + // Remove orphaned checksum files (runs last after metadata cleanup) + if (removeOrphanedChecksums) { + logger.info("Removing orphaned checksum files..."); logger.info(""); - pruneOrphanedChecksums(distroDir, stats, filesToDelete); + removeOrphanedChecksums(distroDir, stats, filesToDelete); } // Print summary @@ -151,22 +198,47 @@ public Integer call() throws Exception { logger.info(" - missing release info): {}", stats.incompleteReleaseInfo); logger.info("Invalid files: {}", stats.invalidFiles); logger.info("Old EA releases: {}", stats.oldEaReleases); + logger.info("Unlisted releases: {}", stats.unlistedReleases); logger.info("Orphaned checksum files: {}", stats.orphanedChecksums); logger.info("Errors: {}", stats.errors); logger.info(""); - if (filesToDelete.isEmpty()) { - logger.info("No files to delete."); + if (filesToDelete.isEmpty() && filesToPrune.isEmpty()) { + logger.info("No files to delete or prune."); return 0; } + logger.info("Files to prune: {}", filesToPrune.size()); logger.info("Files to delete: {}", filesToDelete.size()); if (dryRun) { logger.info(""); - logger.info("DRY RUN - No files were actually deleted."); - logger.info("Run without --dry-run to perform actual deletion."); + logger.info("DRY RUN - No files were actually pruned or deleted."); + logger.info("Run without --dry-run to perform actual prune/delete operations."); } else { + logger.info(""); + logger.info("Pruning files..."); + int prunedCount = 0; + int pruneFailedCount = 0; + + for (Path file : filesToPrune) { + try { + Path target = resolvePruneTarget(file); + if (target == null) { + logger.error(" Failed to prune {}: unable to resolve prune target", file.getFileName()); + pruneFailedCount++; + continue; + } + Files.createDirectories(target.getParent()); + Files.move(file, target, StandardCopyOption.REPLACE_EXISTING); + prunedCount++; + logger.info(" Pruned: {}", file.getFileName()); + } catch (IOException e) { + logger.error(" Failed to prune {}: {}", file.getFileName(), e.getMessage()); + pruneFailedCount++; + } + } + logger.info(""); logger.info("Deleting files..."); int deletedCount = 0; @@ -184,9 +256,13 @@ public Integer call() throws Exception { } logger.info(""); + logger.info("Pruned: {} files", prunedCount); + if (pruneFailedCount > 0) { + logger.info("Failed to prune: {} files", pruneFailedCount); + } logger.info("Deleted: {} files", deletedCount); if (failedCount > 0) { - logger.info("Failed: {} files", failedCount); + logger.info("Failed to delete: {} files", failedCount); } } @@ -194,13 +270,19 @@ public Integer call() throws Exception { } private void processMetadataFile( - JdkMetadata metadata, CleanStats stats, List filesToDelete, Instant pruneThreshold) + JdkMetadata metadata, + CleanStats stats, + List filesToDelete, + List filesToPrune, + Instant pruneEaThreshold, + Instant pruneUnlistedThreshold) throws IOException { stats.totalFiles++; Path metadataFile = metadata.metadataFile(); boolean shouldDelete = false; + boolean shouldPrune = false; String reason = null; // Check for invalid metadata @@ -245,26 +327,134 @@ private void processMetadataFile( } // Check for old EA releases - if (pruneThreshold != null && "ea".equalsIgnoreCase(metadata.getReleaseType()) && !shouldDelete) { + if (pruneEaThreshold != null && "ea".equalsIgnoreCase(metadata.getReleaseType()) && !shouldDelete) { FileTime lastModified = Files.getLastModifiedTime(metadataFile); - if (lastModified.toInstant().isBefore(pruneThreshold)) { + if (lastModified.toInstant().isBefore(pruneEaThreshold)) { stats.oldEaReleases++; - shouldDelete = true; + shouldPrune = true; reason = "old EA release (last modified: " + lastModified.toInstant() + ")"; } } + if (pruneUnlistedThreshold != null && !shouldDelete) { + String unlistedSince = metadata.getUnlistedSince(); + if (matchesUnlistedSince(unlistedSince, pruneUnlistedThreshold)) { + stats.unlistedReleases++; + if (!shouldPrune) { + shouldPrune = true; + reason = "unlisted since " + unlistedSince; + } + } + } + if (shouldDelete) { - filesToDelete.add(metadataFile); + addIfMissing(filesToDelete, metadataFile); + logger.debug(" - {} ({})", metadataFile.getFileName(), reason); + } else if (shouldPrune) { + addIfMissing(filesToPrune, metadataFile); + for (Path checksumFile : getRelatedChecksumFiles(metadata, metadataFile)) { + addIfMissing(filesToPrune, checksumFile); + } logger.debug(" - {} ({})", metadataFile.getFileName(), reason); } } + private List getRelatedChecksumFiles(JdkMetadata metadata, Path metadataFile) { + List checksumFiles = new ArrayList<>(); + + String distro = metadata.getDistro(); + if (distro == null || distro.isBlank()) { + Path parent = metadataFile.getParent(); + if (parent != null) { + distro = parent.getFileName().toString(); + } + } + + if (distro == null || distro.isBlank()) { + return checksumFiles; + } + + Path distroChecksumDir = checksumDir.resolve(distro); + if (!Files.exists(distroChecksumDir) || !Files.isDirectory(distroChecksumDir)) { + return checksumFiles; + } + + String metadataFileName = metadataFile.getFileName().toString(); + if (!metadataFileName.endsWith(".json")) { + return checksumFiles; + } + + String baseFileName = metadataFileName.substring(0, metadataFileName.length() - 5); + String[] extensions = {".md5", ".sha1", ".sha256", ".sha512"}; + + for (String extension : extensions) { + Path checksumFile = distroChecksumDir.resolve(baseFileName + extension); + if (Files.exists(checksumFile) && Files.isRegularFile(checksumFile)) { + checksumFiles.add(checksumFile); + } + } + + return checksumFiles; + } + + private Path resolvePruneTarget(Path file) { + Path absoluteFile = file.toAbsolutePath().normalize(); + Path absoluteMetadataDir = metadataDir.toAbsolutePath().normalize(); + Path absoluteChecksumDir = checksumDir.toAbsolutePath().normalize(); + + if (absoluteFile.startsWith(absoluteMetadataDir)) { + Path relative = absoluteMetadataDir.relativize(absoluteFile); + if (relative.getNameCount() >= 2) { + return pruneDir.resolve(relative.getName(0).toString()) + .resolve(relative.getFileName().toString()); + } + } + + if (absoluteFile.startsWith(absoluteChecksumDir)) { + Path relative = absoluteChecksumDir.relativize(absoluteFile); + if (relative.getNameCount() >= 2) { + return pruneDir.resolve(relative.getName(0).toString()) + .resolve(relative.getFileName().toString()); + } + } + + Path parent = absoluteFile.getParent(); + if (parent != null) { + Path distro = parent.getFileName(); + if (distro != null) { + return pruneDir.resolve(distro.toString()) + .resolve(absoluteFile.getFileName().toString()); + } + } + + return null; + } + + private void addIfMissing(List files, Path file) { + if (!files.contains(file)) { + files.add(file); + } + } + + private boolean matchesUnlistedSince(String unlistedSince, Instant threshold) { + if (unlistedSince == null || unlistedSince.isBlank()) { + return false; + } + + String value = unlistedSince.trim(); + try { + return !Instant.parse(value).isAfter(threshold); + } catch (RuntimeException ignored) { + logger.debug("Ignoring metadata with non-instant unlisted_since value: {}", value); + return false; + } + } + /** - * Prune orphaned checksum files that don't have corresponding metadata files. + * Remove orphaned checksum files that don't have corresponding metadata files. * This is run after metadata cleanup to remove checksums for deleted metadata. */ - private void pruneOrphanedChecksums(Path metadataDistroDir, CleanStats stats, List filesToDelete) { + private void removeOrphanedChecksums(Path metadataDistroDir, CleanStats stats, List filesToDelete) { if (!Files.exists(checksumDir) || !Files.isDirectory(checksumDir)) { logger.warn("Checksum directory not found: {}", checksumDir.toAbsolutePath()); return; @@ -321,7 +511,7 @@ private void pruneOrphanedChecksums(Path metadataDistroDir, CleanStats stats, Li if (!Files.exists(metadataFile)) { // Metadata file doesn't exist, mark checksum for deletion stats.orphanedChecksums++; - filesToDelete.add(checksumFile); + addIfMissing(filesToDelete, checksumFile); logger.debug( " - {} (orphaned - no metadata file {})", checksumFile.getFileName(), @@ -347,6 +537,7 @@ private static class CleanStats { int incompleteReleaseInfo = 0; int invalidFiles = 0; int oldEaReleases = 0; + int unlistedReleases = 0; int orphanedChecksums = 0; int errors = 0; } diff --git a/src/main/java/dev/jbang/jdkdb/UpdateCommand.java b/src/main/java/dev/jbang/jdkdb/UpdateCommand.java index 98086ec..7d19e38 100644 --- a/src/main/java/dev/jbang/jdkdb/UpdateCommand.java +++ b/src/main/java/dev/jbang/jdkdb/UpdateCommand.java @@ -11,10 +11,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -22,7 +22,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,12 +48,6 @@ public class UpdateCommand implements Callable { defaultValue = "db/checksums") private Path checksumDir; - @Option( - names = {"-p", "--prune-dir"}, - description = - "Move prunable metadata and checksum files to this directory (retaining distro subfolder structure)") - private Path pruneDir; - @Option( names = {"-s", "--scrapers"}, description = "Comma-separated list of scraper IDs to run (if not specified, all scrapers run)", @@ -96,6 +89,12 @@ public class UpdateCommand implements Callable { defaultValue = "-1") private int limitTotal; + @Option( + names = {"--mark-unlisted"}, + description = + "Mark metadata files that were not listed in the current full successful distro scan by adding unlisted_since") + private boolean markUnlisted; + @Option( names = {"--skip-ea"}, description = @@ -294,7 +293,9 @@ public Integer call() throws Exception { logger.info(""); logger.info("All scrapers completed in {} seconds", duration); - pruneOldMetadata(results, allDiscoveries, scrapers); + if (markUnlisted) { + markUnlistedMetadata(results, allDiscoveries, scrapers); + } return successful > 0 ? 0 : 1; } @@ -381,133 +382,173 @@ private void listAvailableScrapers() { logger.info("Total: {} scrapers", names.size()); } - private void pruneOldMetadata( + void markUnlistedMetadata( Map results, Map allDiscoveries, Map scrapers) { - // 1. Collect all local metadata file paths (exclude index files) - var candidatePaths = new HashSet(); - try (var pathStream = Files.walk(metadataDir, 2)) { - pathStream - .filter(Files::isRegularFile) - .filter(p -> p.getFileName().toString().endsWith(".json")) - .filter(p -> !p.getFileName().toString().equals("all.json")) - .filter(p -> !p.getFileName().toString().equals("latest.json")) - .map(p -> p.toAbsolutePath().normalize()) - .forEach(candidatePaths::add); - } catch (IOException e) { - logger.error("Failed to collect metadata files for pruning: {}", e.getMessage(), e); + var eligibleDistros = findDistrosEligibleForUnlistedTreatment(results, allDiscoveries, scrapers); + logger.info(""); + logger.info("Unlisted Marking"); + logger.info("==============="); + logger.info("Overview of unlisted metadata files"); + if (eligibleDistros.isEmpty()) { + logger.info("No distros were eligible for unlisted marking."); + logger.info("A distro is eligible only when all its scrapers ran and completed without errors."); return; } - // 2. Build protected-distro set: scrapers not scheduled to run, or that failed / returned empty results - var protectedDistros = new HashSet(); + LocalDate markDate = LocalDate.now(); + int totalMarked = 0; + int totalRelisted = 0; + + logger.info("Mark date: {}", markDate); + + for (var distro : eligibleDistros) { + var listedPaths = collectListedMetadataPathsForDistro(distro, results, allDiscoveries); + UnlistedUpdateSummary distroSummary = markUnlistedMetadataForDistro(distro, listedPaths, markDate); + int distroMarked = distroSummary.marked(); + int distroRelisted = distroSummary.relisted(); + totalMarked += distroMarked; + totalRelisted += distroRelisted; + logger.info( + " {}: {} file(s) marked as unlisted, {} file(s) restored from unlisted", + distro, + distroMarked, + distroRelisted); + } + + logger.info("Total marked as unlisted: {}", totalMarked); + logger.info("Total restored from unlisted: {}", totalRelisted); + } + + record UnlistedUpdateSummary(int marked, int relisted) {} + + Set findDistrosEligibleForUnlistedTreatment( + Map results, + Map allDiscoveries, + Map scrapers) { + var scrapersByDistro = new HashMap>(); for (var entry : allDiscoveries.entrySet()) { - if (!scrapers.containsKey(entry.getKey())) { - protectedDistros.add(entry.getValue().distro()); - } + scrapersByDistro + .computeIfAbsent(entry.getValue().distro(), ignored -> new HashSet<>()) + .add(entry.getKey()); } - for (var entry : results.entrySet()) { - var result = entry.getValue(); - var discovery = allDiscoveries.get(entry.getKey()); - if (discovery == null) continue; - if (!result.success() || result.allMetadata().isEmpty()) { - protectedDistros.add(discovery.distro()); + + var eligibleDistros = new HashSet(); + for (var entry : scrapersByDistro.entrySet()) { + var distro = entry.getKey(); + var distroScrapers = entry.getValue(); + + boolean allRan = distroScrapers.stream().allMatch(scrapers::containsKey); + if (!allRan) { + continue; + } + + boolean allSuccessful = distroScrapers.stream().allMatch(scraperId -> { + var result = results.get(scraperId); + return result != null + && result.success() + && result.itemsFailed() == 0 + && !result.allMetadata().isEmpty(); + }); + + if (allSuccessful) { + eligibleDistros.add(distro); } } - // 3. Remove files belonging to protected distros from candidates - candidatePaths.removeIf(p -> { - var parent = p.getParent(); - return parent != null - && protectedDistros.contains(parent.getFileName().toString()); - }); + return eligibleDistros; + } - // 4. Remove known-good files (still publicly listed by a successful scraper) + private Set collectListedMetadataPathsForDistro( + String distro, Map results, Map allDiscoveries) { + var listedPaths = new HashSet(); for (var entry : results.entrySet()) { - var result = entry.getValue(); - if (!result.success() || result.allMetadata().isEmpty()) continue; var discovery = allDiscoveries.get(entry.getKey()); - if (discovery == null) continue; - String distro = discovery.distro(); - for (var metadata : result.allMetadata()) { - Path expectedPath = metadataDir + if (discovery == null || !distro.equals(discovery.distro())) { + continue; + } + + for (var metadata : entry.getValue().allMetadata()) { + Path metadataFile = metadata.metadataFile(); + if (metadataFile == null) { + continue; + } + listedPaths.add(metadataDir .resolve(distro) - .resolve(metadata.metadataFile()) + .resolve(metadataFile) .toAbsolutePath() - .normalize(); - candidatePaths.remove(expectedPath); + .normalize()); } } + return listedPaths; + } - // 5. Log summary - var byDistro = new TreeMap>(); - for (var path : candidatePaths) { - String distro = path.getParent().getFileName().toString(); - byDistro.computeIfAbsent(distro, k -> new ArrayList<>()).add(path); + UnlistedUpdateSummary markUnlistedMetadataForDistro(String distro, Set listedPaths, LocalDate markDate) { + Path distroDir = metadataDir.resolve(distro); + if (!Files.exists(distroDir) || !Files.isDirectory(distroDir)) { + return new UnlistedUpdateSummary(0, 0); } - logger.info(""); - logger.info("Prunable Metadata Files"); - logger.info("======================="); - if (candidatePaths.isEmpty()) { - logger.info("No prunable metadata files found."); - return; - } - logger.info("Total prunable files: {}", candidatePaths.size()); - for (var entry : byDistro.entrySet()) { - logger.info(" {}: {} file(s)", entry.getKey(), entry.getValue().size()); - } + int marked = 0; + int relisted = 0; + try (var pathStream = Files.list(distroDir)) { + var metadataFiles = pathStream + .filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(".json")) + .filter(p -> !p.getFileName().toString().equals("all.json")) + .filter(p -> !p.getFileName().toString().equals("latest.json")) + .map(p -> p.toAbsolutePath().normalize()) + .toList(); - // 6. Move files if --prune-dir was specified - if (pruneDir != null) { - logger.info("Moving prunable files to: {}", pruneDir.toAbsolutePath()); - int moved = 0; - int errors = 0; - for (var entry : byDistro.entrySet()) { - String distro = entry.getKey(); - Path targetDistroDir = pruneDir.resolve(distro); - try { - Files.createDirectories(targetDistroDir); - } catch (IOException e) { - logger.error("Failed to create prune directory {}: {}", targetDistroDir, e.getMessage()); - errors++; + for (var metadataPath : metadataFiles) { + if (listedPaths.contains(metadataPath)) { + if (removeUnlistedMarkerFromMetadataFile(metadataPath)) { + relisted++; + } continue; } - for (var srcJson : entry.getValue()) { - // Move the .json metadata file - Path targetJson = targetDistroDir.resolve(srcJson.getFileName()); - try { - Files.move(srcJson, targetJson, StandardCopyOption.REPLACE_EXISTING); - moved++; - } catch (IOException e) { - logger.error("Failed to move {}: {}", srcJson, e.getMessage()); - errors++; - continue; - } - // Move corresponding checksum files alongside the json - String baseName = srcJson.getFileName().toString(); - baseName = baseName.substring(0, baseName.length() - 5); // strip ".json" - for (var ext : List.of(".md5", ".sha1", ".sha256", ".sha512")) { - Path srcChecksum = checksumDir.resolve(distro).resolve(baseName + ext); - if (Files.exists(srcChecksum)) { - Path targetChecksum = targetDistroDir.resolve(baseName + ext); - try { - Files.move(srcChecksum, targetChecksum, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - logger.error("Failed to move checksum file {}: {}", srcChecksum, e.getMessage()); - errors++; - } - } - } + + if (markMetadataFileAsUnlisted(metadataPath, markDate)) { + marked++; } } - logger.info("Moved {} metadata file(s) to {}", moved, pruneDir.toAbsolutePath()); - if (errors > 0) { - logger.warn("{} error(s) occurred during pruning", errors); + } catch (IOException e) { + logger.error("Failed to mark unlisted metadata for distro {}: {}", distro, e.getMessage(), e); + } + + return new UnlistedUpdateSummary(marked, relisted); + } + + private boolean markMetadataFileAsUnlisted(Path metadataFile, LocalDate markDate) { + try { + JdkMetadata metadata = MetadataUtils.readMetadataFile(metadataFile); + if (metadata.getUnlistedSince() != null + && !metadata.getUnlistedSince().isBlank()) { + return false; } - } else { - logger.info("Use --prune-dir to move prunable files to a separate directory."); + metadata.setUnlistedSince(markDate.toString()); + MetadataUtils.saveMetadataFile(metadataFile, metadata); + return true; + } catch (IOException e) { + logger.error("Failed to update metadata file {}: {}", metadataFile, e.getMessage(), e); + return false; + } + } + + private boolean removeUnlistedMarkerFromMetadataFile(Path metadataFile) { + try { + JdkMetadata metadata = MetadataUtils.readMetadataFile(metadataFile); + if (metadata.getUnlistedSince() == null + || metadata.getUnlistedSince().isBlank()) { + return false; + } + metadata.setUnlistedSince(null); + MetadataUtils.saveMetadataFile(metadataFile, metadata); + return true; + } catch (IOException e) { + logger.error("Failed to update metadata file {}: {}", metadataFile, e.getMessage(), e); + return false; } } diff --git a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java index fad5e8b..0ec15a3 100644 --- a/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java +++ b/src/main/java/dev/jbang/jdkdb/model/JdkMetadata.java @@ -138,6 +138,10 @@ public enum DistroChecksumType { @JsonInclude(JsonInclude.Include.NON_NULL) private Map releaseInfo; + @JsonProperty("unlisted_since") + @JsonInclude(JsonInclude.Include.NON_NULL) + private String unlistedSince; + @JsonIgnore private transient Path metadataFile; @@ -352,6 +356,15 @@ public JdkMetadata setReleaseInfo(Map releaseInfo) { return this; } + public String getUnlistedSince() { + return unlistedSince; + } + + public JdkMetadata setUnlistedSince(String unlistedSince) { + this.unlistedSince = unlistedSince; + return this; + } + public Path metadataFile() { if (metadataFile == null && filename != null) { return Path.of(filename + ".json"); diff --git a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java index a4a4f7c..d6cb932 100644 --- a/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java +++ b/src/main/java/dev/jbang/jdkdb/scraper/DefaultDownloadManager.java @@ -38,7 +38,6 @@ public class DefaultDownloadManager implements DownloadManager { private final ConcurrentHashMap submittedPerDistro; private final ConcurrentHashMap completedPerDistro; private final ConcurrentHashMap failedPerDistro; - private static final Logger logger = LoggerFactory.getLogger(DefaultDownloadManager.class); /** @@ -308,6 +307,7 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE JdkMetadata metadata = task.metadata; String filename = metadata.getFilename(); String url = metadata.getUrl(); + Optional unlistedSince = findUnlistedSince(metadata); if (filename == null || url == null) { return; @@ -321,8 +321,27 @@ private void processDownload(DownloadTask task) throws IOException, InterruptedE Path tempFile = Files.createTempFile("jdk-metadata-", "-" + filename); try { + if (unlistedSince.isPresent()) { + task.downloadLogger() + .info( + "Metadata item {} has unlisted_since={} - attempting download", + filename, + unlistedSince.get()); + } task.downloadLogger().info("Downloading " + filename); - httpUtils.downloadFile(url, tempFile); + try { + httpUtils.downloadFile(url, tempFile); + } catch (IOException e) { + if (unlistedSince.isPresent() && isHttp404(e)) { + task.downloadLogger() + .warn( + "Download returned 404 for unlisted package {} (unlisted_since={}). " + + "This package is most likely not available anymore and is a candidate for pruning.", + filename, + unlistedSince.get()); + } + throw e; + } long size = Files.size(tempFile); @@ -391,6 +410,26 @@ private void saveChecksumFile(Path checksumDir, String filename, String algorith Files.writeString(checksumFile, checksum + " " + filename + "\n"); } + private Optional findUnlistedSince(JdkMetadata metadata) { + String value = metadata.getUnlistedSince(); + if (value == null || value.isBlank()) { + return Optional.empty(); + } + return Optional.of(value); + } + + private boolean isHttp404(IOException exception) { + Throwable current = exception; + while (current != null) { + String message = current.getMessage(); + if (message != null && message.contains("HTTP status: 404")) { + return true; + } + current = current.getCause(); + } + return false; + } + /** * Extract the host from a URL. * diff --git a/src/main/java/dev/jbang/jdkdb/util/MetadataUtils.java b/src/main/java/dev/jbang/jdkdb/util/MetadataUtils.java index ed7c599..0d580a1 100644 --- a/src/main/java/dev/jbang/jdkdb/util/MetadataUtils.java +++ b/src/main/java/dev/jbang/jdkdb/util/MetadataUtils.java @@ -686,7 +686,10 @@ private static List filterLatestVersions(List metadata public static List collectAllMetadata( Path dir, int maxDepth, boolean includeComplete, boolean includeIncomplete) throws IOException { List allMetadata = new ArrayList<>(); + long[] lastProgressTime = {System.currentTimeMillis()}; + final long PROGRESS_INTERVAL = 10000; // 10 seconds + logger.info("Collecting metadata..."); try (Stream paths = Files.walk(dir, maxDepth)) { paths.filter(Files::isRegularFile) .filter(p -> p.getFileName().toString().endsWith(".json")) @@ -700,12 +703,20 @@ public static List collectAllMetadata( if ((includeComplete && !isIncomplete) || (includeIncomplete && isIncomplete)) { allMetadata.add(metadata); } + + // Print progress every 10 seconds + long currentTime = System.currentTimeMillis(); + if (currentTime - lastProgressTime[0] >= PROGRESS_INTERVAL) { + logger.info(" Scanned {} files...", allMetadata.size()); + lastProgressTime[0] = currentTime; + } } catch (IOException e) { logger.error("Failed to read metadata file: {} - {}", metadataFile, e.getMessage()); } }); } + logger.info("Collected {} metadata files", allMetadata.size()); return allMetadata; } diff --git a/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java b/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java new file mode 100644 index 0000000..22cbef4 --- /dev/null +++ b/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java @@ -0,0 +1,126 @@ +package dev.jbang.jdkdb; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +class CleanCommandTest { + + @TempDir + Path tempDir; + + @Test + void pruneUnlistedWithoutSinceReturnsError() throws Exception { + Path metadataRoot = tempDir.resolve("metadata"); + Path checksumRoot = tempDir.resolve("checksums"); + Path pruneRoot = tempDir.resolve("pruned"); + + Path distroMetadata = metadataRoot.resolve("temurin"); + Path distroChecksums = checksumRoot.resolve("temurin"); + Files.createDirectories(distroMetadata); + Files.createDirectories(distroChecksums); + + Path unlisted = distroMetadata.resolve("unlisted.json"); + Path listed = distroMetadata.resolve("listed.json"); + Files.writeString( + unlisted, + metadataJson( + "temurin", + "unlisted", + Instant.now() + .minus(1, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString())); + Files.writeString(listed, metadataJson("temurin", "listed", null)); + Files.writeString(distroChecksums.resolve("unlisted.sha256"), "abc"); + + int exit = new CommandLine(new CleanCommand()) + .execute( + "--metadata-dir", + metadataRoot.toString(), + "--checksum-dir", + checksumRoot.toString(), + "--prune-dir", + pruneRoot.toString(), + "--prune-unlisted"); + + assertThat(exit).isEqualTo(1); + assertThat(unlisted).exists(); + assertThat(pruneRoot.resolve("temurin/unlisted.json")).doesNotExist(); + assertThat(distroChecksums.resolve("unlisted.sha256")).exists(); + assertThat(pruneRoot.resolve("temurin/unlisted.sha256")).doesNotExist(); + assertThat(listed).exists(); + } + + @Test + void pruneUnlistedWithDurationOnlyPrunesOlderMarkers() throws Exception { + Path metadataRoot = tempDir.resolve("metadata"); + Path checksumRoot = tempDir.resolve("checksums"); + Path pruneRoot = tempDir.resolve("pruned"); + Path distroMetadata = metadataRoot.resolve("temurin"); + Files.createDirectories(distroMetadata); + Files.createDirectories(checksumRoot.resolve("temurin")); + + String oldDate = Instant.now() + .minus(10, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + String recentDate = Instant.now() + .minus(2, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + + Path oldUnlisted = distroMetadata.resolve("old-unlisted.json"); + Path recentUnlisted = distroMetadata.resolve("recent-unlisted.json"); + Files.writeString(oldUnlisted, metadataJson("temurin", "old-unlisted", oldDate)); + Files.writeString(recentUnlisted, metadataJson("temurin", "recent-unlisted", recentDate)); + + int exit = new CommandLine(new CleanCommand()) + .execute( + "--metadata-dir", + metadataRoot.toString(), + "--checksum-dir", + checksumRoot.toString(), + "--prune-dir", + pruneRoot.toString(), + "--prune-unlisted", + "5d"); + + assertThat(exit).isZero(); + assertThat(oldUnlisted).doesNotExist(); + assertThat(pruneRoot.resolve("temurin/old-unlisted.json")).exists(); + assertThat(recentUnlisted).exists(); + } + + @Test + void pruneUnlistedWithInvalidSinceReturnsError() { + int exit = new CommandLine(new CleanCommand()).execute("--prune-unlisted", "not-a-date"); + + assertThat(exit).isEqualTo(1); + } + + private String metadataJson(String distro, String filename, String unlistedSince) { + String unlisted = unlistedSince == null ? "" : "\n \"unlisted_since\": \"" + unlistedSince + "\","; + return """ + { + \"distro\": \"%s\", + \"filename\": \"%s\",%s + \"version\": \"21.0.1\", + \"java_version\": \"21\", + \"release_type\": \"ga\", + \"jvm_impl\": \"hotspot\", + \"os\": \"linux\", + \"architecture\": \"x86_64\", + \"file_type\": \"tar.gz\", + \"image_type\": \"jdk\", + \"url\": \"https://example.com/%s.tar.gz\" + } + """.formatted(distro, filename, unlisted, filename); + } +} diff --git a/src/test/java/dev/jbang/jdkdb/UpdateCommandTest.java b/src/test/java/dev/jbang/jdkdb/UpdateCommandTest.java new file mode 100644 index 0000000..9f590a6 --- /dev/null +++ b/src/test/java/dev/jbang/jdkdb/UpdateCommandTest.java @@ -0,0 +1,186 @@ +package dev.jbang.jdkdb; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.jbang.jdkdb.model.JdkMetadata; +import dev.jbang.jdkdb.scraper.Scraper; +import dev.jbang.jdkdb.scraper.ScraperResult; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +class UpdateCommandTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + @Test + void findDistrosEligibleForUnlistedMarkingRequiresAllDistroScrapersAndNoFailures() { + UpdateCommand command = new UpdateCommand(); + + Map allDiscoveries = new HashMap<>(); + allDiscoveries.put("a-1", discovery("a-1", "distro-a")); + allDiscoveries.put("a-2", discovery("a-2", "distro-a")); + allDiscoveries.put("b-1", discovery("b-1", "distro-b")); + allDiscoveries.put("b-2", discovery("b-2", "distro-b")); + allDiscoveries.put("c-1", discovery("c-1", "distro-c")); + allDiscoveries.put("d-1", discovery("d-1", "distro-d")); + + Map ranScrapers = new HashMap<>(); + ranScrapers.put("a-1", () -> null); + ranScrapers.put("a-2", () -> null); + ranScrapers.put("b-1", () -> null); + ranScrapers.put("c-1", () -> null); + ranScrapers.put("d-1", () -> null); + + Map results = new HashMap<>(); + results.put("a-1", ScraperResult.success(1, 0, 0, List.of(metadataRef("a-1.json")))); + results.put("a-2", ScraperResult.success(1, 0, 0, List.of(metadataRef("a-2.json")))); + results.put("b-1", ScraperResult.success(1, 0, 0, List.of(metadataRef("b-1.json")))); + results.put("c-1", ScraperResult.success(1, 0, 1, List.of(metadataRef("c-1.json")))); + results.put("d-1", ScraperResult.failure(new RuntimeException("boom"))); + + Set eligible = command.findDistrosEligibleForUnlistedTreatment(results, allDiscoveries, ranScrapers); + + assertThat(eligible).containsExactly("distro-a"); + } + + @Test + void pruneOptionsAreNoLongerAccepted() { + UpdateCommand command = new UpdateCommand(); + CommandLine commandLine = new CommandLine(command); + + assertThatThrownBy(() -> commandLine.parseArgs("--prune-dir", "db/pruned")) + .isInstanceOf(CommandLine.ParameterException.class) + .hasMessageContaining("Unknown option"); + assertThatThrownBy(() -> commandLine.parseArgs("--prune-unlisted")) + .isInstanceOf(CommandLine.ParameterException.class) + .hasMessageContaining("Unknown option"); + } + + @Test + void markUnlistedMetadataAlwaysLogsOverviewEvenWithoutEligibleDistros() { + UpdateCommand command = new UpdateCommand(); + Logger logger = (Logger) LoggerFactory.getLogger("command"); + ListAppender listAppender = new ListAppender<>(); + listAppender.start(); + logger.addAppender(listAppender); + + try { + command.markUnlistedMetadata(new HashMap<>(), new HashMap<>(), new HashMap<>()); + } finally { + logger.detachAppender(listAppender); + listAppender.stop(); + } + + assertThat(listAppender.list) + .extracting(ILoggingEvent::getFormattedMessage) + .contains("Overview of unlisted metadata files"); + } + + @Test + void findDistrosEligibleForUnlistedMarkingExcludesEmptySuccessfulResults() { + UpdateCommand command = new UpdateCommand(); + + Map allDiscoveries = new HashMap<>(); + allDiscoveries.put("x-1", discovery("x-1", "distro-x")); + + Map ranScrapers = new HashMap<>(); + ranScrapers.put("x-1", () -> null); + + Map results = new HashMap<>(); + results.put("x-1", ScraperResult.success(0, 0, 0, List.of())); + + Set eligible = command.findDistrosEligibleForUnlistedTreatment(results, allDiscoveries, ranScrapers); + + assertThat(eligible).isEmpty(); + } + + @Test + void markUnlistedMetadataForDistroMarksUnlistedAndClearsRelistedFiles() throws Exception { + UpdateCommand command = new UpdateCommand(); + Path metadataRoot = tempDir.resolve("metadata"); + Path distroDir = metadataRoot.resolve("temurin"); + Files.createDirectories(distroDir); + setField(command, "metadataDir", metadataRoot); + + Path listedFile = distroDir.resolve("listed.json"); + Path unlistedFile = distroDir.resolve("unlisted.json"); + Path alreadyMarkedFile = distroDir.resolve("already-marked.json"); + Path allJson = distroDir.resolve("all.json"); + + Files.writeString(listedFile, "{\n \"filename\": \"listed\",\n \"unlisted_since\": \"2024-01-01\"\n}\n"); + Files.writeString(unlistedFile, "{\n \"filename\": \"unlisted\"\n}\n"); + Files.writeString( + alreadyMarkedFile, "{\n \"filename\": \"already-marked\",\n \"unlisted_since\": \"2024-01-01\"\n}\n"); + Files.writeString(allJson, "[]\n"); + + Set listedPaths = Set.of(listedFile.toAbsolutePath().normalize()); + LocalDate markDate = LocalDate.of(2026, 8, 7); + + UpdateCommand.UnlistedUpdateSummary summary = + command.markUnlistedMetadataForDistro("temurin", listedPaths, markDate); + int marked = summary.marked(); + int relisted = summary.relisted(); + + assertThat(marked).isEqualTo(1); + assertThat(relisted).isEqualTo(1); + assertThat(MAPPER.readTree(listedFile.toFile()).get("unlisted_since")).isNull(); + assertThat(MAPPER.readTree(unlistedFile.toFile()).get("unlisted_since").asText()) + .isEqualTo("2026-08-07"); + assertThat(MAPPER.readTree(alreadyMarkedFile.toFile()) + .get("unlisted_since") + .asText()) + .isEqualTo("2024-01-01"); + } + + private Scraper.Discovery discovery(String name, String distro) { + return new Scraper.Discovery() { + @Override + public String name() { + return name; + } + + @Override + public String distro() { + return distro; + } + + @Override + public String vendor() { + return "test-vendor"; + } + + @Override + public Scraper create(dev.jbang.jdkdb.scraper.ScraperConfig config) { + return () -> ScraperResult.success(0, 0, 0, List.of()); + } + }; + } + + private JdkMetadata metadataRef(String fileName) { + return JdkMetadata.create().metadataFile(Path.of(fileName)); + } + + private void setField(Object target, String fieldName, Object value) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } +}