From 77204b4846ea7cbf803d8a8a889dd67885c1ebd7 Mon Sep 17 00:00:00 2001 From: Tako Schotanus Date: Thu, 13 Aug 2026 12:20:31 +0200 Subject: [PATCH] feat: added more prune commands to clean command This includes `--prune-missing` and `--prune-unlisted-missing` --- .../java/dev/jbang/jdkdb/CleanCommand.java | 107 +++++++++++- .../dev/jbang/jdkdb/CleanCommandTest.java | 153 +++++++++++++++++- 2 files changed, 251 insertions(+), 9 deletions(-) diff --git a/src/main/java/dev/jbang/jdkdb/CleanCommand.java b/src/main/java/dev/jbang/jdkdb/CleanCommand.java index b9eb449..c7e084a 100644 --- a/src/main/java/dev/jbang/jdkdb/CleanCommand.java +++ b/src/main/java/dev/jbang/jdkdb/CleanCommand.java @@ -9,6 +9,8 @@ import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -76,6 +78,22 @@ public enum IncompleteType { "Prune metadata files with unlisted_since up to SINCE. SINCE uses duration format [number][d|w|m|y].") private String pruneUnlisted; + @Option( + names = {"--prune-missing"}, + arity = "0..1", + paramLabel = "SINCE", + description = + "Prune metadata files with missing_since up to SINCE. SINCE uses duration format [number][d|w|m|y].") + private String pruneMissing; + + @Option( + names = {"--prune-unlisted-missing"}, + arity = "0..1", + paramLabel = "SINCE", + description = + "Prune metadata files where both unlisted_since and missing_since are up to SINCE. SINCE uses duration format [number][d|w|m|y].") + private String pruneUnlistedMissing; + @Option( names = {"-p", "--prune-dir"}, description = "Directory to move pruned files into (default: db/pruned)", @@ -100,6 +118,8 @@ public Integer call() throws Exception { && !pruneInvalid && pruneEa == null && pruneUnlisted == null + && pruneMissing == null + && pruneUnlistedMissing == null && !removeOrphanedChecksums && !dryRun) { logger.info("No options specified, using defaults: --remove-incomplete=all --prune-ea=6m --dry-run"); @@ -121,6 +141,8 @@ public Integer call() throws Exception { logger.info(" Prune invalid: {}", pruneInvalid); logger.info(" Prune EA: {}", (pruneEa != null ? pruneEa : "disabled")); logger.info(" Prune unlisted: {}", (pruneUnlisted != null ? pruneUnlisted : "disabled")); + logger.info(" Prune missing: {}", (pruneMissing != null ? pruneMissing : "disabled")); + logger.info(" Prune unlisted+missing: {}", (pruneUnlistedMissing != null ? pruneUnlistedMissing : "disabled")); logger.info(" Remove orphaned checksums: {}", removeOrphanedChecksums); logger.info(" Dry run: {}", dryRun); logger.info(""); @@ -158,12 +180,43 @@ public Integer call() throws Exception { logger.info(""); } + Instant pruneMissingThreshold = null; + if (pruneMissing != null) { + Duration duration = MetadataUtils.parseDuration(pruneMissing); + if (duration == null) { + logger.error("Error: Invalid --prune-missing SINCE value: {}", pruneMissing); + logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)"); + return 1; + } + pruneMissingThreshold = Instant.now().minus(duration); + logger.info("Pruning missing metadata up to: {} (<= {})", pruneMissing, pruneMissingThreshold); + logger.info(""); + } + + Instant pruneUnlistedMissingThreshold = null; + if (pruneUnlistedMissing != null) { + Duration duration = MetadataUtils.parseDuration(pruneUnlistedMissing); + if (duration == null) { + logger.error("Error: Invalid --prune-unlisted-missing SINCE value: {}", pruneUnlistedMissing); + logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)"); + return 1; + } + pruneUnlistedMissingThreshold = Instant.now().minus(duration); + logger.info( + "Pruning unlisted+missing metadata up to: {} (<= {})", + pruneUnlistedMissing, + pruneUnlistedMissingThreshold); + logger.info(""); + } + // Collect files to delete/prune final CleanStats stats = new CleanStats(); final List filesToDelete = new ArrayList<>(); final List filesToPrune = new ArrayList<>(); final Instant finalPruneEaThreshold = pruneEaThreshold; final Instant finalPruneUnlistedThreshold = pruneUnlistedThreshold; + final Instant finalPruneMissingThreshold = pruneMissingThreshold; + final Instant finalPruneUnlistedMissingThreshold = pruneUnlistedMissingThreshold; List metadataList = MetadataUtils.collectAllMetadata(distroDir, 2, true, true); for (JdkMetadata metadata : metadataList) { @@ -174,7 +227,9 @@ public Integer call() throws Exception { filesToDelete, filesToPrune, finalPruneEaThreshold, - finalPruneUnlistedThreshold); + finalPruneUnlistedThreshold, + finalPruneMissingThreshold, + finalPruneUnlistedMissingThreshold); } catch (IOException e) { logger.error("Failed to process {}: {}", metadata.metadataFile().getFileName(), e.getMessage()); stats.errors++; @@ -199,6 +254,8 @@ public Integer call() throws Exception { logger.info("Invalid files: {}", stats.invalidFiles); logger.info("Old EA releases: {}", stats.oldEaReleases); logger.info("Unlisted releases: {}", stats.unlistedReleases); + logger.info("Missing releases: {}", stats.missingReleases); + logger.info("Unlisted+missing releases: {}", stats.unlistedMissingReleases); logger.info("Orphaned checksum files: {}", stats.orphanedChecksums); logger.info("Errors: {}", stats.errors); logger.info(""); @@ -275,7 +332,9 @@ private void processMetadataFile( List filesToDelete, List filesToPrune, Instant pruneEaThreshold, - Instant pruneUnlistedThreshold) + Instant pruneUnlistedThreshold, + Instant pruneMissingThreshold, + Instant pruneUnlistedMissingThreshold) throws IOException { stats.totalFiles++; @@ -338,7 +397,7 @@ private void processMetadataFile( if (pruneUnlistedThreshold != null && !shouldDelete) { String unlistedSince = metadata.getUnlistedSince(); - if (matchesUnlistedSince(unlistedSince, pruneUnlistedThreshold)) { + if (matchesSince(unlistedSince, pruneUnlistedThreshold)) { stats.unlistedReleases++; if (!shouldPrune) { shouldPrune = true; @@ -347,6 +406,30 @@ private void processMetadataFile( } } + if (pruneMissingThreshold != null && !shouldDelete) { + String missingSince = metadata.getMissingSince(); + if (matchesSince(missingSince, pruneMissingThreshold)) { + stats.missingReleases++; + if (!shouldPrune) { + shouldPrune = true; + reason = "missing since " + missingSince; + } + } + } + + if (pruneUnlistedMissingThreshold != null && !shouldDelete) { + String unlistedSince = metadata.getUnlistedSince(); + String missingSince = metadata.getMissingSince(); + if (matchesSince(unlistedSince, pruneUnlistedMissingThreshold) + && matchesSince(missingSince, pruneUnlistedMissingThreshold)) { + stats.unlistedMissingReleases++; + if (!shouldPrune) { + shouldPrune = true; + reason = "unlisted since " + unlistedSince + " and missing since " + missingSince; + } + } + } + if (shouldDelete) { addIfMissing(filesToDelete, metadataFile); logger.debug(" - {} ({})", metadataFile.getFileName(), reason); @@ -436,16 +519,24 @@ private void addIfMissing(List files, Path file) { } } - private boolean matchesUnlistedSince(String unlistedSince, Instant threshold) { - if (unlistedSince == null || unlistedSince.isBlank()) { + private boolean matchesSince(String since, Instant threshold) { + if (since == null || since.isBlank()) { return false; } - String value = unlistedSince.trim(); + String value = since.trim(); try { return !Instant.parse(value).isAfter(threshold); } catch (RuntimeException ignored) { - logger.debug("Ignoring metadata with non-instant unlisted_since value: {}", value); + // fall through + } + try { + return !LocalDate.parse(value) + .atStartOfDay(ZoneOffset.UTC) + .toInstant() + .isAfter(threshold); + } catch (RuntimeException ignored) { + logger.debug("Ignoring metadata with unparseable since value: {}", value); return false; } } @@ -538,6 +629,8 @@ private static class CleanStats { int invalidFiles = 0; int oldEaReleases = 0; int unlistedReleases = 0; + int missingReleases = 0; + int unlistedMissingReleases = 0; int orphanedChecksums = 0; int errors = 0; } diff --git a/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java b/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java index 277764f..b9e7432 100644 --- a/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java +++ b/src/test/java/dev/jbang/jdkdb/CleanCommandTest.java @@ -5,6 +5,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.time.LocalDate; import java.time.temporal.ChronoUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -132,12 +133,160 @@ void pruneInvalidMovesMetadataToPruneDir() throws Exception { assertThat(pruneRoot.resolve("temurin/invalid.json")).exists(); } + @Test + void pruneMissingWithInvalidSinceReturnsError() { + int exit = new CommandLine(new CleanCommand()).execute("--prune-missing", "not-a-date"); + + assertThat(exit).isEqualTo(1); + } + + @Test + void pruneMissingWithDateOnlyStringPrunesCorrectly() throws Exception { + Path metadataRoot = tempDir.resolve("metadata"); + Path checksumRoot = tempDir.resolve("checksums"); + Path pruneRoot = tempDir.resolve("pruned"); + Path distroMetadata = metadataRoot.resolve("temurin"); + Files.createDirectories(distroMetadata); + Files.createDirectories(checksumRoot.resolve("temurin")); + + // Date-only format (no time component) as produced by the download manager + String oldDate = LocalDate.now().minusDays(10).toString(); + String recentDate = LocalDate.now().minusDays(2).toString(); + + Path oldMissing = distroMetadata.resolve("old-missing.json"); + Path recentMissing = distroMetadata.resolve("recent-missing.json"); + Files.writeString(oldMissing, metadataJson("temurin", "old-missing", null, oldDate)); + Files.writeString(recentMissing, metadataJson("temurin", "recent-missing", null, recentDate)); + + int exit = new CommandLine(new CleanCommand()) + .execute( + "--metadata-dir", + metadataRoot.toString(), + "--checksum-dir", + checksumRoot.toString(), + "--prune-dir", + pruneRoot.toString(), + "--prune-missing", + "5d"); + + assertThat(exit).isZero(); + assertThat(oldMissing).doesNotExist(); + assertThat(pruneRoot.resolve("temurin/old-missing.json")).exists(); + assertThat(recentMissing).exists(); + } + + @Test + void pruneMissingWithDurationOnlyPrunesOlderMarkers() throws Exception { + Path metadataRoot = tempDir.resolve("metadata"); + Path checksumRoot = tempDir.resolve("checksums"); + Path pruneRoot = tempDir.resolve("pruned"); + Path distroMetadata = metadataRoot.resolve("temurin"); + Files.createDirectories(distroMetadata); + Files.createDirectories(checksumRoot.resolve("temurin")); + + String oldDate = Instant.now() + .minus(10, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + String recentDate = Instant.now() + .minus(2, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + + Path oldMissing = distroMetadata.resolve("old-missing.json"); + Path recentMissing = distroMetadata.resolve("recent-missing.json"); + Path notMissing = distroMetadata.resolve("not-missing.json"); + Files.writeString(oldMissing, metadataJson("temurin", "old-missing", null, oldDate)); + Files.writeString(recentMissing, metadataJson("temurin", "recent-missing", null, recentDate)); + Files.writeString(notMissing, metadataJson("temurin", "not-missing", null, null)); + + int exit = new CommandLine(new CleanCommand()) + .execute( + "--metadata-dir", + metadataRoot.toString(), + "--checksum-dir", + checksumRoot.toString(), + "--prune-dir", + pruneRoot.toString(), + "--prune-missing", + "5d"); + + assertThat(exit).isZero(); + assertThat(oldMissing).doesNotExist(); + assertThat(pruneRoot.resolve("temurin/old-missing.json")).exists(); + assertThat(recentMissing).exists(); + assertThat(notMissing).exists(); + } + + @Test + void pruneUnlistedMissingWithInvalidSinceReturnsError() { + int exit = new CommandLine(new CleanCommand()).execute("--prune-unlisted-missing", "not-a-date"); + + assertThat(exit).isEqualTo(1); + } + + @Test + void pruneUnlistedMissingOnlyPrunesWhenBothMatch() throws Exception { + Path metadataRoot = tempDir.resolve("metadata"); + Path checksumRoot = tempDir.resolve("checksums"); + Path pruneRoot = tempDir.resolve("pruned"); + Path distroMetadata = metadataRoot.resolve("temurin"); + Files.createDirectories(distroMetadata); + Files.createDirectories(checksumRoot.resolve("temurin")); + + String oldDate = Instant.now() + .minus(10, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + String recentDate = Instant.now() + .minus(2, ChronoUnit.DAYS) + .truncatedTo(ChronoUnit.SECONDS) + .toString(); + + // Both unlisted_since and missing_since are old -> should be pruned + Path bothOld = distroMetadata.resolve("both-old.json"); + // Only unlisted_since is old -> should NOT be pruned + Path onlyUnlisted = distroMetadata.resolve("only-unlisted.json"); + // Only missing_since is old -> should NOT be pruned + Path onlyMissing = distroMetadata.resolve("only-missing.json"); + // Both are recent -> should NOT be pruned + Path bothRecent = distroMetadata.resolve("both-recent.json"); + + Files.writeString(bothOld, metadataJson("temurin", "both-old", oldDate, oldDate)); + Files.writeString(onlyUnlisted, metadataJson("temurin", "only-unlisted", oldDate, recentDate)); + Files.writeString(onlyMissing, metadataJson("temurin", "only-missing", recentDate, oldDate)); + Files.writeString(bothRecent, metadataJson("temurin", "both-recent", recentDate, recentDate)); + + int exit = new CommandLine(new CleanCommand()) + .execute( + "--metadata-dir", + metadataRoot.toString(), + "--checksum-dir", + checksumRoot.toString(), + "--prune-dir", + pruneRoot.toString(), + "--prune-unlisted-missing", + "5d"); + + assertThat(exit).isZero(); + assertThat(bothOld).doesNotExist(); + assertThat(pruneRoot.resolve("temurin/both-old.json")).exists(); + assertThat(onlyUnlisted).exists(); + assertThat(onlyMissing).exists(); + assertThat(bothRecent).exists(); + } + private String metadataJson(String distro, String filename, String unlistedSince) { + return metadataJson(distro, filename, unlistedSince, null); + } + + private String metadataJson(String distro, String filename, String unlistedSince, String missingSince) { String unlisted = unlistedSince == null ? "" : "\n \"unlisted_since\": \"" + unlistedSince + "\","; + String missing = missingSince == null ? "" : "\n \"missing_since\": \"" + missingSince + "\","; return """ { \"distro\": \"%s\", - \"filename\": \"%s\",%s + \"filename\": \"%s\",%s%s \"version\": \"21.0.1\", \"java_version\": \"21\", \"release_type\": \"ga\", @@ -148,6 +297,6 @@ private String metadataJson(String distro, String filename, String unlistedSince \"image_type\": \"jdk\", \"url\": \"https://example.com/%s.tar.gz\" } - """.formatted(distro, filename, unlisted, filename); + """.formatted(distro, filename, unlisted, missing, filename); } }