Skip to content

feat: added more prune commands to clean command #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
quintesse merged 1 commit into from
Aug 13, 2026
Merged

feat: added more prune commands to clean command #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 100 additions & 7 deletions src/main/java/dev/jbang/jdkdb/CleanCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
Expand Down Expand Up @@ -76,6 +78,22 @@ public enum IncompleteType {
"Prune metadata files with unlisted_since up to SINCE. SINCE uses duration format [number][d|w|m|y].")
private String pruneUnlisted;

@Option(
names = {"--prune-missing"},
arity = "0..1",
paramLabel = "SINCE",
description =
"Prune metadata files with missing_since up to SINCE. SINCE uses duration format [number][d|w|m|y].")
private String pruneMissing;

@Option(
names = {"--prune-unlisted-missing"},
arity = "0..1",
paramLabel = "SINCE",
description =
"Prune metadata files where both unlisted_since and missing_since are up to SINCE. SINCE uses duration format [number][d|w|m|y].")
private String pruneUnlistedMissing;

@Option(
names = {"-p", "--prune-dir"},
description = "Directory to move pruned files into (default: db/pruned)",
Expand All @@ -100,6 +118,8 @@ public Integer call() throws Exception {
&& !pruneInvalid
&& pruneEa == null
&& pruneUnlisted == null
&& pruneMissing == null
&& pruneUnlistedMissing == null
&& !removeOrphanedChecksums
&& !dryRun) {
logger.info("No options specified, using defaults: --remove-incomplete=all --prune-ea=6m --dry-run");
Expand All @@ -121,6 +141,8 @@ public Integer call() throws Exception {
logger.info(" Prune invalid: {}", pruneInvalid);
logger.info(" Prune EA: {}", (pruneEa != null ? pruneEa : "disabled"));
logger.info(" Prune unlisted: {}", (pruneUnlisted != null ? pruneUnlisted : "disabled"));
logger.info(" Prune missing: {}", (pruneMissing != null ? pruneMissing : "disabled"));
logger.info(" Prune unlisted+missing: {}", (pruneUnlistedMissing != null ? pruneUnlistedMissing : "disabled"));
logger.info(" Remove orphaned checksums: {}", removeOrphanedChecksums);
logger.info(" Dry run: {}", dryRun);
logger.info("");
Expand Down Expand Up @@ -158,12 +180,43 @@ public Integer call() throws Exception {
logger.info("");
}

Instant pruneMissingThreshold = null;
if (pruneMissing != null) {
Duration duration = MetadataUtils.parseDuration(pruneMissing);
if (duration == null) {
logger.error("Error: Invalid --prune-missing SINCE value: {}", pruneMissing);
logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)");
return 1;
}
pruneMissingThreshold = Instant.now().minus(duration);
logger.info("Pruning missing metadata up to: {} (<= {})", pruneMissing, pruneMissingThreshold);
logger.info("");
}

Instant pruneUnlistedMissingThreshold = null;
if (pruneUnlistedMissing != null) {
Duration duration = MetadataUtils.parseDuration(pruneUnlistedMissing);
if (duration == null) {
logger.error("Error: Invalid --prune-unlisted-missing SINCE value: {}", pruneUnlistedMissing);
logger.error("Expected format: [number][d|w|m|y] (e.g., 30d, 3w, 6m, 1y)");
return 1;
}
pruneUnlistedMissingThreshold = Instant.now().minus(duration);
logger.info(
"Pruning unlisted+missing metadata up to: {} (<= {})",
pruneUnlistedMissing,
pruneUnlistedMissingThreshold);
logger.info("");
}

// Collect files to delete/prune
final CleanStats stats = new CleanStats();
final List<Path> filesToDelete = new ArrayList<>();
final List<Path> filesToPrune = new ArrayList<>();
final Instant finalPruneEaThreshold = pruneEaThreshold;
final Instant finalPruneUnlistedThreshold = pruneUnlistedThreshold;
final Instant finalPruneMissingThreshold = pruneMissingThreshold;
final Instant finalPruneUnlistedMissingThreshold = pruneUnlistedMissingThreshold;

List<JdkMetadata> metadataList = MetadataUtils.collectAllMetadata(distroDir, 2, true, true);
for (JdkMetadata metadata : metadataList) {
Expand All @@ -174,7 +227,9 @@ public Integer call() throws Exception {
filesToDelete,
filesToPrune,
finalPruneEaThreshold,
finalPruneUnlistedThreshold);
finalPruneUnlistedThreshold,
finalPruneMissingThreshold,
finalPruneUnlistedMissingThreshold);
} catch (IOException e) {
logger.error("Failed to process {}: {}", metadata.metadataFile().getFileName(), e.getMessage());
stats.errors++;
Expand All @@ -199,6 +254,8 @@ public Integer call() throws Exception {
logger.info("Invalid files: {}", stats.invalidFiles);
logger.info("Old EA releases: {}", stats.oldEaReleases);
logger.info("Unlisted releases: {}", stats.unlistedReleases);
logger.info("Missing releases: {}", stats.missingReleases);
logger.info("Unlisted+missing releases: {}", stats.unlistedMissingReleases);
logger.info("Orphaned checksum files: {}", stats.orphanedChecksums);
logger.info("Errors: {}", stats.errors);
logger.info("");
Expand Down Expand Up @@ -275,7 +332,9 @@ private void processMetadataFile(
List<Path> filesToDelete,
List<Path> filesToPrune,
Instant pruneEaThreshold,
Instant pruneUnlistedThreshold)
Instant pruneUnlistedThreshold,
Instant pruneMissingThreshold,
Instant pruneUnlistedMissingThreshold)
throws IOException {
stats.totalFiles++;

Expand Down Expand Up @@ -338,7 +397,7 @@ private void processMetadataFile(

if (pruneUnlistedThreshold != null && !shouldDelete) {
String unlistedSince = metadata.getUnlistedSince();
if (matchesUnlistedSince(unlistedSince, pruneUnlistedThreshold)) {
if (matchesSince(unlistedSince, pruneUnlistedThreshold)) {
stats.unlistedReleases++;
if (!shouldPrune) {
shouldPrune = true;
Expand All @@ -347,6 +406,30 @@ private void processMetadataFile(
}
}

if (pruneMissingThreshold != null && !shouldDelete) {
String missingSince = metadata.getMissingSince();
if (matchesSince(missingSince, pruneMissingThreshold)) {
stats.missingReleases++;
if (!shouldPrune) {
shouldPrune = true;
reason = "missing since " + missingSince;
}
}
}

if (pruneUnlistedMissingThreshold != null && !shouldDelete) {
String unlistedSince = metadata.getUnlistedSince();
String missingSince = metadata.getMissingSince();
if (matchesSince(unlistedSince, pruneUnlistedMissingThreshold)
&& matchesSince(missingSince, pruneUnlistedMissingThreshold)) {
stats.unlistedMissingReleases++;
if (!shouldPrune) {
shouldPrune = true;
reason = "unlisted since " + unlistedSince + " and missing since " + missingSince;
}
}
}

if (shouldDelete) {
addIfMissing(filesToDelete, metadataFile);
logger.debug(" - {} ({})", metadataFile.getFileName(), reason);
Expand Down Expand Up @@ -436,16 +519,24 @@ private void addIfMissing(List<Path> files, Path file) {
}
}

private boolean matchesUnlistedSince(String unlistedSince, Instant threshold) {
if (unlistedSince == null || unlistedSince.isBlank()) {
private boolean matchesSince(String since, Instant threshold) {
if (since == null || since.isBlank()) {
return false;
}

String value = unlistedSince.trim();
String value = since.trim();
try {
return !Instant.parse(value).isAfter(threshold);
} catch (RuntimeException ignored) {
logger.debug("Ignoring metadata with non-instant unlisted_since value: {}", value);
// fall through
}
try {
return !LocalDate.parse(value)
.atStartOfDay(ZoneOffset.UTC)
.toInstant()
.isAfter(threshold);
} catch (RuntimeException ignored) {
logger.debug("Ignoring metadata with unparseable since value: {}", value);
return false;
}
}
Expand Down Expand Up @@ -538,6 +629,8 @@ private static class CleanStats {
int invalidFiles = 0;
int oldEaReleases = 0;
int unlistedReleases = 0;
int missingReleases = 0;
int unlistedMissingReleases = 0;
int orphanedChecksums = 0;
int errors = 0;
}
Expand Down
153 changes: 151 additions & 2 deletions src/test/java/dev/jbang/jdkdb/CleanCommandTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
Expand Down Expand Up @@ -132,12 +133,160 @@ void pruneInvalidMovesMetadataToPruneDir() throws Exception {
assertThat(pruneRoot.resolve("temurin/invalid.json")).exists();
}

@Test
void pruneMissingWithInvalidSinceReturnsError() {
int exit = new CommandLine(new CleanCommand()).execute("--prune-missing", "not-a-date");

assertThat(exit).isEqualTo(1);
}

@Test
void pruneMissingWithDateOnlyStringPrunesCorrectly() throws Exception {
Path metadataRoot = tempDir.resolve("metadata");
Path checksumRoot = tempDir.resolve("checksums");
Path pruneRoot = tempDir.resolve("pruned");
Path distroMetadata = metadataRoot.resolve("temurin");
Files.createDirectories(distroMetadata);
Files.createDirectories(checksumRoot.resolve("temurin"));

// Date-only format (no time component) as produced by the download manager
String oldDate = LocalDate.now().minusDays(10).toString();
String recentDate = LocalDate.now().minusDays(2).toString();

Path oldMissing = distroMetadata.resolve("old-missing.json");
Path recentMissing = distroMetadata.resolve("recent-missing.json");
Files.writeString(oldMissing, metadataJson("temurin", "old-missing", null, oldDate));
Files.writeString(recentMissing, metadataJson("temurin", "recent-missing", null, recentDate));

int exit = new CommandLine(new CleanCommand())
.execute(
"--metadata-dir",
metadataRoot.toString(),
"--checksum-dir",
checksumRoot.toString(),
"--prune-dir",
pruneRoot.toString(),
"--prune-missing",
"5d");

assertThat(exit).isZero();
assertThat(oldMissing).doesNotExist();
assertThat(pruneRoot.resolve("temurin/old-missing.json")).exists();
assertThat(recentMissing).exists();
}

@Test
void pruneMissingWithDurationOnlyPrunesOlderMarkers() throws Exception {
Path metadataRoot = tempDir.resolve("metadata");
Path checksumRoot = tempDir.resolve("checksums");
Path pruneRoot = tempDir.resolve("pruned");
Path distroMetadata = metadataRoot.resolve("temurin");
Files.createDirectories(distroMetadata);
Files.createDirectories(checksumRoot.resolve("temurin"));

String oldDate = Instant.now()
.minus(10, ChronoUnit.DAYS)
.truncatedTo(ChronoUnit.SECONDS)
.toString();
String recentDate = Instant.now()
.minus(2, ChronoUnit.DAYS)
.truncatedTo(ChronoUnit.SECONDS)
.toString();

Path oldMissing = distroMetadata.resolve("old-missing.json");
Path recentMissing = distroMetadata.resolve("recent-missing.json");
Path notMissing = distroMetadata.resolve("not-missing.json");
Files.writeString(oldMissing, metadataJson("temurin", "old-missing", null, oldDate));
Files.writeString(recentMissing, metadataJson("temurin", "recent-missing", null, recentDate));
Files.writeString(notMissing, metadataJson("temurin", "not-missing", null, null));

int exit = new CommandLine(new CleanCommand())
.execute(
"--metadata-dir",
metadataRoot.toString(),
"--checksum-dir",
checksumRoot.toString(),
"--prune-dir",
pruneRoot.toString(),
"--prune-missing",
"5d");

assertThat(exit).isZero();
assertThat(oldMissing).doesNotExist();
assertThat(pruneRoot.resolve("temurin/old-missing.json")).exists();
assertThat(recentMissing).exists();
assertThat(notMissing).exists();
}

@Test
void pruneUnlistedMissingWithInvalidSinceReturnsError() {
int exit = new CommandLine(new CleanCommand()).execute("--prune-unlisted-missing", "not-a-date");

assertThat(exit).isEqualTo(1);
}

@Test
void pruneUnlistedMissingOnlyPrunesWhenBothMatch() throws Exception {
Path metadataRoot = tempDir.resolve("metadata");
Path checksumRoot = tempDir.resolve("checksums");
Path pruneRoot = tempDir.resolve("pruned");
Path distroMetadata = metadataRoot.resolve("temurin");
Files.createDirectories(distroMetadata);
Files.createDirectories(checksumRoot.resolve("temurin"));

String oldDate = Instant.now()
.minus(10, ChronoUnit.DAYS)
.truncatedTo(ChronoUnit.SECONDS)
.toString();
String recentDate = Instant.now()
.minus(2, ChronoUnit.DAYS)
.truncatedTo(ChronoUnit.SECONDS)
.toString();

// Both unlisted_since and missing_since are old -> should be pruned
Path bothOld = distroMetadata.resolve("both-old.json");
// Only unlisted_since is old -> should NOT be pruned
Path onlyUnlisted = distroMetadata.resolve("only-unlisted.json");
// Only missing_since is old -> should NOT be pruned
Path onlyMissing = distroMetadata.resolve("only-missing.json");
// Both are recent -> should NOT be pruned
Path bothRecent = distroMetadata.resolve("both-recent.json");

Files.writeString(bothOld, metadataJson("temurin", "both-old", oldDate, oldDate));
Files.writeString(onlyUnlisted, metadataJson("temurin", "only-unlisted", oldDate, recentDate));
Files.writeString(onlyMissing, metadataJson("temurin", "only-missing", recentDate, oldDate));
Files.writeString(bothRecent, metadataJson("temurin", "both-recent", recentDate, recentDate));

int exit = new CommandLine(new CleanCommand())
.execute(
"--metadata-dir",
metadataRoot.toString(),
"--checksum-dir",
checksumRoot.toString(),
"--prune-dir",
pruneRoot.toString(),
"--prune-unlisted-missing",
"5d");

assertThat(exit).isZero();
assertThat(bothOld).doesNotExist();
assertThat(pruneRoot.resolve("temurin/both-old.json")).exists();
assertThat(onlyUnlisted).exists();
assertThat(onlyMissing).exists();
assertThat(bothRecent).exists();
}

private String metadataJson(String distro, String filename, String unlistedSince) {
return metadataJson(distro, filename, unlistedSince, null);
}

private String metadataJson(String distro, String filename, String unlistedSince, String missingSince) {
String unlisted = unlistedSince == null ? "" : "\n \"unlisted_since\": \"" + unlistedSince + "\",";
String missing = missingSince == null ? "" : "\n \"missing_since\": \"" + missingSince + "\",";
return """
{
\"distro\": \"%s\",
\"filename\": \"%s\",%s
\"filename\": \"%s\",%s%s
\"version\": \"21.0.1\",
\"java_version\": \"21\",
\"release_type\": \"ga\",
Expand All @@ -148,6 +297,6 @@ private String metadataJson(String distro, String filename, String unlistedSince
\"image_type\": \"jdk\",
\"url\": \"https://example.com/%s.tar.gz\"
}
""".formatted(distro, filename, unlisted, filename);
""".formatted(distro, filename, unlisted, missing, filename);
}
}