diff --git a/README.md b/README.md index df512b9..bc999db 100644 --- a/README.md +++ b/README.md @@ -296,8 +296,7 @@ Http connection timeout can be set in the plugin configuration. This configurati ``` 30 - 45 - 100 + 300 ``` @@ -397,11 +396,7 @@ The following configuration options can be set via environment variables. RESPONSE_TIMEOUT_SECONDS - Determines the timeout until arrival of a response from the DOCKER_HOST, default is 45s. - - - DOWNLOAD_IMAGE_TIMEOUT_SECONDS - Determines the timeout for an image pull to be completed, default is 300s. + Determines the timeout until arrival of a response from the DOCKER_HOST, default is 300s. diff --git a/docker-versions-maven-plugin-usage/pom.xml b/docker-versions-maven-plugin-usage/pom.xml index 6b41744..62f488b 100644 --- a/docker-versions-maven-plugin-usage/pom.xml +++ b/docker-versions-maven-plugin-usage/pom.xml @@ -24,7 +24,7 @@ com.github.cafapi.plugins.docker.versions docker-versions - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT docker-versions-maven-plugin-usage @@ -40,7 +40,7 @@ com.github.cafapi.plugins.docker.versions docker-versions-maven-plugin - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT populate-project-registry @@ -70,8 +70,9 @@ 3-management - cafapi/opensuse-jre8 - 3.9.3 + cafapi/opensuse-jre21 + 7.0.2 + sha256:d1b8c3467a16cab3aa81e35a3db364f8d94f7d328b53f5aa8dd5b0059ea274b2 diff --git a/docker-versions-maven-plugin/pom.xml b/docker-versions-maven-plugin/pom.xml index 99a0bb2..2902d27 100644 --- a/docker-versions-maven-plugin/pom.xml +++ b/docker-versions-maven-plugin/pom.xml @@ -24,7 +24,7 @@ com.github.cafapi.plugins.docker.versions docker-versions - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT docker-versions-maven-plugin diff --git a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/docker/client/DockerRestClient.java b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/docker/client/DockerRestClient.java index 81e1466..5645ee8 100644 --- a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/docker/client/DockerRestClient.java +++ b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/docker/client/DockerRestClient.java @@ -22,6 +22,8 @@ import com.github.dockerjava.api.command.PullImageResultCallback; import com.github.dockerjava.api.exception.NotFoundException; import com.github.dockerjava.api.model.AuthConfig; +import com.github.dockerjava.api.model.PullResponseItem; +import com.github.dockerjava.api.model.ResponseItem; import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientConfig; import com.github.dockerjava.core.DockerClientImpl; @@ -32,8 +34,12 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Map; import java.util.Optional; -import java.util.concurrent.TimeUnit; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; @@ -44,7 +50,6 @@ public final class DockerRestClient { private static final Logger LOGGER = LoggerFactory.getLogger(DockerRestClient.class); - private final long downloadImageTimeout; private final DockerClient dockerClient; public DockerRestClient(final HttpConfiguration httpConfiguration, final String dockerHost) @@ -71,7 +76,6 @@ public DockerRestClient(final HttpConfiguration httpConfiguration, final String .responseTimeout(Duration.ofSeconds(httpConfig.getResponseTimout())) .build(); - this.downloadImageTimeout = httpConfig.getDownloadImageTimout(); this.dockerClient = DockerClientImpl.getInstance(config, httpClient); } @@ -88,7 +92,7 @@ public Optional findImage(final String imageName) } } - public boolean pullImage( + public void pullImage( final String repository, final String tag, final AuthConfig authConfig @@ -102,17 +106,91 @@ public boolean pullImage( } final PullImageResultCallback callback = new PullImageResultCallback() { + + private static final double ONE_GB = 1_000_000_000.0; + private static final double ONE_MB = 1_000_000.0; + private final Map layerBytes = new ConcurrentHashMap<>(); + private final Map totalBytes = new ConcurrentHashMap<>(); + private final Set countedLayers = ConcurrentHashMap.newKeySet(); + private final Map lastStatusByLayer = new ConcurrentHashMap<>(); + private final AtomicLong knownTotalBytesExpected = new AtomicLong(0L); + private final AtomicInteger pullPercentage = new AtomicInteger(0); + @Override - public void onError(final Throwable throwable) { + public void onError(final Throwable throwable) + { LOGGER.error("Error pulling image {}:{} ", repository, tag, throwable); super.onError(throwable); } + + @Override + public void onNext(final PullResponseItem item) { + super.onNext(item); + + if (item.getId() != null && item.getProgressDetail() != null) { + final String layerId = item.getId(); + final ResponseItem.ProgressDetail progressDetail = item.getProgressDetail(); + final String status = item.getStatus(); + final String previous = lastStatusByLayer.get(layerId); + lastStatusByLayer.put(layerId, status); + final Long layerCurrent = Optional.ofNullable(progressDetail.getCurrent()).orElse(0L); + final Long layerTotal = Optional.ofNullable(progressDetail.getTotal()).orElse(0L); + + if ("Extracting".equalsIgnoreCase(status)) { + final long pulledBytes = totalBytes.getOrDefault(layerId, 0L); + if (!"Extracting".equalsIgnoreCase(previous)) { + LOGGER.info("Extracting Layer {} ({})", layerId, stringifyBytes(pulledBytes)); + } + return; + } else if ("Downloading".equalsIgnoreCase(status)) { + if (layerCurrent > 0) { + layerBytes.put(layerId, layerCurrent); + } + + // Add to aggregate total only on first progress for this layer + if (layerTotal > 0 && countedLayers.add(layerId)) { + LOGGER.info("Pulling Layer {} ({})", layerId, stringifyBytes(layerTotal)); + totalBytes.put(layerId, layerTotal); + knownTotalBytesExpected.addAndGet(layerTotal); + // If a new layer is received pull percentage is now invalid. + pullPercentage.set(0); + } + + final long bytesDownloaded = layerBytes.values().stream().mapToLong(Long::longValue).sum(); + if (knownTotalBytesExpected.get() > 0) { + final int percent = (int) Math.round(bytesDownloaded * 100.0 / knownTotalBytesExpected.get()); + final int progress = (percent / 10) * 10; + if (LOGGER.isInfoEnabled() && progress > pullPercentage.get()) { + pullPercentage.set(progress); + LOGGER.info("Pulling layer {} ({}) {}% of {} layer{} completed, {} of {}", + layerId, + stringifyBytes(totalBytes.get(layerId)), + progress, + countedLayers.size(), + (countedLayers.size() > 1)?"s":"", + stringifyBytes(bytesDownloaded), + stringifyBytes(knownTotalBytesExpected.get())); + } + } + } + } + } + + private String stringifyBytes(final long bytes) + { + if (bytes >= ONE_GB) { + return String.format("%.2fGB", (bytes / ONE_GB)); + } else if (bytes >= ONE_MB) { + return String.format("%.2fMB", (bytes / ONE_MB)); + } + return bytes + "bytes"; + } }; - return pullCommand + pullCommand .withTag(tag) .exec(callback) - .awaitCompletion(downloadImageTimeout, TimeUnit.SECONDS); + .awaitCompletion(); } public void tagImage( diff --git a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/HttpConfiguration.java b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/HttpConfiguration.java index ab4cd62..de2321e 100644 --- a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/HttpConfiguration.java +++ b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/HttpConfiguration.java @@ -20,8 +20,7 @@ public final class HttpConfiguration { private static final int CONNECTION_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("CONNECTION_TIMEOUT_SECONDS", "30"); - private static final int RESPONSE_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("RESPONSE_TIMEOUT_SECONDS", "45"); - private static final long DOWNLOAD_IMAGE_TIMEOUT_SECONDS = getLongPropertyOrEnvVar("DOWNLOAD_IMAGE_TIMEOUT_SECONDS", "300"); + private static final int RESPONSE_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("RESPONSE_TIMEOUT_SECONDS", "300"); @Parameter private int connectionTimout = CONNECTION_TIMEOUT_SECONDS; @@ -29,9 +28,6 @@ public final class HttpConfiguration @Parameter() private int responseTimout = RESPONSE_TIMEOUT_SECONDS; - @Parameter() - private long downloadImageTimout = DOWNLOAD_IMAGE_TIMEOUT_SECONDS; - public int getConnectionTimout() { return connectionTimout; @@ -42,18 +38,12 @@ public int getResponseTimout() return responseTimout; } - public long getDownloadImageTimout() - { - return downloadImageTimout; - } - @Override public String toString() { return "HttpConfiguration [ " + "connectionTimout=" + connectionTimout + "s, " - + "responseTimout=" + responseTimout + "s, " - + "downloadImageTimout=" + downloadImageTimout + "s ]"; + + "responseTimout=" + responseTimout + "s"; } private static int getIntPropertyOrEnvVar(final String key, final String defaultValue) @@ -62,12 +52,6 @@ private static int getIntPropertyOrEnvVar(final String key, final String default return Integer.parseInt(propertyValue); } - private static long getLongPropertyOrEnvVar(final String key, final String defaultValue) - { - final String propertyValue = getPropertyOrEnvVar(key, defaultValue); - return Long.parseLong(propertyValue); - } - private static String getPropertyOrEnvVar(final String key, final String defaultValue) { final String propertyValue = System.getProperty(key); diff --git a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/PopulateProjectRegistryMojo.java b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/PopulateProjectRegistryMojo.java index df5a19c..d2d5718 100644 --- a/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/PopulateProjectRegistryMojo.java +++ b/docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/PopulateProjectRegistryMojo.java @@ -155,17 +155,13 @@ private InspectImageResponse pullImage(final ImageMoniker imageMoniker) { final AuthConfig authConfig = AuthConfigHelper.getAuthConfig(settings, imageMoniker.getRegistry()); - final boolean imagePullCompleted = dockerClient.pullImage( + dockerClient.pullImage( imageMoniker.getFullImageNameWithoutTag(), imageMoniker.getTag(), authConfig); final String imageName = imageMoniker.getFullImageNameWithTag(); - if (!imagePullCompleted) { - throw new ImagePullException("Image was not pulled: " + imageName); - } - LOGGER.debug("Pulled image '{}', verify that it is now present...", imageName); final Optional image = dockerClient.findImage(imageName); if (!image.isPresent()) { diff --git a/pom.xml b/pom.xml index 621bba6..95789cd 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ com.github.cafapi.plugins.docker.versions docker-versions - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT pom docker-versions diff --git a/release-notes-2.1.0.md b/release-notes-2.1.0.md deleted file mode 100644 index ac41e1d..0000000 --- a/release-notes-2.1.0.md +++ /dev/null @@ -1,10 +0,0 @@ -!not-ready-for-release! - -#### Version Number -${version-number} - -#### New Features -- **US984062**: Added a new `skipPull` config param to skip pulling an image before retagging it. This could be used when working with developer images. - -#### Known Issues -- None diff --git a/release-notes-3.0.0.md b/release-notes-3.0.0.md new file mode 100644 index 0000000..f80e466 --- /dev/null +++ b/release-notes-3.0.0.md @@ -0,0 +1,16 @@ +!not-ready-for-release! + +#### Version Number +${version-number} + +#### Breaking Changes +- **D1200007**: Changes in image pull timeouts. Image pull will only time out if no response is received from the Docker Daemon +within the configured time. + * DOWNLOAD_IMAGE_TIMEOUT_SECONDS : No longer used. + * RESPONSE_TIMEOUT_SECONDS : default increased to 300 seconds. + +#### New Features +- **US984062**: Added a new `skipPull` config param to skip pulling an image before retagging it. This could be used when working with developer images. + +#### Known Issues +- None