US1200007: Removed download timeout, added pull progress logging

README.md

@@ -296,8 +296,7 @@ Http connection timeout can be set in the plugin configuration. This configurati
 ```
 <httpConfiguration>
     <connectionTimout>30</connectionTimout>
-    <responseTimout>45</responseTimout>
-    <downloadImageTimout>100</downloadImageTimout>
+    <responseTimout>300</responseTimout>
 </httpConfiguration>
 ```
 
@@ -397,11 +396,7 @@ The following configuration options can be set via environment variables.
   </tr>
   <tr>
     <td> RESPONSE_TIMEOUT_SECONDS </td>
-    <td> Determines the timeout until arrival of a response from the DOCKER_HOST, default is 45s. </td>
-  </tr>
-  <tr>
-    <td> DOWNLOAD_IMAGE_TIMEOUT_SECONDS </td>
-    <td> Determines the timeout for an image pull to be completed, default is 300s. </td>
+    <td> Determines the timeout until arrival of a response from the DOCKER_HOST, default is 300s. </td>
   </tr>
 </table>
 

docker-versions-maven-plugin-usage/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>com.github.cafapi.plugins.docker.versions</groupId>
         <artifactId>docker-versions</artifactId>
-        <version>2.1.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>docker-versions-maven-plugin-usage</artifactId>
@@ -40,7 +40,7 @@
             <plugin>
                 <groupId>com.github.cafapi.plugins.docker.versions</groupId>
                 <artifactId>docker-versions-maven-plugin</artifactId>
-                <version>2.1.0-SNAPSHOT</version>
+                <version>3.0.0-SNAPSHOT</version>
                 <executions>
                     <execution>
                         <id>populate-project-registry</id>
@@ -70,8 +70,9 @@
                             <tag>3-management</tag>
                         </image>
                         <image>
-                            <repository>cafapi/opensuse-jre8</repository>
-                            <tag>3.9.3</tag>
+                            <repository>cafapi/opensuse-jre21</repository>
+                            <tag>7.0.2</tag>
+                            <digest>sha256:d1b8c3467a16cab3aa81e35a3db364f8d94f7d328b53f5aa8dd5b0059ea274b2</digest>
                         </image>
                     </imageManagement>
                 </configuration>

docker-versions-maven-plugin/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>com.github.cafapi.plugins.docker.versions</groupId>
         <artifactId>docker-versions</artifactId>
-        <version>2.1.0-SNAPSHOT</version>
+        <version>3.0.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>docker-versions-maven-plugin</artifactId>

docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/docker/client/DockerRestClient.java

@@ -22,6 +22,8 @@
 import com.github.dockerjava.api.command.PullImageResultCallback;
 import com.github.dockerjava.api.exception.NotFoundException;
 import com.github.dockerjava.api.model.AuthConfig;
+import com.github.dockerjava.api.model.PullResponseItem;
+import com.github.dockerjava.api.model.ResponseItem;
 import com.github.dockerjava.core.DefaultDockerClientConfig;
 import com.github.dockerjava.core.DockerClientConfig;
 import com.github.dockerjava.core.DockerClientImpl;
@@ -32,8 +34,12 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.util.Map;
 import java.util.Optional;
-import java.util.concurrent.TimeUnit;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.SystemUtils;
@@ -44,7 +50,6 @@ public final class DockerRestClient
 {
     private static final Logger LOGGER = LoggerFactory.getLogger(DockerRestClient.class);
 
-    private final long downloadImageTimeout;
     private final DockerClient dockerClient;
 
     public DockerRestClient(final HttpConfiguration httpConfiguration, final String dockerHost)
@@ -71,7 +76,6 @@ public DockerRestClient(final HttpConfiguration httpConfiguration, final String
             .responseTimeout(Duration.ofSeconds(httpConfig.getResponseTimout()))
             .build();
 
-        this.downloadImageTimeout = httpConfig.getDownloadImageTimout();
         this.dockerClient = DockerClientImpl.getInstance(config, httpClient);
     }
 
@@ -88,7 +92,7 @@ public Optional<InspectImageResponse> findImage(final String imageName)
         }
     }
 
-    public boolean pullImage(
+    public void pullImage(
         final String repository,
         final String tag,
         final AuthConfig authConfig
@@ -102,17 +106,91 @@ public boolean pullImage(
         }
 
         final PullImageResultCallback callback = new PullImageResultCallback() {
+
+            private static final double ONE_GB = 1_000_000_000.0;
+            private static final double ONE_MB = 1_000_000.0;
+            private final Map<String, Long> layerBytes = new ConcurrentHashMap<>();
+            private final Map<String, Long> totalBytes = new ConcurrentHashMap<>();
+            private final Set<String> countedLayers = ConcurrentHashMap.newKeySet();
+            private final Map<String, String> lastStatusByLayer = new ConcurrentHashMap<>();
+            private final AtomicLong knownTotalBytesExpected = new AtomicLong(0L);
+            private final AtomicInteger pullPercentage = new AtomicInteger(0);
+
             @Override
-            public void onError(final Throwable throwable) {
+            public void onError(final Throwable throwable)
+            {
                 LOGGER.error("Error pulling image {}:{} ", repository, tag, throwable);
                 super.onError(throwable);
             }
+
+            @Override
+            public void onNext(final PullResponseItem item) {
+                super.onNext(item);
+
+                if (item.getId() != null && item.getProgressDetail() != null) {
+                    final String layerId = item.getId();
+                    final ResponseItem.ProgressDetail progressDetail = item.getProgressDetail();
+                    final String status = item.getStatus();
+                    final String previous = lastStatusByLayer.get(layerId);
+                    lastStatusByLayer.put(layerId, status);
+                    final Long layerCurrent = Optional.ofNullable(progressDetail.getCurrent()).orElse(0L);
+                    final Long layerTotal = Optional.ofNullable(progressDetail.getTotal()).orElse(0L);
+
+                    if ("Extracting".equalsIgnoreCase(status)) {
+                        final long pulledBytes = totalBytes.getOrDefault(layerId, 0L);
+                        if (!"Extracting".equalsIgnoreCase(previous)) {
+                            LOGGER.info("Extracting Layer {} ({})", layerId, stringifyBytes(pulledBytes));
+                        }
+                        return;
+                    } else if ("Downloading".equalsIgnoreCase(status)) {
+                        if (layerCurrent > 0) {
+                            layerBytes.put(layerId, layerCurrent);
+                        }
+
+                        // Add to aggregate total only on first progress for this layer
+                        if (layerTotal > 0 && countedLayers.add(layerId)) {
+                            LOGGER.info("Pulling Layer {} ({})", layerId, stringifyBytes(layerTotal));
+                            totalBytes.put(layerId, layerTotal);
+                            knownTotalBytesExpected.addAndGet(layerTotal);
+                            // If a new layer is received pull percentage is now invalid.
+                            pullPercentage.set(0);
+                        }
+
+                        final long bytesDownloaded = layerBytes.values().stream().mapToLong(Long::longValue).sum();
+                        if (knownTotalBytesExpected.get() > 0) {
+                            final int percent = (int) Math.round(bytesDownloaded * 100.0 / knownTotalBytesExpected.get());
+                            final int progress = (percent / 10) * 10;
+                            if (LOGGER.isInfoEnabled() && progress > pullPercentage.get()) {
+                                pullPercentage.set(progress);
+                                LOGGER.info("Pulling layer {} ({}) {}% of {} layer{} completed, {} of {}",
+                                    layerId,
+                                    stringifyBytes(totalBytes.get(layerId)),
+                                    progress,
+                                    countedLayers.size(),
+                                    (countedLayers.size() > 1)?"s":"",
+                                    stringifyBytes(bytesDownloaded),
+                                    stringifyBytes(knownTotalBytesExpected.get()));
+                            }
+                        }
+                    }
+                }
+            }
+
+            private String stringifyBytes(final long bytes)
+            {
+                if (bytes >= ONE_GB) {
+                    return String.format("%.2fGB",  (bytes / ONE_GB));
+                } else if (bytes >= ONE_MB) {
+                    return String.format("%.2fMB",  (bytes / ONE_MB));
+                }
+                return bytes + "bytes";
+            }
         };
 
-         return pullCommand
+        pullCommand
             .withTag(tag)
             .exec(callback)
-            .awaitCompletion(downloadImageTimeout, TimeUnit.SECONDS);
+            .awaitCompletion();
     }
 
     public void tagImage(

docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/HttpConfiguration.java

@@ -20,18 +20,14 @@
 public final class HttpConfiguration
 {
     private static final int CONNECTION_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("CONNECTION_TIMEOUT_SECONDS", "30");
-    private static final int RESPONSE_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("RESPONSE_TIMEOUT_SECONDS", "45");
-    private static final long DOWNLOAD_IMAGE_TIMEOUT_SECONDS = getLongPropertyOrEnvVar("DOWNLOAD_IMAGE_TIMEOUT_SECONDS", "300");
+    private static final int RESPONSE_TIMEOUT_SECONDS = getIntPropertyOrEnvVar("RESPONSE_TIMEOUT_SECONDS", "300");
 
     @Parameter
     private int connectionTimout = CONNECTION_TIMEOUT_SECONDS;
 
     @Parameter()
     private int responseTimout = RESPONSE_TIMEOUT_SECONDS;
 
-    @Parameter()
-    private long downloadImageTimout = DOWNLOAD_IMAGE_TIMEOUT_SECONDS;
-
     public int getConnectionTimout()
     {
         return connectionTimout;
@@ -42,18 +38,12 @@ public int getResponseTimout()
         return responseTimout;
     }
 
-    public long getDownloadImageTimout()
-    {
-        return downloadImageTimout;
-    }
-
     @Override
     public String toString()
     {
         return "HttpConfiguration [ "
             + "connectionTimout=" + connectionTimout + "s, "
-            + "responseTimout=" + responseTimout + "s, "
-            + "downloadImageTimout=" + downloadImageTimout + "s ]";
+            + "responseTimout=" + responseTimout + "s";
     }
 
     private static int getIntPropertyOrEnvVar(final String key, final String defaultValue)
@@ -62,12 +52,6 @@ private static int getIntPropertyOrEnvVar(final String key, final String default
         return Integer.parseInt(propertyValue);
     }
 
-    private static long getLongPropertyOrEnvVar(final String key, final String defaultValue)
-    {
-        final String propertyValue = getPropertyOrEnvVar(key, defaultValue);
-        return Long.parseLong(propertyValue);
-    }
-
     private static String getPropertyOrEnvVar(final String key, final String defaultValue)
     {
         final String propertyValue = System.getProperty(key);

docker-versions-maven-plugin/src/main/java/com/github/cafapi/docker_versions/plugins/PopulateProjectRegistryMojo.java

@@ -155,17 +155,13 @@ private InspectImageResponse pullImage(final ImageMoniker imageMoniker)
         {
             final AuthConfig authConfig = AuthConfigHelper.getAuthConfig(settings, imageMoniker.getRegistry());
 
-            final boolean imagePullCompleted = dockerClient.pullImage(
+            dockerClient.pullImage(
                 imageMoniker.getFullImageNameWithoutTag(),
                 imageMoniker.getTag(),
                 authConfig);
 
             final String imageName = imageMoniker.getFullImageNameWithTag();
 
-            if (!imagePullCompleted) {
-                throw new ImagePullException("Image was not pulled: " + imageName);
-            }
-
             LOGGER.debug("Pulled image '{}', verify that it is now present...", imageName);
             final Optional<InspectImageResponse> image = dockerClient.findImage(imageName);
             if (!image.isPresent()) {

pom.xml

@@ -29,7 +29,7 @@
 
     <groupId>com.github.cafapi.plugins.docker.versions</groupId>
     <artifactId>docker-versions</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>3.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>docker-versions</name>

release-notes-3.0.0.md

@@ -0,0 +1,16 @@
+!not-ready-for-release!
+
+#### Version Number
+${version-number}
+
+#### Breaking Changes
+- **D1200007**: Changes in image pull timeouts. Image pull will only time out if no response is received from the Docker Daemon 
+within the configured time.
+  * DOWNLOAD_IMAGE_TIMEOUT_SECONDS : No longer used.
+  * RESPONSE_TIMEOUT_SECONDS : default increased to 300 seconds. 
+
+#### New Features
+- **US984062**: Added a new `skipPull` config param to skip pulling an image before retagging it. This could be used when working with developer images.
+
+#### Known Issues
+- None