Avoid reopening jars per resource request and deleteOnExit per upload

http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectivesSpec.scala

@@ -267,6 +267,35 @@ class FileAndResourceDirectivesSpec extends RoutingSpec with Inspectors with Ins
           1.second.dilated).data.asByteBuffer.getInt shouldEqual 0xCAFEBABE
       }
     }
+    "return the resource content from an archive with metadata taken from the archive entry" in {
+      val route = getFromResource("com/typesafe/config/Config.class")
+
+      def runCheck() =
+        Get() ~> route ~> check {
+          val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated)
+          entity.contentLength shouldEqual entity.data.length
+          header[`Last-Modified`] shouldBe defined
+        }
+
+      runCheck()
+      runCheck() // the archive is shared between requests, so make sure it is still usable afterwards
+    }
+    "return the resource content from an archive when the jar file cache is disabled" in {
+      val route =
+        withSettings(RoutingSettings(system).withUseJarFileCache(false)) {
+          getFromResource("com/typesafe/config/Config.class")
+        }
+
+      def runCheck() =
+        Get() ~> route ~> check {
+          val entity = responseEntity.toStrict(1.second.dilated).awaitResult(1.second.dilated)
+          entity.contentLength shouldEqual entity.data.length
+          entity.data.asByteBuffer.getInt shouldEqual 0xCAFEBABE
+        }
+
+      runCheck()
+      runCheck() // every request opens and closes the jar file of its own, so make sure that is repeatable
+    }
     "return the file content with MediaType 'application/octet-stream' on unknown file extensions" in {
       Get() ~> getFromResource("sample.xyz") ~> check {
         mediaType shouldEqual `application/octet-stream`

http-tests/src/test/scala/org/apache/pekko/http/scaladsl/server/directives/FileUploadDirectivesSpec.scala

@@ -16,7 +16,7 @@ package org.apache.pekko.http.scaladsl.server.directives
 import java.io.File
 import java.nio.file.Files
 
-import scala.concurrent.Future
+import scala.concurrent.{ Await, Future }
 import scala.concurrent.duration._
 
 import org.apache.pekko
@@ -29,6 +29,7 @@ import pekko.testkit._
 import pekko.util.ByteString
 
 import org.scalatest.concurrent.Eventually
+import org.scalatest.time.{ Seconds, Span }
 
 class FileUploadDirectivesSpec extends RoutingSpec with Eventually {
 
@@ -458,6 +459,45 @@ class FileUploadDirectivesSpec extends RoutingSpec with Eventually {
 
     }
 
+    "collect its temporary files in a single directory" in {
+      // the directory is removed by a CoordinatedShutdown task when the actor system terminates, instead of
+      // registering every single uploaded file with `File.deleteOnExit`, which the JVM would remember for the
+      // lifetime of the process
+      val uploadTempFiles = UploadTempFiles(system)
+      val first = uploadTempFiles.create()
+      val second = uploadTempFiles.create()
+      try {
+        first.getParentFile.getName should startWith("pekko-http-uploads")
+        second.getParentFile shouldEqual first.getParentFile
+        (first should not).equal(second)
+      } finally {
+        first.delete()
+        second.delete()
+      }
+    }
+
+    "recreate its temporary directory when it was removed while the server runs" in {
+      val uploadTempFiles = UploadTempFiles(system)
+      val first = uploadTempFiles.create()
+      val directory = first.getParentFile
+      first.delete()
+      Files.delete(directory.toPath) // a temp-file reaper may remove the directory whenever it is empty
+      val second = uploadTempFiles.create()
+      second.getParentFile shouldEqual directory
+      second.delete()
+    }
+
+    "remove its temporary directory when the actor system terminates" in {
+      val separateSystem = pekko.actor.ActorSystem("upload-temp-files-cleanup-spec")
+      val file = UploadTempFiles(separateSystem).create()
+      val directory = file.getParentFile
+      directory.exists() shouldBe true
+      Await.ready(separateSystem.terminate(), 10.seconds.dilated)
+      eventually(timeout(Span(3, Seconds))) {
+        directory.exists() shouldBe false
+      }
+    }
+
   }
 
   private def read(file: File): String = {

http/src/main/mima-filters/2.0.x.backwards.excludes/routing-use-jar-file-cache.excludes

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# new use-jar-file-cache routing setting
+ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.javadsl.settings.RoutingSettings.getUseJarFileCache")
+ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.pekko.http.scaladsl.settings.RoutingSettings.useJarFileCache")

http/src/main/resources/reference.conf

@@ -19,6 +19,17 @@ pekko.http {
     # Enables/disables ETag and `If-Modified-Since` support for FileAndResourceDirectives
     file-get-conditional = on
 
+    # Enables/disables the use of the JDK's jar file cache when FileAndResourceDirectives serve a resource that
+    # lives in a jar file. With the cache enabled the jar is opened once and kept open, so a resource is served
+    # without opening and parsing the jar file for every single request; with it disabled every request opens and
+    # parses the jar once for the resource's metadata and once more for its content.
+    #
+    # Turn this off if the jar files that resources are served from have to be replaceable while the server is
+    # running (an open jar file cannot be replaced on Windows). This can only help for jars that no class loader
+    # holds open: a class loader keeps its own handle to every jar it loads classes from for the lifetime of the
+    # JVM, unaffected by this setting, so a jar on the class path stays locked either way.
+    use-jar-file-cache = on
+
     # Enables/disables the rendering of the "rendered by" footer in directory listings
     render-vanity-footer = yes
 

http/src/main/scala/org/apache/pekko/http/impl/settings/RoutingSettingsImpl.scala

@@ -28,7 +28,8 @@ private[http] final case class RoutingSettingsImpl(
     rangeCountLimit: Int,
     rangeCoalescingThreshold: Long,
     decodeMaxBytesPerChunk: Int,
-    decodeMaxSize: Long) extends pekko.http.scaladsl.settings.RoutingSettings {
+    decodeMaxSize: Long,
+    useJarFileCache: Boolean) extends pekko.http.scaladsl.settings.RoutingSettings {
 
   override def productPrefix = "RoutingSettings"
 }
@@ -41,5 +42,6 @@ object RoutingSettingsImpl extends SettingsCompanionImpl[RoutingSettingsImpl]("p
     c.getInt("range-count-limit"),
     c.getBytes("range-coalescing-threshold"),
     c.getIntBytes("decode-max-bytes-per-chunk"),
-    c.getPossiblyInfiniteBytes("decode-max-size"))
+    c.getPossiblyInfiniteBytes("decode-max-size"),
+    c.getBoolean("use-jar-file-cache"))
 }

http/src/main/scala/org/apache/pekko/http/javadsl/settings/RoutingSettings.scala

@@ -32,6 +32,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl =>
   def getRangeCoalescingThreshold: Long
   def getDecodeMaxBytesPerChunk: Int
 
+  /**
+   * @since 2.0.0
+   */
+  def getUseJarFileCache: Boolean
+
   def withVerboseErrorMessages(verboseErrorMessages: Boolean): RoutingSettings =
     self.copy(verboseErrorMessages = verboseErrorMessages)
   def withFileGetConditional(fileGetConditional: Boolean): RoutingSettings =
@@ -44,6 +49,11 @@ abstract class RoutingSettings private[pekko] () { self: RoutingSettingsImpl =>
   def withDecodeMaxBytesPerChunk(decodeMaxBytesPerChunk: Int): RoutingSettings =
     self.copy(decodeMaxBytesPerChunk = decodeMaxBytesPerChunk)
   def withDecodeMaxSize(decodeMaxSize: Long): RoutingSettings = self.copy(decodeMaxSize = decodeMaxSize)
+
+  /**
+   * @since 2.0.0
+   */
+  def withUseJarFileCache(useJarFileCache: Boolean): RoutingSettings = self.copy(useJarFileCache = useJarFileCache)
 }
 
 object RoutingSettings extends SettingsCompanion[RoutingSettings] {

http/src/main/scala/org/apache/pekko/http/scaladsl/server/directives/FileAndResourceDirectives.scala

@@ -14,8 +14,8 @@
 package org.apache.pekko.http.scaladsl.server
 package directives
 
-import java.io.File
-import java.net.{ URI, URL }
+import java.io.{ File, FileNotFoundException, IOException, InputStream }
+import java.net.{ JarURLConnection, URL, URLConnection }
 
 import scala.annotation.tailrec
 import scala.jdk.CollectionConverters._
@@ -109,17 +109,20 @@ trait FileAndResourceDirectives {
       resourceName: String, contentType: ContentType, classLoader: ClassLoader = _defaultClassLoader): Route =
     if (!resourceName.endsWith('/'))
       get {
-        Option(classLoader.getResource(resourceName)).flatMap(ResourceFile.apply) match {
-          case Some(ResourceFile(url, length, lastModified)) =>
-            conditionalFor(length, lastModified) {
-              if (length > 0) {
-                withRangeSupportAndPrecompressedMediaTypeSupport {
-                  complete(HttpEntity.Default(contentType, length,
-                    StreamConverters.fromInputStream(() => url.openStream())))
-                }
-              } else complete(HttpEntity.Empty)
-            }
-          case _ => reject // not found or directory
+        extractSettings { settings =>
+          val useJarFileCache = settings.useJarFileCache
+          Option(classLoader.getResource(resourceName)).flatMap(ResourceFile(_, useJarFileCache)) match {
+            case Some(ResourceFile(url, length, lastModified)) =>
+              conditionalFor(length, lastModified) {
+                if (length > 0) {
+                  withRangeSupportAndPrecompressedMediaTypeSupport {
+                    complete(HttpEntity.Default(contentType, length,
+                      StreamConverters.fromInputStream(() => openStream(url, useJarFileCache))))
+                  }
+                } else complete(HttpEntity.Empty)
+              }
+            case _ => reject // not found or directory
+          }
         }
       }
     else reject // don't serve the content of resource "directories"
@@ -278,35 +281,79 @@ object FileAndResourceDirectives extends FileAndResourceDirectives {
   }
 
   object ResourceFile {
-    def apply(url: URL): Option[ResourceFile] = url.getProtocol match {
+
+    /**
+     * Probes the resource without keeping any handle open: for a resource inside a jar file the jar is opened, read
+     * and closed again, as this overload always did before the jar file cache became configurable. Use the
+     * two-argument overload to let the JDK's jar file cache keep the jar open across requests.
+     */
+    def apply(url: URL): Option[ResourceFile] = apply(url, useJarFileCache = false)
+
+    /**
+     * @param useJarFileCache whether the JDK's jar file cache may be used for resources inside a jar file, see the
+     *                        `pekko.http.routing.use-jar-file-cache` setting
+     * @since 2.0.0
+     */
+    def apply(url: URL, useJarFileCache: Boolean): Option[ResourceFile] = url.getProtocol match {
       case "file" =>
         val file = new File(url.toURI)
         if (file.isDirectory) None
         else Some(ResourceFile(url, file.length(), file.lastModified()))
-      case "jar" =>
-        val path = new URI(url.getPath).getPath // remove "file:" prefix and normalize whitespace
-        val bangIndex = path.indexOf('!')
-        val filePath = path.substring(0, bangIndex)
-        val resourcePath = path.substring(bangIndex + 2)
-        val jar = new java.util.zip.ZipFile(filePath)
-        try {
-          val entry = jar.getEntry(resourcePath)
-          if (entry.isDirectory) None
-          else Option(jar.getInputStream(entry)).map { is =>
-            is.close()
-            ResourceFile(url, entry.getSize, entry.getTime)
-          }
-        } finally jar.close()
       case _ =>
-        val conn = url.openConnection()
-        try {
-          conn.setUseCaches(false) // otherwise the JDK will keep the connection open when we close!
-          val len = conn.getContentLength
-          val lm = conn.getLastModified
-          Some(ResourceFile(url, len, lm))
-        } finally conn.getInputStream.close()
+        openConnection(url, useJarFileCache) match {
+          case jarConnection: JarURLConnection =>
+            // Ask the connection for the entry instead of opening the jar file here: opening it means reading and
+            // parsing the whole central directory again for every single request. With the JDK's cache in use the
+            // same open jar file is reused across requests, so nothing is opened here at all in the common case.
+            // A connection that does not use the cache owns its jar file and has to close it again.
+            val ownsJarFile = !jarConnection.getUseCaches
+            try {
+              val entry = Option(jarConnection.getJarEntry).filterNot(_.isDirectory)
+              entry.map(e => ResourceFile(url, e.getSize, e.getTime))
+            } catch {
+              case _: FileNotFoundException => None // the entry disappeared from the jar in the meantime
+            } finally if (ownsJarFile) {
+                try jarConnection.getJarFile.close()
+                catch { case _: IOException => } // the jar itself may be gone, then there is nothing to close
+              }
+          case connection => fromUrlConnection(url, connection)
+        }
+    }
+
+    private def fromUrlConnection(url: URL, connection: URLConnection): Option[ResourceFile] =
+      try {
+        connection.setUseCaches(false) // otherwise the JDK will keep the connection open when we close!
+        val len = connection.getContentLengthLong
+        val lm = connection.getLastModified
+        if (len < 0) None // an unknown length would be served as an empty response, so reject instead
+        else Some(ResourceFile(url, len, lm))
+      } catch {
+        case _: FileNotFoundException => None // nothing behind the URL: reject instead of failing the request
+      } finally {
+        try connection.getInputStream.close()
+        catch { case _: IOException => } // a stream that cannot be opened holds nothing to close
+      }
+
+    /**
+     * The one place that decides how a connection relates to the JDK's caches: with the jar file cache disabled the
+     * connection must own its jar file, with it enabled the JVM-wide default is left untouched, so that an
+     * application-wide `URLConnection.setDefaultUseCaches(false)` keeps its effect.
+     */
+    private[directives] def openConnection(url: URL, useJarFileCache: Boolean): URLConnection = {
+      val connection = url.openConnection()
+      if (!useJarFileCache) connection.setUseCaches(false)
+      connection
     }
   }
+
+  /**
+   * Opens the resource content. Metadata and content are read through separate connections, so with the jar file
+   * cache disabled a jar-hosted resource is opened and parsed once more here: the entity may never be materialized
+   * at all (HEAD or conditional requests), so the metadata connection cannot simply be handed over. Closing the
+   * returned stream also closes the jar file it came from when the connection owns it.
+   */
+  private def openStream(url: URL, useJarFileCache: Boolean): InputStream =
+    ResourceFile.openConnection(url, useJarFileCache).getInputStream
   case class ResourceFile(url: URL, length: Long, lastModified: Long)
 
   trait DirectoryRenderer extends pekko.http.javadsl.server.directives.DirectoryRenderer {