Fix driver crash on Spark 4.2 (Jetty 12 EE10 relocation)

spark-plugin/build.sbt

@@ -355,4 +355,16 @@ lazy val example_4_1_0 = (project in file("example_4_1_0"))
     libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.1.0",
     libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.1.0",
     publish / skip := true
+  ).dependsOn(pluginspark4)
+
+lazy val example_4_2_0 = (project in file("example_4_2_0"))
+  .settings(
+    name := "DataflintSparkExample420",
+    organization := "io.dataflint",
+    scalaVersion := scala213,
+    crossScalaVersions := List(scala213), // Only Scala 2.13 for Spark 4.x
+    // there is no scala 2.12 version so we need to force 2.13 to make it compile
+    libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.2.0",
+    libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.2.0",
+    publish / skip := true
   ).dependsOn(pluginspark4)

spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala

@@ -0,0 +1,50 @@
+package io.dataflint.example
+
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.functions._
+
+/**
+ * Spark 4.2.0 smoke example.
+ *
+ * Spark 4.2 upgraded to Jetty 12, which relocated the shaded servlet classes to
+ * the EE10 package. Before the DataflintJettyUtils fix this example does not
+ * merely lose the DataFlint tab -- SparkContext initialization fails outright
+ * with ClassNotFoundException, so running it is a regression test for that.
+ */
+object ShakespeareSpark420 extends App {
+  def df(spark: SparkSession): DataFrame = spark.read
+    .format("csv")
+    .option("sep", ";")
+    .option("inferSchema", true)
+    .load("./test_data/will_play_text.csv")
+    .toDF("line_id", "play_name", "speech_number", "line_number", "speaker", "text_entry")
+    .repartition(1000)
+
+  val spark = SparkSession
+    .builder()
+    .appName("Shakespeare Statistics")
+    .config("spark.plugins", "io.dataflint.spark.SparkDataflintPlugin")
+    .config("spark.dataflint.telemetry.enabled", false)
+    .config("spark.ui.port", "10000")
+    .master("local[*]")
+    .getOrCreate()
+
+  import spark.implicits._
+
+  val shakespeareText = df(spark)
+
+  shakespeareText.printSchema()
+
+  val count = shakespeareText.count()
+  println(s"number of records : $count")
+
+  val uniqueSpeakers = shakespeareText.select($"speaker").distinct().count()
+  println(s"number of unique speakers : $uniqueSpeakers")
+
+  val uniqueWords = shakespeareText.select(explode(split($"text_entry", " "))).distinct().count()
+
+  println(s"number of unique words : $uniqueWords")
+
+  scala.io.StdIn.readLine()
+  spark.stop()
+}

spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala

@@ -20,11 +20,21 @@ object DataflintJettyUtils {
   private def createStaticHandler(resourceBase: String, path: String): Any = {
     // Try to load classes from both packages
     def getClassForName(className: String): Class[_] = {
-      try {
-        Class.forName(s"org.sparkproject.jetty.servlet.$className")
-      } catch {
-        case _: ClassNotFoundException => Class.forName(s"org.eclipse.jetty.servlet.$className")
-      }
+      // Spark 4.2 upgraded to Jetty 12, which relocated the servlet classes into
+      // the EE10 package (org.sparkproject.jetty.ee10.servlet.*). Spark 4.0/4.1
+      // and 3.x use the older org.sparkproject.jetty.servlet.*, and an unshaded
+      // Spark build uses org.eclipse.jetty.*. Probe all of them, newest first.
+      val candidates = Seq(
+        s"org.sparkproject.jetty.ee10.servlet.$className",
+        s"org.sparkproject.jetty.servlet.$className",
+        s"org.eclipse.jetty.ee10.servlet.$className",
+        s"org.eclipse.jetty.servlet.$className"
+      )
+      candidates.iterator
+        .map(n => try Some(Class.forName(n)) catch { case _: ClassNotFoundException => None })
+        .collectFirst { case Some(c) => c }
+        .getOrElse(throw new ClassNotFoundException(
+          s"Could not locate Jetty class $className in any known package: ${candidates.mkString(", ")}"))
     }
 
     val servletContextHandlerClass = getClassForName("ServletContextHandler")