From a6c29639e2bac23e83058d9e0d6c0d9e992ebd50 Mon Sep 17 00:00:00 2001 From: menishmueli Date: Thu, 20 Aug 2026 11:47:59 +0300 Subject: [PATCH] Fix driver crash on Spark 4.2 (Jetty 12 EE10 relocation) Spark 4.2 upgraded to Jetty 12, which relocated the shaded servlet classes into the EE10 package: Spark 3.x / 4.0 / 4.1 : org.sparkproject.jetty.servlet.ServletContextHandler Spark 4.2 : org.sparkproject.jetty.ee10.servlet.ServletContextHandler DataflintJettyUtils.getClassForName probed only org.sparkproject.jetty.servlet.* and org.eclipse.jetty.servlet.*, so on 4.2 both lookups miss. This is not a degraded-UI failure: the exception propagates out of SparkDataflintDriverPlugin.registerMetrics and SparkContext initialization fails, so the whole Spark application dies at startup. java.lang.ClassNotFoundException: org.eclipse.jetty.servlet.ServletContextHandler at org.apache.spark.dataflint.api.DataflintJettyUtils$.getClassForName$1(DataflintJettyUtils.scala:26) at org.apache.spark.dataflint.api.DataflintJettyUtils$.createStaticHandler(DataflintJettyUtils.scala:30) at org.apache.spark.dataflint.api.Spark4PageFactory.addStaticHandler(Spark4PageFactory.scala:45) at org.apache.spark.dataflint.DataflintSparkUICommonInstaller.loadUI(DataflintSparkUICommonLoader.scala:119) at io.dataflint.spark.SparkDataflintDriverPlugin.registerMetrics(SparkDataflintPlugin.scala:27) Probe the EE10 packages first and fall back to the older ones, so a single build keeps working across Spark 3.x, 4.0, 4.1 and 4.2. Everything downstream is reflective (setInitParameter, setContextPath, addServlet) and those methods all exist on the EE10 ServletContextHandler, so no other change is required. Failure now reports every package that was tried instead of surfacing the last ClassNotFoundException. Also adds an example_4_2_0 project mirroring example_4_0_1, which doubles as a regression test: before this change it cannot start a SparkSession at all. Verified on Apache Spark 4.2.0 (local mode, Java 17, Scala 2.13): - driver starts and the plugin registers - a DataFlint tab appears in the Spark UI nav - GET /dataflint/ returns 200 - GET /dataflint/applicationinfo/json/ returns real application data Known follow-up, NOT addressed here: with the backend healthy on 4.2 the web UI still renders only a loading spinner and issues no XHR. That reproduces with the published 0.9.9 static assets too, so it is independent of this fix. --- spark-plugin/build.sbt | 12 +++++ .../example/ShakespeareSpark420.scala | 50 +++++++++++++++++++ .../dataflint/api/DataflintJettyUtils.scala | 20 ++++++-- 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala diff --git a/spark-plugin/build.sbt b/spark-plugin/build.sbt index 5715a85a..1b6ac820 100644 --- a/spark-plugin/build.sbt +++ b/spark-plugin/build.sbt @@ -355,4 +355,16 @@ lazy val example_4_1_0 = (project in file("example_4_1_0")) libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.1.0", libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.1.0", publish / skip := true + ).dependsOn(pluginspark4) + +lazy val example_4_2_0 = (project in file("example_4_2_0")) + .settings( + name := "DataflintSparkExample420", + organization := "io.dataflint", + scalaVersion := scala213, + crossScalaVersions := List(scala213), // Only Scala 2.13 for Spark 4.x + // there is no scala 2.12 version so we need to force 2.13 to make it compile + libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.2.0", + libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.2.0", + publish / skip := true ).dependsOn(pluginspark4) \ No newline at end of file diff --git a/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala b/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala new file mode 100644 index 00000000..09bad359 --- /dev/null +++ b/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala @@ -0,0 +1,50 @@ +package io.dataflint.example + +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.functions._ + +/** + * Spark 4.2.0 smoke example. + * + * Spark 4.2 upgraded to Jetty 12, which relocated the shaded servlet classes to + * the EE10 package. Before the DataflintJettyUtils fix this example does not + * merely lose the DataFlint tab -- SparkContext initialization fails outright + * with ClassNotFoundException, so running it is a regression test for that. + */ +object ShakespeareSpark420 extends App { + def df(spark: SparkSession): DataFrame = spark.read + .format("csv") + .option("sep", ";") + .option("inferSchema", true) + .load("./test_data/will_play_text.csv") + .toDF("line_id", "play_name", "speech_number", "line_number", "speaker", "text_entry") + .repartition(1000) + + val spark = SparkSession + .builder() + .appName("Shakespeare Statistics") + .config("spark.plugins", "io.dataflint.spark.SparkDataflintPlugin") + .config("spark.dataflint.telemetry.enabled", false) + .config("spark.ui.port", "10000") + .master("local[*]") + .getOrCreate() + + import spark.implicits._ + + val shakespeareText = df(spark) + + shakespeareText.printSchema() + + val count = shakespeareText.count() + println(s"number of records : $count") + + val uniqueSpeakers = shakespeareText.select($"speaker").distinct().count() + println(s"number of unique speakers : $uniqueSpeakers") + + val uniqueWords = shakespeareText.select(explode(split($"text_entry", " "))).distinct().count() + + println(s"number of unique words : $uniqueWords") + + scala.io.StdIn.readLine() + spark.stop() +} diff --git a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala index 7f7994de..1acebdec 100644 --- a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala +++ b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala @@ -20,11 +20,21 @@ object DataflintJettyUtils { private def createStaticHandler(resourceBase: String, path: String): Any = { // Try to load classes from both packages def getClassForName(className: String): Class[_] = { - try { - Class.forName(s"org.sparkproject.jetty.servlet.$className") - } catch { - case _: ClassNotFoundException => Class.forName(s"org.eclipse.jetty.servlet.$className") - } + // Spark 4.2 upgraded to Jetty 12, which relocated the servlet classes into + // the EE10 package (org.sparkproject.jetty.ee10.servlet.*). Spark 4.0/4.1 + // and 3.x use the older org.sparkproject.jetty.servlet.*, and an unshaded + // Spark build uses org.eclipse.jetty.*. Probe all of them, newest first. + val candidates = Seq( + s"org.sparkproject.jetty.ee10.servlet.$className", + s"org.sparkproject.jetty.servlet.$className", + s"org.eclipse.jetty.ee10.servlet.$className", + s"org.eclipse.jetty.servlet.$className" + ) + candidates.iterator + .map(n => try Some(Class.forName(n)) catch { case _: ClassNotFoundException => None }) + .collectFirst { case Some(c) => c } + .getOrElse(throw new ClassNotFoundException( + s"Could not locate Jetty class $className in any known package: ${candidates.mkString(", ")}")) } val servletContextHandlerClass = getClassForName("ServletContextHandler")