diff --git a/spark-plugin/build.sbt b/spark-plugin/build.sbt index 5715a85a..1b6ac820 100644 --- a/spark-plugin/build.sbt +++ b/spark-plugin/build.sbt @@ -355,4 +355,16 @@ lazy val example_4_1_0 = (project in file("example_4_1_0")) libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.1.0", libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.1.0", publish / skip := true + ).dependsOn(pluginspark4) + +lazy val example_4_2_0 = (project in file("example_4_2_0")) + .settings( + name := "DataflintSparkExample420", + organization := "io.dataflint", + scalaVersion := scala213, + crossScalaVersions := List(scala213), // Only Scala 2.13 for Spark 4.x + // there is no scala 2.12 version so we need to force 2.13 to make it compile + libraryDependencies += "org.apache.spark" % "spark-core_2.13" % "4.2.0", + libraryDependencies += "org.apache.spark" % "spark-sql_2.13" % "4.2.0", + publish / skip := true ).dependsOn(pluginspark4) \ No newline at end of file diff --git a/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala b/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala new file mode 100644 index 00000000..09bad359 --- /dev/null +++ b/spark-plugin/example_4_2_0/src/main/scala/io/dataflint/example/ShakespeareSpark420.scala @@ -0,0 +1,50 @@ +package io.dataflint.example + +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.sql.functions._ + +/** + * Spark 4.2.0 smoke example. + * + * Spark 4.2 upgraded to Jetty 12, which relocated the shaded servlet classes to + * the EE10 package. Before the DataflintJettyUtils fix this example does not + * merely lose the DataFlint tab -- SparkContext initialization fails outright + * with ClassNotFoundException, so running it is a regression test for that. + */ +object ShakespeareSpark420 extends App { + def df(spark: SparkSession): DataFrame = spark.read + .format("csv") + .option("sep", ";") + .option("inferSchema", true) + .load("./test_data/will_play_text.csv") + .toDF("line_id", "play_name", "speech_number", "line_number", "speaker", "text_entry") + .repartition(1000) + + val spark = SparkSession + .builder() + .appName("Shakespeare Statistics") + .config("spark.plugins", "io.dataflint.spark.SparkDataflintPlugin") + .config("spark.dataflint.telemetry.enabled", false) + .config("spark.ui.port", "10000") + .master("local[*]") + .getOrCreate() + + import spark.implicits._ + + val shakespeareText = df(spark) + + shakespeareText.printSchema() + + val count = shakespeareText.count() + println(s"number of records : $count") + + val uniqueSpeakers = shakespeareText.select($"speaker").distinct().count() + println(s"number of unique speakers : $uniqueSpeakers") + + val uniqueWords = shakespeareText.select(explode(split($"text_entry", " "))).distinct().count() + + println(s"number of unique words : $uniqueWords") + + scala.io.StdIn.readLine() + spark.stop() +} diff --git a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala index 7f7994de..1acebdec 100644 --- a/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala +++ b/spark-plugin/pluginspark4/src/main/scala/org/apache/spark/dataflint/api/DataflintJettyUtils.scala @@ -20,11 +20,21 @@ object DataflintJettyUtils { private def createStaticHandler(resourceBase: String, path: String): Any = { // Try to load classes from both packages def getClassForName(className: String): Class[_] = { - try { - Class.forName(s"org.sparkproject.jetty.servlet.$className") - } catch { - case _: ClassNotFoundException => Class.forName(s"org.eclipse.jetty.servlet.$className") - } + // Spark 4.2 upgraded to Jetty 12, which relocated the servlet classes into + // the EE10 package (org.sparkproject.jetty.ee10.servlet.*). Spark 4.0/4.1 + // and 3.x use the older org.sparkproject.jetty.servlet.*, and an unshaded + // Spark build uses org.eclipse.jetty.*. Probe all of them, newest first. + val candidates = Seq( + s"org.sparkproject.jetty.ee10.servlet.$className", + s"org.sparkproject.jetty.servlet.$className", + s"org.eclipse.jetty.ee10.servlet.$className", + s"org.eclipse.jetty.servlet.$className" + ) + candidates.iterator + .map(n => try Some(Class.forName(n)) catch { case _: ClassNotFoundException => None }) + .collectFirst { case Some(c) => c } + .getOrElse(throw new ClassNotFoundException( + s"Could not locate Jetty class $className in any known package: ${candidates.mkString(", ")}")) } val servletContextHandlerClass = getClassForName("ServletContextHandler")