Update spark-catalyst, spark-core, ... to 3.5.8

.github/workflows/production-release.yml

@@ -6,7 +6,7 @@ on:
             description: "Git tag for release"
             required: true
         spark-version:
-            description: "Spark version (3.5.1 for Spark 3 release, 4.1.0 for Spark 4 release)"
+            description: "Spark version (3.5.8 for Spark 3 release, 4.1.0 for Spark 4 release)"
             default: "3.5.1"
         java-version:
             description: "Java version (8 for Spark 3, 17 for Spark 4)"

.github/workflows/staging-release.yml

@@ -6,7 +6,7 @@ on:
         description: "Git tag for release"
         required: true
       spark-version:
-        description: "Spark version to build against. Use 3.5.1 for Spark 3 release, 4.1.0 for Spark 4 release."
+        description: "Spark version to build against. Use 3.5.8 for Spark 3 release, 4.1.0 for Spark 4 release."
         default: "3.5.1"
       scala-version:
         description: "Scala version. Use 2.12.19 for Spark 3, 2.13.14 for Spark 4."

.github/workflows/tests.yml

@@ -61,7 +61,7 @@ jobs:
         shell: bash -el {0}
     strategy:
       matrix:
-        spark_version: [3.4.1, 3.5.1]
+        spark_version: [3.4.1, 3.5.8]
         scala_version: [2.12.19]
     env:
       SPARK_VERSION: ${{ matrix.spark_version }}

build.sbt

@@ -11,7 +11,7 @@ import sbt.nio.Keys._
 lazy val scala212 = "2.12.19"
 lazy val scala213 = "2.13.15"
 
-lazy val spark3 = "3.5.1"
+lazy val spark3 = "3.5.8"
 lazy val spark4 = "4.1.0"
 
 lazy val sparkVersion = settingKey[String]("sparkVersion")
@@ -122,7 +122,10 @@ lazy val commonSettings = Seq(
   assembly / assemblyMergeStrategy := {
     case p if p.toLowerCase.contains("manifest.mf") =>
       MergeStrategy.discard
-    case p if p.toLowerCase.endsWith(".sf") || p.toLowerCase.endsWith(".dsa") || p.toLowerCase.endsWith(".rsa") =>
+    case p
+        if p.toLowerCase.endsWith(".sf") || p.toLowerCase.endsWith(".dsa") || p
+          .toLowerCase
+          .endsWith(".rsa") =>
       MergeStrategy.discard
     case p if p.startsWith("com/fasterxml/jackson/") =>
       MergeStrategy.discard
@@ -140,7 +143,7 @@ lazy val commonSettings = Seq(
     if (sparkVersion.value.contains("SNAPSHOT"))
       Seq("Apache Snapshots" at "https://repository.apache.org/snapshots/")
     else Seq.empty
-  },
+  }
 )
 
 lazy val functionsYml = settingKey[File]("functionsYml")
@@ -206,7 +209,7 @@ ThisBuild / coreDependencies := {
   // against the same ABI Spark itself was compiled against (avoids NoSuchMethod
   // style runtime breaks). Verified against the Spark POMs:
   //   - Spark 4.1.0: netty.version = 4.2.7.Final (Spark 4.0.0 used 4.1.118; 4.1 bumped to the 4.2 line)
-  //   - Spark 3.5.1: netty 4.1.96.Final
+  //   - Spark 3.5.8: netty 4.1.96.Final
   val hadoopVersion = if (sparkMajor == "3") "3.3.6" else "3.4.2"
   val nettyVersion = if (sparkMajor == "3") "4.1.96.Final" else "4.2.7.Final"
   val avroVersion = if (sparkMajor == "3") "1.11.4" else "1.12.0"

python/environment.yml

@@ -24,7 +24,7 @@ dependencies:
   - typeguard
   - yapf=0.40.1
   - pip:
-    - pyspark==3.5.1
+    - pyspark==3.5.8
     - databricks-cli==0.18 # Docs notebook source generation
     - databricks-sdk
     - setuptools==65.6.3 # Python packaging