[CELEBORN-1298] Support Spark2.4 with Scala2.12

### What changes were proposed in this pull request? As title ### Why are the changes needed? As title ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? GA Closes apache#2344 from waitinfuture/1298-1. Lead-authored-by: Keyong Zhou <waitinfuture@gmail.com> Co-authored-by: waitinfuture <zky.zhoukeyong@alibaba-inc.com> Signed-off-by: waitinfuture <zky.zhoukeyong@alibaba-inc.com>
Jay-ju · Feb 29, 2024 · af8c159 · af8c159
1 parent 6d9fbf5
commit af8c159
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ package apache-celeborn-${project.version}-bin.tgz will be generated.
 
 |            | Java 8/Scala 2.11 | Java 8/Scala 2.12 | Java 11/Scala 2.12 | Java 17/Scala 2.12 | Java 8/Scala 2.13 | Java 11/Scala 2.13 | Java 17/Scala 2.13 |
 |------------|-------------------|-------------------|--------------------|--------------------|-------------------|--------------------|--------------------|
-| Spark 2.4  | &#10004;          | &#x274C;          | &#x274C;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
+| Spark 2.4  | &#10004;          | &#10004;          | &#x274C;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
 | Spark 3.0  | &#x274C;          | &#10004;          | &#10004;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
 | Spark 3.1  | &#x274C;          | &#10004;          | &#10004;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
 | Spark 3.2  | &#x274C;          | &#10004;          | &#10004;           | &#x274C;           | &#10004;          | &#10004;           | &#x274C;           |
@@ -67,6 +67,17 @@ package apache-celeborn-${project.version}-bin.tgz will be generated.
 | Flink 1.17 | &#x274C;          | &#10004;          | &#10004;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
 | Flink 1.18 | &#x274C;          | &#10004;          | &#10004;           | &#x274C;           | &#x274C;          | &#x274C;           | &#x274C;           |
 
+To compile the client for Spark 2.4 with Scala 2.12, please use the following command
+
+- Scala 2.12.8/2.12.9/2.12.10
+```shell
+./build/make-distribution.sh -DskipTests -Pspark-2.4 -Dscala.version=${scala.version} -Dscala.binary.version=2.12 -Dmaven.plugin.scala.version=3.2.2 -Dmaven.plugin.silencer.version=1.6.0
+```
+- Scala 2.12.13-2.12.18
+```shell
+./build/make-distribution.sh -DskipTests -Pspark-2.4 -Dscala.version=${scala.version} -Dscala.binary.version=2.12
+```
+
 ### Package Details
 Build procedure will create a compressed package.
 

diff --git a/...park/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala b/...park/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
@@ -142,7 +142,7 @@ class CelebornShuffleReader[K, C](
         metricsCallback.incReadTime(
           TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startFetchWait))
         // ensure inputStream is closed when task completes
-        context.addTaskCompletionListener(_ => inputStream.close())
+        context.addTaskCompletionListener[Unit](_ => inputStream.close())
         (partitionId, inputStream)
       } else {
         (partitionId, CelebornInputStream.empty())
@@ -211,7 +211,7 @@ class CelebornShuffleReader[K, C](
         context.taskMetrics().incDiskBytesSpilled(sorter.diskBytesSpilled)
         context.taskMetrics().incPeakExecutionMemory(sorter.peakMemoryUsedBytes)
         // Use completion callback to stop sorter if task was finished/cancelled.
-        context.addTaskCompletionListener(_ => {
+        context.addTaskCompletionListener[Unit](_ => {
           sorter.stop()
         })
         CompletionIterator[Product2[K, C], Iterator[Product2[K, C]]](sorter.iterator, sorter.stop())