[SYSTEMDS-3655] Update Spark Dependencies

- Remove all lang and update to lang3. - reduce include jars in bin package. We are still missing updating antlr version, but there is no breaking features from using an old version in my testing. Closes #1981 Closes #1960
apache · Jan 17, 2024 · 4f52f55 · 4f52f55
1 parent 02d4b01
commit 4f52f55
Show file tree

Hide file tree

Showing 12 changed files with 25 additions and 24 deletions.
diff --git a/pom.xml b/pom.xml
@@ -41,9 +41,9 @@
 	<properties>
 		<hadoop.version>3.3.6</hadoop.version>
 		<antlr.version>4.8</antlr.version>
-		<protobuf.version>3.20.3</protobuf.version>
-		<spark.version>3.3.1</spark.version>
-		<scala.version>2.12.0</scala.version>
+		<protobuf.version>3.23.4</protobuf.version>
+		<spark.version>3.5.0</spark.version>
+		<scala.version>2.12.18</scala.version>
 		<scala.binary.version>2.12</scala.binary.version>
 		<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
 		<project.build.outputTimestamp>1</project.build.outputTimestamp>
@@ -1369,7 +1369,7 @@
 		<dependency>
 			<groupId>com.fasterxml.jackson.core</groupId>
 			<artifactId>jackson-databind</artifactId>
-			<version>2.13.4.2</version>
+			<version>2.15.2</version>
 		</dependency>
 
 		<dependency>
@@ -1405,7 +1405,7 @@
 		<dependency>
 			<groupId>org.codehaus.janino</groupId>
 			<artifactId>janino</artifactId>
-			<version>3.0.16</version>
+			<version>3.1.9</version>
 			<scope>provided</scope>
 		</dependency>
 
@@ -1438,7 +1438,7 @@
 		<dependency>
 			<groupId>io.netty</groupId>
 			<artifactId>netty-all</artifactId>
-			<version>4.1.68.Final</version>
+			<version>4.1.96.Final</version>
 			<!-- <scope>provided</scope> -->
 			<exclusions>
 				<exclusion>
@@ -1477,17 +1477,17 @@
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
-			<artifactId>jul-to-slf4j</artifactId>
+			<artifactId>slf4j-reload4j</artifactId>
 			<version>${slf4j.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
-			<artifactId>jcl-over-slf4j</artifactId>
+			<artifactId>jul-to-slf4j</artifactId>
 			<version>${slf4j.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.slf4j</groupId>
-			<artifactId>slf4j-reload4j</artifactId>
+			<artifactId>jcl-over-slf4j</artifactId>
 			<version>${slf4j.version}</version>
 		</dependency>
 

diff --git a/src/assembly/bin.xml b/src/assembly/bin.xml
@@ -92,7 +92,6 @@
 				<include>*:commons-compress*</include>
 				<include>*:commons-compiler*</include>
 				<include>*:commons-io*</include>
-				<include>*:commons-lang</include>
 				<include>*:commons-lang3</include>
 				<include>*:commons-logging*</include>
 				<include>*:commons-math3*</include>
@@ -107,7 +106,6 @@
 				<include>*:hadoop-yarn*</include>
 				<include>*:hadoop-shaded-guava*</include>
 				<include>*:jackson-core*</include>
-				<include>*:jackson-mapper*</include>
 				<include>*:janino*</include>
 				<include>*:log4j*</include>
 				<include>*:netty*</include>
@@ -117,6 +115,7 @@
 				<include>*:reload4j*</include>
 				<include>*:slf4j-api*</include>
 				<include>*:spark-core*</include>
+				<include>*:spark-common-utils*</include>
 				<include>*:stax2-api*</include>
 				<include>*:woodstox*</include>
 			</includes>

diff --git a/src/main/java/org/apache/sysds/hops/UnaryOp.java b/src/main/java/org/apache/sysds/hops/UnaryOp.java
@@ -21,7 +21,7 @@
 
 import java.util.ArrayList;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.api.DMLScript;
 import org.apache.sysds.common.Types.AggOp;
 import org.apache.sysds.common.Types.DataType;

diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
@@ -19,7 +19,7 @@
 
 package org.apache.sysds.runtime.compress.colgroup;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.runtime.DMLRuntimeException;
 import org.apache.sysds.runtime.compress.DMLCompressionException;
 import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary;

diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
@@ -24,7 +24,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.runtime.compress.DMLCompressionException;
 import org.apache.sysds.runtime.compress.utils.IntArrayList;
 

diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java
@@ -26,7 +26,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.sysds.runtime.compress.CompressedMatrixBlock;

diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java
@@ -21,7 +21,7 @@
 
 import java.lang.ref.SoftReference;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.runtime.compress.colgroup.AColGroup;
 import org.apache.sysds.runtime.compress.colgroup.ASDC;
 import org.apache.sysds.runtime.compress.colgroup.ASDCZero;

diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java
@@ -19,7 +19,7 @@
 
 package org.apache.sysds.runtime.compress.lib;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.runtime.matrix.data.MatrixBlock;
 
 public class CLALibMerge {

diff --git a/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java b/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java
@@ -19,7 +19,7 @@
 
 package org.apache.sysds.runtime.compress.utils;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.sysds.runtime.compress.utils.ACount.DCounts;

diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java
@@ -19,7 +19,7 @@
 
 package org.apache.sysds.runtime.data;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.sysds.common.Types;
 import org.apache.sysds.runtime.util.UtilFunctions;
 import org.apache.sysds.utils.MemoryEstimates;

diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java
@@ -40,8 +40,8 @@
 import org.junit.Test;
 
 public class CompressedLoggingTests {
-	protected static final Log LOG = LogFactory.getLog(CompressedLoggingTests.class.getName());
-
+	protected static final Log 	LOG = LogFactory.getLog(CompressedLoggingTests.class.getName());
+	
 	@Test
 	public void compressedLoggingTest_Trace() {
 		final TestAppender appender = LoggingUtils.overwrite();
@@ -346,14 +346,16 @@ public void compressedLoggingTestEmpty() {
 
 	@Test
 	public void compressedLoggingTest_recompress() {
-		final TestAppender appender = LoggingUtils.overwrite();
-
+		TestAppender appender = null;
+		
 		try {
+			appender = LoggingUtils.overwrite();
 			Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
 			MatrixBlock mb = TestUtils.generateTestMatrixBlock(100, 3, 1, 1, 0.5, 235);
 			MatrixBlock m2 = CompressedMatrixBlockFactory.compress(mb).getLeft();
 			CompressedMatrixBlockFactory.compress(m2).getLeft();
 			final List<LoggingEvent> log = LoggingUtils.reinsert(appender);
+
 			for(LoggingEvent l : log) {
 				if(l.getMessage().toString().contains("Recompressing"))
 					return;

diff --git a/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java b/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java
@@ -21,7 +21,7 @@
 
 import java.util.Random;
 
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.sysds.common.Types.ValueType;