From 4f52f55b89857d549a38be012274d540bff8098e Mon Sep 17 00:00:00 2001 From: Badrul Chowdhury Date: Wed, 13 Dec 2023 09:07:14 -0800 Subject: [PATCH] [SYSTEMDS-3655] Update Spark Dependencies - Remove all lang and update to lang3. - reduce include jars in bin package. We are still missing updating antlr version, but there is no breaking features from using an old version in my testing. Closes #1981 Closes #1960 --- pom.xml | 18 +++++++++--------- src/assembly/bin.xml | 3 +-- .../java/org/apache/sysds/hops/UnaryOp.java | 2 +- .../runtime/compress/colgroup/APreAgg.java | 2 +- .../compress/colgroup/indexes/RangeIndex.java | 2 +- .../colgroup/scheme/CompressionScheme.java | 2 +- .../compress/colgroup/scheme/SDCSchemeSC.java | 2 +- .../runtime/compress/lib/CLALibMerge.java | 2 +- .../runtime/compress/utils/ACountHashMap.java | 2 +- .../runtime/data/DenseBlockFP64DEDUP.java | 2 +- .../compress/CompressedLoggingTests.java | 10 ++++++---- .../frame/compress/FrameCompressTestUtils.java | 2 +- 12 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index 9808bfac51e..49246ebf78c 100644 --- a/pom.xml +++ b/pom.xml @@ -41,9 +41,9 @@ 3.3.6 4.8 - 3.20.3 - 3.3.1 - 2.12.0 + 3.23.4 + 3.5.0 + 2.12.18 2.12 yyyy-MM-dd HH:mm:ss z 1 @@ -1369,7 +1369,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.4.2 + 2.15.2 @@ -1405,7 +1405,7 @@ org.codehaus.janino janino - 3.0.16 + 3.1.9 provided @@ -1438,7 +1438,7 @@ io.netty netty-all - 4.1.68.Final + 4.1.96.Final @@ -1477,17 +1477,17 @@ org.slf4j - jul-to-slf4j + slf4j-reload4j ${slf4j.version} org.slf4j - jcl-over-slf4j + jul-to-slf4j ${slf4j.version} org.slf4j - slf4j-reload4j + jcl-over-slf4j ${slf4j.version} diff --git a/src/assembly/bin.xml b/src/assembly/bin.xml index 8f3558bbfca..b19d74dcf40 100644 --- a/src/assembly/bin.xml +++ b/src/assembly/bin.xml @@ -92,7 +92,6 @@ *:commons-compress* *:commons-compiler* *:commons-io* - *:commons-lang *:commons-lang3 *:commons-logging* *:commons-math3* @@ -107,7 +106,6 @@ *:hadoop-yarn* *:hadoop-shaded-guava* *:jackson-core* - *:jackson-mapper* *:janino* *:log4j* *:netty* @@ -117,6 +115,7 @@ *:reload4j* *:slf4j-api* *:spark-core* + *:spark-common-utils* *:stax2-api* *:woodstox* diff --git a/src/main/java/org/apache/sysds/hops/UnaryOp.java b/src/main/java/org/apache/sysds/hops/UnaryOp.java index d394beaf0eb..f046ffe85c2 100644 --- a/src/main/java/org/apache/sysds/hops/UnaryOp.java +++ b/src/main/java/org/apache/sysds/hops/UnaryOp.java @@ -21,7 +21,7 @@ import java.util.ArrayList; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Types.AggOp; import org.apache.sysds.common.Types.DataType; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java index 7f585f2d7ac..9fe286ddada 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java @@ -19,7 +19,7 @@ package org.apache.sysds.runtime.compress.colgroup; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java index 17c2bed3ba1..7729389b8da 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.util.Arrays; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; import org.apache.sysds.runtime.compress.utils.IntArrayList; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java index 27f2a4e0235..1360e93e999 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/CompressionScheme.java @@ -26,7 +26,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java index e273b814d1f..c48a9da0080 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/SDCSchemeSC.java @@ -21,7 +21,7 @@ import java.lang.ref.SoftReference; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.colgroup.AColGroup; import org.apache.sysds.runtime.compress.colgroup.ASDC; import org.apache.sysds.runtime.compress.colgroup.ASDCZero; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java index 90047fd8e26..0fac4b80288 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMerge.java @@ -19,7 +19,7 @@ package org.apache.sysds.runtime.compress.lib; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.matrix.data.MatrixBlock; public class CLALibMerge { diff --git a/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java b/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java index 9e8e87c83b6..b1d310939d9 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java +++ b/src/main/java/org/apache/sysds/runtime/compress/utils/ACountHashMap.java @@ -19,7 +19,7 @@ package org.apache.sysds.runtime.compress.utils; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.runtime.compress.utils.ACount.DCounts; diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java index c9789a9e646..d1b0c8a91b9 100644 --- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java +++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64DEDUP.java @@ -19,7 +19,7 @@ package org.apache.sysds.runtime.data; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.common.Types; import org.apache.sysds.runtime.util.UtilFunctions; import org.apache.sysds.utils.MemoryEstimates; diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java index 28b012cbf5d..22d43639939 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java @@ -40,8 +40,8 @@ import org.junit.Test; public class CompressedLoggingTests { - protected static final Log LOG = LogFactory.getLog(CompressedLoggingTests.class.getName()); - + protected static final Log LOG = LogFactory.getLog(CompressedLoggingTests.class.getName()); + @Test public void compressedLoggingTest_Trace() { final TestAppender appender = LoggingUtils.overwrite(); @@ -346,14 +346,16 @@ public void compressedLoggingTestEmpty() { @Test public void compressedLoggingTest_recompress() { - final TestAppender appender = LoggingUtils.overwrite(); - + TestAppender appender = null; + try { + appender = LoggingUtils.overwrite(); Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG); MatrixBlock mb = TestUtils.generateTestMatrixBlock(100, 3, 1, 1, 0.5, 235); MatrixBlock m2 = CompressedMatrixBlockFactory.compress(mb).getLeft(); CompressedMatrixBlockFactory.compress(m2).getLeft(); final List log = LoggingUtils.reinsert(appender); + for(LoggingEvent l : log) { if(l.getMessage().toString().contains("Recompressing")) return; diff --git a/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java b/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java index 2837cdde933..74178cef8d3 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java +++ b/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTestUtils.java @@ -21,7 +21,7 @@ import java.util.Random; -import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.common.Types.ValueType;