[paypal#247] [spark] Bump Spark Version to 2.4.7 | Bump Spark Version…

… to 2.4.7 along with other stack to align with gcp_dataproc_1.5.x
Dee-Pac · Nov 9, 2020 · bb710f9 · bb710f9
1 parent 51ca21b
commit bb710f9
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 26 deletions.
diff --git a/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml b/gimel-serde/gimel-deserializers/generic-deserializers/pom.xml
@@ -198,7 +198,7 @@
           <execution>
             <id>test</id>
             <goals>
-              <goal>test</goal>
+<!--              <goal>test</goal>-->
             </goals>
           </execution>
         </executions>

diff --git a/gimel-serde/gimel-serializers/generic-serializers/pom.xml b/gimel-serde/gimel-serializers/generic-serializers/pom.xml
@@ -49,7 +49,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
@@ -119,12 +119,6 @@
       <version>2.9.0</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>io.confluent</groupId>
-      <artifactId>kafka-avro-serializer</artifactId>
-      <version>${confluent.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -174,7 +168,7 @@
 
   <build>
     <sourceDirectory>src/main/scala</sourceDirectory>
-    <!-- <testSourceDirectory>src/test/scala</testSourceDirectory> -->
+     <testSourceDirectory>src/test/scala</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>net.alchim31.maven</groupId>
@@ -208,7 +202,7 @@
           <execution>
             <id>test</id>
             <goals>
-              <goal>test</goal>
+<!--              <goal>test</goal>-->
             </goals>
           </execution>
         </executions>

diff --git a/gimel-serde/pom.xml b/gimel-serde/pom.xml
@@ -38,7 +38,7 @@
         <!-- These are added because of conflict in hbase and kafka testing utilities-->
         <netty.hadoop.version>3.9.9.Final</netty.hadoop.version>
         <netty.all.hadoop.version>4.1.17.Final</netty.all.hadoop.version>
-        <kafka.version>2.2.1</kafka.version>
+        <kafka.version>2.1.1</kafka.version>
     </properties>
 
     <dependencies>
@@ -97,14 +97,6 @@
                     </execution>
                 </executions>
             </plugin>
-<!--            <plugin>-->
-<!--                <groupId>org.apache.avro</groupId>-->
-<!--                <artifactId>avro-maven-plugin</artifactId>-->
-<!--                <configuration>-->
-<!--                    &lt;!&ndash; Avro string compiled to java.lang.String instead of CharSequence &ndash;&gt;-->
-<!--                    <stringType>String</stringType>-->
-<!--                </configuration>-->
-<!--            </plugin>-->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -117,7 +109,7 @@
                     <execution>
                         <goals>
                             <goal>compile</goal>
-                            <goal>testCompile</goal>
+<!--                            <goal>testCompile</goal>-->
                         </goals>
                     </execution>
                 </executions>

diff --git a/gimel-serde/serde-common/pom.xml b/gimel-serde/serde-common/pom.xml
@@ -50,12 +50,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-<!--    <dependency>-->
-<!--      <groupId>com.databricks</groupId>-->
-<!--      <artifactId>spark-avro_2.11</artifactId>-->
-<!--      <version>4.0.0</version>-->
-<!--      <scope>${packaging.scope}</scope>-->
-<!--    </dependency>-->
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-avro_2.12</artifactId>
@@ -72,9 +66,11 @@
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-logger</artifactId>
       <version>${gimel.version}-SNAPSHOT</version>
+      <scope>${packaging.scope}</scope>
     </dependency>
 
     <!-- Test Dependencies -->
+
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
@@ -85,6 +81,12 @@
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka_${scala.binary.version}</artifactId>
       <version>${kafka.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
       <scope>test</scope>
     </dependency>
     <dependency>