Issue #33

mattcasters · Feb 23, 2019 · 74a7b4c · 74a7b4c
1 parent da42332
commit 74a7b4c
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/pom.xml b/pom.xml
@@ -80,11 +80,6 @@
       <artifactId>beam-runners-apex</artifactId>
       <version>${apache-beam-version}</version>
     </dependency>
-    <dependency>
-      <groupId>kettle-beam</groupId>
-      <artifactId>kettle-beam-core</artifactId>
-      <version>${kettle-beam-version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.beam</groupId>
       <artifactId>beam-sdks-java-extensions-join-library</artifactId>
@@ -145,6 +140,11 @@
       <artifactId>google-cloud-bigquery</artifactId>
       <version>1.55.0</version>
     </dependency>
+    <dependency>
+      <groupId>kettle-beam</groupId>
+      <artifactId>kettle-beam-core</artifactId>
+      <version>${kettle-beam-version}</version>
+    </dependency>
 
     <dependency>
       <groupId>junit</groupId>

diff --git a/src/main/java/org/kettle/beam/pipeline/KettleBeamPipelineExecutor.java b/src/main/java/org/kettle/beam/pipeline/KettleBeamPipelineExecutor.java
@@ -393,7 +393,10 @@ private void setVariablesInTransformation( BeamJobConfig config, TransMeta trans
 
   private void configureDataFlowOptions( BeamJobConfig config, DataflowPipelineOptions options, VariableSpace space ) throws IOException {
 
-    options.setFilesToStage( BeamConst.findLibraryFilesToStage( null, space.environmentSubstitute( config.getPluginsToStage() ), true, true ) );
+    List<String> files = BeamConst.findLibraryFilesToStage( null, space.environmentSubstitute( config.getPluginsToStage() ), true, true );
+    files.removeIf( s-> s.contains( "commons-logging" ) || s.contains( "log4j" ) );
+
+    options.setFilesToStage( files );
     options.setProject( space.environmentSubstitute( config.getGcpProjectId() ) );
     options.setAppName( space.environmentSubstitute( config.getGcpAppName() ) );
     options.setStagingLocation( space.environmentSubstitute( config.getGcpStagingLocation() ) );