Polish

longvu-db · Aug 13, 2024 · 53e6c32 · 53e6c32
1 parent babdddb
commit 53e6c32
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 16 deletions.
diff --git a/.github/workflows/spark_master_test.yaml b/.github/workflows/spark_master_test.yaml
@@ -46,7 +46,7 @@ jobs:
       - name: Run Spark Master tests
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_test.yaml
         run: |
+          TEST_PARALLELISM_COUNT=2 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean spark/test
           TEST_PARALLELISM_COUNT=2 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/test
           TEST_PARALLELISM_COUNT=2 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/assembly connectClient/test
-          TEST_PARALLELISM_COUNT=2 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean spark/test
         if: steps.git-diff.outputs.diff
diff --git a/spark-connect/server/src/main/scala-spark-master/io/delta/connect/DeltaRelationPlugin.scala b/spark-connect/server/src/main/scala-spark-master/io/delta/connect/DeltaRelationPlugin.scala
@@ -18,7 +18,6 @@ package org.apache.spark.sql.connect.delta
 
 import java.util.Optional
 
-import com.databricks.spark.util.DatabricksLogging
 import com.google.protobuf
 import com.google.protobuf.{ByteString, InvalidProtocolBufferException}
 import io.delta.connect.proto
@@ -39,7 +38,7 @@ import org.apache.spark.sql.types.StructType
 /**
  * Planner plugin for relation extensions using [[proto.DeltaRelation]].
  */
-class DeltaRelationPlugin extends RelationPlugin with DeltaPlannerBase with DatabricksLogging {
+class DeltaRelationPlugin extends RelationPlugin with DeltaPlannerBase {
   override def transform(raw: Array[Byte], planner: SparkConnectPlanner): Optional[LogicalPlan] = {
     val relation = parseAnyFrom(raw,
       SparkEnv.get.conf.get(Connect.CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT))
@@ -58,7 +57,6 @@ class DeltaRelationPlugin extends RelationPlugin with DeltaPlannerBase with Data
 
   private def transform(
       relation: proto.DeltaRelation, planner: SparkConnectPlanner): LogicalPlan = {
-    logConsole("Transforming DeltaRelation " + relation.getRelationTypeCase)
     relation.getRelationTypeCase match {
       case proto.DeltaRelation.RelationTypeCase.SCAN =>
         transformScan(planner.session, relation.getScan)

diff --git a/...onnect/server/src/test/scala-spark-master/io/delta/connect/DeltaConnectPlannerSuite.scala b/...onnect/server/src/test/scala-spark-master/io/delta/connect/DeltaConnectPlannerSuite.scala
@@ -24,7 +24,6 @@ import com.google.protobuf
 import io.delta.connect.proto
 import io.delta.connect.spark.{proto => spark_proto}
 import io.delta.tables.DeltaTable
-import org.apache.hadoop.fs.Path
 
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.{Dataset, QueryTest, SparkSession}
@@ -131,7 +130,7 @@ class DeltaConnectPlannerSuite
       val plan = transform(input)
       val result = Dataset.ofRows(spark, plan).collect()
 
-      assert(result.length == 1)
+      assert(result.length === 1)
       val deltaTable = DeltaTable.forName(spark, result.head.getString(0))
       assert(!deltaTable.toDF.isEmpty)
     }
@@ -160,7 +159,7 @@ class DeltaConnectPlannerSuite
       val plan = transform(input)
       val result = Dataset.ofRows(spark, plan).collect()
 
-      assert(result.length == 1)
+      assert(result.length === 1)
       val deltaTable = DeltaTable.forName(spark, result.head.getString(0))
       assert(!deltaTable.toDF.isEmpty)
     }
@@ -207,7 +206,7 @@ class DeltaConnectPlannerSuite
       val plan = transform(input)
       val result = Dataset.ofRows(spark, plan).collect()
 
-      assert(result.length == 1)
+      assert(result.length === 1)
       val deltaTable = DeltaTable.forName(spark, result.head.getString(0))
       assert(!deltaTable.toDF.isEmpty)
     }
@@ -254,7 +253,7 @@ class DeltaConnectPlannerSuite
       val childResult = result.asInstanceOf[DescribeDeltaDetailCommand].child
       val childExpected = expected.asInstanceOf[DescribeDeltaDetailCommand].child
 
-      assert(childResult.asInstanceOf[ResolvedTable].identifier.name ==
+      assert(childResult.asInstanceOf[ResolvedTable].identifier.name ===
         childExpected.asInstanceOf[ResolvedTable].identifier.name)
     }
   }
@@ -288,9 +287,9 @@ class DeltaConnectPlannerSuite
       val plan = transform(input)
       assert(plan.columns.toSeq == expectedRestoreOutputColumns)
       val result = Dataset.ofRows(spark, plan).collect()
-      assert(result.length == 1)
-      assert(result.head.getLong(2) == 2) // Two files should have been removed.
-      assert(spark.read.table("table").count() == 1000)
+      assert(result.length === 1)
+      assert(result.head.getLong(2) === 2) // Two files should have been removed.
+      assert(spark.read.table("table").count() === 1000)
     }
   }
 
@@ -336,9 +335,9 @@ class DeltaConnectPlannerSuite
       )
 
       val plan = transform(input)
-      assert(plan.schema.length == 1)
+      assert(plan.schema.length === 1)
       val result = Dataset.ofRows(spark, plan).collect()
-      assert(result.length == 1)
+      assert(result.length === 1)
       assert(result.head.getBoolean(0))
     }
   }
@@ -354,9 +353,9 @@ class DeltaConnectPlannerSuite
       )
 
       val plan = transform(input)
-      assert(plan.schema.length == 1)
+      assert(plan.schema.length === 1)
       val result = Dataset.ofRows(spark, plan).collect()
-      assert(result.length == 1)
+      assert(result.length === 1)
       assert(!result.head.getBoolean(0))
     }
   }