DRILL-3993: Changes to support Calcite 1.13

- fixed all compiling errors (main changes were: Maven changes, chenges RelNode -> RelRoot, implementing some new methods from updated interfaces, chenges some literals, logger changes); - fixed unexpected column errors, validation errors and assertion errors after Calcite update; - fixed describe table/schema statement according to updated logic; - added fixes with time-intervals; - changed precision of BINARY to 65536 (was 1048576) according to updated logic (Calcite overrides bigger precision to own maxPrecision); - ignored some incorrect tests with DRILL-3244; - changed "Table not found" message to "Object not found within" according to new Calcite changes.
rajrahul · Jan 16, 2018 · 9fabb61 · 9fabb61
1 parent 450e670
commit 9fabb61
Show file tree

Hide file tree

Showing 71 changed files with 692 additions and 436 deletions.
diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml
@@ -42,6 +42,10 @@
           <groupId>commons-codec</groupId>
           <artifactId>commons-codec</artifactId>
         </exclusion>
+        <exclusion>
+          <artifactId>calcite-avatica</artifactId>
+          <groupId>org.apache.calcite</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
   </dependencies>
@@ -71,7 +75,7 @@
                   <include>com.twitter:parquet-encoding</include>
                   <include>com.twitter:parquet-generator</include>
                   <include>org.apache.calcite:calcite-core</include>
-                  <include>org.apache.calcite:calcite-avatica</include>
+                  <include>org.apache.calcite.avatica:avatica-core</include>
                 </includes>
               </artifactSet>
               <createDependencyReducedPom>false</createDependencyReducedPom>

diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcExpressionCheck.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcExpressionCheck.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,7 +27,10 @@
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexPatternFieldRef;
 import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexTableInputRef;
 import org.apache.calcite.rex.RexVisitor;
 import org.apache.calcite.rex.RexWindow;
 import org.apache.drill.exec.planner.sql.DrillSqlOperator;
@@ -116,4 +119,19 @@ public Boolean visitFieldAccess(RexFieldAccess paramRexFieldAccess) {
     return paramRexFieldAccess.getReferenceExpr().accept(this);
   }
 
+  @Override
+  public Boolean visitSubQuery(RexSubQuery subQuery) {
+    return null;
+  }
+
+  @Override
+  public Boolean visitTableInputRef(RexTableInputRef fieldRef) {
+    return false;
+  }
+
+  @Override
+  public Boolean visitPatternFieldRef(RexPatternFieldRef fieldRef) {
+    return false;
+  }
+
 }
diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcPrel.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcPrel.java
@@ -62,7 +62,7 @@ public JdbcPrel(RelOptCluster cluster, RelTraitSet traitSet, JdbcIntermediatePre
         (JavaTypeFactory) getCluster().getTypeFactory());
     final JdbcImplementor.Result result =
         jdbcImplementor.visitChild(0, input.accept(new SubsetRemover()));
-    sql = result.asQuery().toSqlString(dialect).getSql();
+    sql = result.asSelect().toSqlString(dialect).getSql();
     rowType = input.getRowType();
   }
 

diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
@@ -211,6 +211,10 @@
       <groupId>org.apache.calcite</groupId>
       <artifactId>calcite-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.calcite.avatica</groupId>
+      <artifactId>avatica</artifactId>
+    </dependency>
     <dependency>
       <groupId>net.sf.jpam</groupId>
       <artifactId>jpam</artifactId>

diff --git a/exec/java-exec/src/main/codegen/data/Parser.tdd b/exec/java-exec/src/main/codegen/data/Parser.tdd
@@ -29,15 +29,12 @@
   # List of keywords.
   keywords: [
     "DATABASES",
-    "REPLACE",
     "SCHEMAS",
-    "SHOW",
     "TABLES",
     "USE",
     "FILES",
     "REFRESH",
     "METADATA",
-    "DATABASE",
     "IF",
     "JAR"
   ]
@@ -75,6 +72,26 @@
   implementationFiles: [
     "parserImpls.ftl"
   ]
+
+  # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls.
+  # Each must accept arguments "(SqlParserPos pos, boolean replace)".
+  createStatementParserMethods: [
+  ]
+
+  # List of methods for parsing extensions to "ALTER <scope>" calls.
+  # Each must accept arguments "(SqlParserPos pos, String scope)".
+  alterStatementParserMethods: [
+  ]
+
+  # List of methods for parsing extensions to "DROP" calls.
+  # Each must accept arguments "(SqlParserPos pos)".
+    dropStatementParserMethods: [
+  ]
+
+  # List of keywords from "keywords" section that are not reserved.
+    nonReservedKeywords: [
+  ]
+
 
   includeCompoundIdentifier: false,
   includeBraces: true,

diff --git a/exec/java-exec/src/main/codegen/includes/parserImpls.ftl b/exec/java-exec/src/main/codegen/includes/parserImpls.ftl
@@ -118,7 +118,7 @@ SqlNode SqlDescribeTable() :
         E()
     )
     {
-        return new SqlDescribeTable(pos, table, column, columnPattern);
+        return new DrillSqlDescribeTable(pos, table, column, columnPattern);
     }
 }
 
@@ -158,7 +158,7 @@ SqlNodeList ParseRequiredFieldList(String relType) :
 }
 {
     <LPAREN>
-    fieldList = SimpleIdentifierCommaList()
+    fieldList = ParenthesizedCompoundIdentifierList()
     <RPAREN>
     {
         for(SqlNode node : fieldList)
@@ -351,4 +351,23 @@ SqlNode SqlDropFunction() :
    {
        return new SqlDropFunction(pos, jar);
    }
-}
+}
+
+<#if !parser.includeCompoundIdentifier >
+/**
+* Parses a comma-separated list of simple identifiers.
+*/
+SqlNodeList ParenthesizedCompoundIdentifierList() :
+{
+    List<SqlIdentifier> list = new ArrayList<SqlIdentifier>();
+    SqlIdentifier id;
+}
+{
+    id = SimpleIdentifier() {list.add(id);}
+    (
+   <COMMA> id = SimpleIdentifier() {list.add(id);}) *
+    {
+       return new SqlNodeList(list, getPos());
+    }
+}
+</#if>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -106,8 +106,19 @@ public FieldType(String name, RelDataType dataType) {
         p = dataType.getPrecision();
         s = dataType.getScale();
         break;
+      case INTERVAL_YEAR:
       case INTERVAL_YEAR_MONTH:
-      case INTERVAL_DAY_TIME:
+      case INTERVAL_MONTH:
+      case INTERVAL_DAY:
+      case INTERVAL_DAY_HOUR:
+      case INTERVAL_DAY_MINUTE:
+      case INTERVAL_DAY_SECOND:
+      case INTERVAL_HOUR:
+      case INTERVAL_HOUR_MINUTE:
+      case INTERVAL_HOUR_SECOND:
+      case INTERVAL_MINUTE:
+      case INTERVAL_MINUTE_SECOND:
+      case INTERVAL_SECOND:
         p = dataType.getIntervalQualifier().getStartPrecisionPreservingDefault();
       default:
         break;

diff --git a/.../java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java b/.../java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -28,6 +28,7 @@
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -259,7 +260,8 @@ private TableScan createNewTableScanFromSelection(EnumerableTableScan oldScan, L
             new DynamicDrillTable(table.getPlugin(), table.getStorageEngineName(),
             table.getUserName(),
             newFormatSelection));
-    final RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(t.getRelOptSchema(), t.getRowType(), newTable);
+    final RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(t.getRelOptSchema(), t.getRowType(), newTable,
+        ImmutableList.<String>of());
 
     // return an EnumerableTableScan with fileSelection being part of digest of TableScan node.
     return DirPrunedEnumerableTableScan.create(oldScan.getCluster(), newOptTableImpl, newFileSelection.toString());

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStoreRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillStoreRelBase.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,16 +26,17 @@
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rex.RexNode;
 
 /**
  * Base class for logical and physical Store implemented in Drill
  */
 public abstract class DrillStoreRelBase extends TableModify implements DrillRelNode {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillStoreRelBase.class);
 
-  protected DrillStoreRelBase(RelOptCluster cluster, RelTraitSet traits, RelOptTable table, CatalogReader catalogReader,
-      RelNode child, Operation operation, List<String> updateColumnList, boolean flattened) {
-    super(cluster, traits, table, catalogReader, child, operation, updateColumnList, flattened);
+  protected DrillStoreRelBase(RelOptCluster cluster, RelTraitSet traits, RelOptTable table, CatalogReader catalogReader, RelNode child,
+                              Operation operation, List<String> updateColumnList, List<RexNode> sourceExpressionList, boolean flattened) {
+    super(cluster, traits, table, catalogReader, child, operation, updateColumnList, sourceExpressionList, flattened);
 
   }
 

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java
@@ -216,7 +216,9 @@ private static void writeLiteral(RexLiteral literal, JsonOutput out) throws IOEx
         }
         return;
 
+      case INTERVAL_YEAR:
       case INTERVAL_YEAR_MONTH:
+      case INTERVAL_MONTH:
         if (isLiteralNull(literal)) {
           out.writeIntervalNull();
         } else {
@@ -225,7 +227,16 @@ private static void writeLiteral(RexLiteral literal, JsonOutput out) throws IOEx
         }
         return;
 
-      case INTERVAL_DAY_TIME:
+      case INTERVAL_DAY:
+      case INTERVAL_DAY_HOUR:
+      case INTERVAL_DAY_MINUTE:
+      case INTERVAL_DAY_SECOND:
+      case INTERVAL_HOUR:
+      case INTERVAL_HOUR_MINUTE:
+      case INTERVAL_HOUR_SECOND:
+      case INTERVAL_MINUTE:
+      case INTERVAL_MINUTE_SECOND:
+      case INTERVAL_SECOND:
         if (isLiteralNull(literal)) {
           out.writeIntervalNull();
         } else {

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRule.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,9 +17,6 @@
  */
 package org.apache.drill.exec.planner.logical;
 
-import java.util.logging.Logger;
-
-import org.apache.calcite.rel.core.Aggregate;
 import org.apache.calcite.rel.InvalidRelException;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.plan.Convention;
@@ -28,6 +25,7 @@
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.logical.LogicalAggregate;
 import org.apache.calcite.util.trace.CalciteTrace;
+import org.slf4j.Logger;
 
 /**
  * Rule that converts an {@link LogicalAggregate} to a {@link DrillAggregateRel}, implemented by a Drill "segment" operation
@@ -43,7 +41,7 @@ private DrillAggregateRule() {
 
   @Override
   public void onMatch(RelOptRuleCall call) {
-    final LogicalAggregate aggregate = (LogicalAggregate) call.rel(0);
+    final LogicalAggregate aggregate = call.rel(0);
     final RelNode input = call.rel(1);
 
     if (aggregate.containsDistinctCall()) {
@@ -57,7 +55,7 @@ public void onMatch(RelOptRuleCall call) {
       call.transformTo(new DrillAggregateRel(aggregate.getCluster(), traits, convertedInput, aggregate.indicator,
           aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList()));
     } catch (InvalidRelException e) {
-      tracer.warning(e.toString());
+      tracer.warn(e.toString());
     }
   }
 }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConditions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConditions.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,6 +24,12 @@
 public final class DrillConditions {
 
   public static PushProjector.ExprCondition PRESERVE_ITEM = new PushProjector.ExprCondition() {
+
+    @Override
+    public boolean apply(RexNode rexNode) {
+      return false;
+    }
+
     @Override
     public boolean test(RexNode expr) {
       if (expr instanceof RexCall) {

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -20,6 +20,10 @@
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
 import io.netty.buffer.DrillBuf;
+import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.TimeString;
+import org.apache.calcite.util.TimestampString;
 import org.apache.calcite.rel.RelNode;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
@@ -79,7 +83,7 @@
 import java.util.Calendar;
 import java.util.List;
 
-public class DrillConstExecutor implements RelOptPlanner.Executor {
+public class DrillConstExecutor implements RexExecutor {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConstExecutor.class);
 
   private final PlannerSettings plannerSettings;
@@ -203,7 +207,7 @@ public RexNode apply(ValueHolder output) {
               Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ?
                 new DateTime(((NullableDateHolder) output).value, DateTimeZone.UTC).toCalendar(null) :
                 new DateTime(((DateHolder) output).value, DateTimeZone.UTC).toCalendar(null);
-              return rexBuilder.makeLiteral(value,
+              return rexBuilder.makeLiteral(DateString.fromCalendarFields(value),
                 TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.DATE, newCall.getType().isNullable()), false);
             }
             case DECIMAL9: {
@@ -282,14 +286,14 @@ public RexNode apply(ValueHolder output) {
               Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ?
                 new DateTime(((NullableTimeHolder) output).value, DateTimeZone.UTC).toCalendar(null) :
                 new DateTime(((TimeHolder) output).value, DateTimeZone.UTC).toCalendar(null);
-              return rexBuilder.makeLiteral(value,
+              return rexBuilder.makeLiteral(TimeString.fromCalendarFields(value),
                 TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.TIME, newCall.getType().isNullable()), false);
             }
             case TIMESTAMP: {
               Calendar value = (materializedExpr.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) ?
                 new DateTime(((NullableTimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null) :
                 new DateTime(((TimeStampHolder) output).value, DateTimeZone.UTC).toCalendar(null);
-              return rexBuilder.makeLiteral(value,
+              return rexBuilder.makeLiteral(TimestampString.fromCalendarFields(value),
                 TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.TIMESTAMP, newCall.getType().isNullable()), false);
             }
             case INTERVALYEAR: {
@@ -313,7 +317,7 @@ public RexNode apply(ValueHolder output) {
               }
               return rexBuilder.makeLiteral(
                 new BigDecimal(days * DateUtility.daysToStandardMillis + milliseconds),
-                TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY_TIME, newCall.getType().isNullable()),
+                TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, SqlTypeName.INTERVAL_DAY, newCall.getType().isNullable()),
                 false);
             }
             // The list of known unsupported types is used to trigger this behavior of re-using the input expression