[PREVIEW] graphQL schema generation for new SQRL language and planner

docs/datasqrl-spec.md

@@ -153,6 +153,7 @@ The `compiler` section of the configuration controls elements of the core compil
   "compiler" : {
     "addArguments": true,
     "logger": "print",
+    "extendedScalarTypes": false,
     "explain": {
       "visual": true,
       "text": true,
@@ -164,9 +165,9 @@ The `compiler` section of the configuration controls elements of the core compil
 
 * `addArguments` specifies whether to include table columns as filters in the generated GraphQL schema. This only applies if the GraphQL schema is generated by the compiler.
 * `logger` configures the logging framework used for logging statements like `EXPORT MyTable TO logger.MyTable;`. It is `print` by default which logs to STDOUT. Set it to the configured log engine for logging output to be sent to that engine, e.g. `"logger": "kafka"`. Set it to `none` to suppress logging output.
+* `extendedScalarTypes` optional parameter (defaults to false) that configures the registration and use of graphQL extended scalar types. If enabled, the primary key of tables will be mapped to BigInteger instead of Float in GraphQL.  
 * `explain` configures how the DAG plan compiled by DataSQRL is presented in the `build` directory. If `visual` is true, a visual representation of the DAG is written to the `pipeline_visual.html` file which you can open in any browser. If `text` is true, a textual representation of the DAG is written to the `pipeline_explain.txt` file. If `extended` is true, the DAG outputs include more information like the relational plan which may be very verbose.
 
-
 ### Profiles
 
 ```json

pom.xml

@@ -55,6 +55,7 @@
     <junit.jupiter.version>5.8.2</junit.jupiter.version>
     <calcite.version>1.27.0</calcite.version>
     <graphql-java.version>19.2</graphql-java.version>
+    <graphql-java-extended-scalars.version>19.1</graphql-java-extended-scalars.version>
     <projectreactor.version>3.5.6</projectreactor.version>
     <avro.version>1.11.3</avro.version>
     <freemarker.version>2.3.32</freemarker.version>
@@ -363,6 +364,13 @@
         <version>${graphql-java.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.graphql-java</groupId>
+        <artifactId>graphql-java-extended-scalars</artifactId>
+        <version>${graphql-java-extended-scalars.version}</version>
+      </dependency>
+
+
       <!-- Vertx -->
       <dependency>
         <groupId>io.vertx</groupId>

sqrl-base/src/main/java/com/datasqrl/config/PackageJson.java

@@ -55,6 +55,8 @@ interface CompilerConfig {
     boolean isAddArguments();
 
     String getLogger();
+
+    boolean isExtendedScalarTypes();
   }
 
   interface OutputConfig {

sqrl-calcite/src/main/java/com/datasqrl/calcite/QueryPlanner.java

@@ -11,6 +11,7 @@
 import com.datasqrl.calcite.schema.ExpandTableMacroRule.ExpandTableMacroConfig;
 import com.datasqrl.calcite.schema.sql.SqlBuilders.SqlSelectBuilder;
 import com.datasqrl.canonicalizer.ReservedName;
+import com.datasqrl.graphql.server.CustomScalars;
 import com.datasqrl.parse.SqrlParserImpl;
 import com.datasqrl.util.DataContextImpl;
 import java.util.Arrays;
@@ -185,6 +186,8 @@ public RelDataType parseDatatype(String datatype) {
       return this.cluster.getTypeFactory().createSqlType(SqlTypeName.INTEGER);
     } else if (datatype.equalsIgnoreCase("datetime")) {
       return this.cluster.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, 3);
+    } else if (datatype.equalsIgnoreCase(CustomScalars.GRAPHQL_BIGINTEGER.getName())) {
+        return this.cluster.getTypeFactory().createSqlType(SqlTypeName.BIGINT);
     }
 
     // Todo fix: only supporting precision for non-alien types

sqrl-planner/src/main/java/com/datasqrl/graphql/generate/GraphqlSchemaFactory.java

@@ -9,7 +9,7 @@
 import static com.datasqrl.graphql.generate.GraphqlSchemaUtil.getInputType;
 import static com.datasqrl.graphql.generate.GraphqlSchemaUtil.getOutputType;
 import static com.datasqrl.graphql.generate.GraphqlSchemaUtil.isValidGraphQLName;
-import static com.datasqrl.graphql.generate.GraphqlSchemaUtil.wrap;
+import static com.datasqrl.graphql.generate.GraphqlSchemaUtil.wrapNullable;
 import static com.datasqrl.graphql.jdbc.SchemaConstants.LIMIT;
 import static com.datasqrl.graphql.jdbc.SchemaConstants.OFFSET;
 import static graphql.schema.GraphQLNonNull.nonNull;
@@ -19,24 +19,22 @@
 import com.datasqrl.canonicalizer.NamePath;
 
 import com.datasqrl.config.PackageJson.CompilerConfig;
-import com.datasqrl.config.SystemBuiltInConnectors;
 import com.datasqrl.engine.log.LogManager;
 import com.datasqrl.function.SqrlFunctionParameter;
 import com.datasqrl.graphql.server.CustomScalars;
 import com.datasqrl.io.tables.TableType;
 import com.datasqrl.plan.table.PhysicalRelationalTable;
 import com.datasqrl.plan.table.ProxyImportRelationalTable;
-import com.datasqrl.plan.table.QueryRelationalTable;
 import com.datasqrl.plan.validate.ExecutionGoal;
 import com.datasqrl.plan.validate.ResolvedImport;
-import com.datasqrl.plan.validate.ScriptPlanner.Mutation;
 import com.datasqrl.schema.Multiplicity;
 import com.datasqrl.schema.NestedRelationship;
 import com.datasqrl.schema.Relationship.JoinType;
 import com.datasqrl.schema.RootSqrlTable;
 import com.google.inject.Inject;
 import graphql.Scalars;
 import graphql.language.IntValue;
+import graphql.scalars.ExtendedScalars;
 import graphql.schema.GraphQLArgument;
 import graphql.schema.GraphQLFieldDefinition;
 import graphql.schema.GraphQLInputType;
@@ -67,7 +65,6 @@
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.commons.collections.ListUtils;
-import scala.annotation.meta.field;
 
 /**
  * Creates a default graphql schema based on the SQRL schema
@@ -80,6 +77,7 @@ public class GraphqlSchemaFactory {
   private final Set<String> usedNames = new HashSet<>();
   private final SqrlSchema schema;
   private final boolean addArguments;
+  private final boolean extendedScalarTypes;
   private final LogManager logManager;
   // Root path signifying the 'Query' type.
   private Map<NamePath, List<SqrlTableMacro>> objectPathToTables;
@@ -89,13 +87,14 @@ public class GraphqlSchemaFactory {
 
   @Inject
   public GraphqlSchemaFactory(SqrlSchema schema, CompilerConfig config, LogManager logManager) {
-    this(schema, config.isAddArguments(), logManager);
+    this(schema, config.isAddArguments(), config.isExtendedScalarTypes(), logManager);
   }
 
-  public GraphqlSchemaFactory(SqrlSchema schema, boolean addArguments, LogManager logManager) {
+  public GraphqlSchemaFactory(SqrlSchema schema, boolean addArguments, boolean extendedScalarTypes, LogManager logManager) {
     this.schema = schema;
     this.addArguments = addArguments;
     this.logManager = logManager;
+    this.extendedScalarTypes = extendedScalarTypes;
   }
 
   public Optional<GraphQLSchema> generate(ExecutionGoal goal) {
@@ -105,21 +104,22 @@ public Optional<GraphQLSchema> generate(ExecutionGoal goal) {
     this.fieldPathToTables = schema.getTableFunctions().stream()
         .collect(Collectors.groupingBy(SqrlTableMacro::getAbsolutePath,
             LinkedHashMap::new, Collectors.toList()));
-    for (Map.Entry<NamePath, List<SqrlTableMacro>> path : fieldPathToTables.entrySet()) {
-      if (path.getKey().getLast().isHidden()) continue;
+    for (Map.Entry<NamePath, List<SqrlTableMacro>> field : fieldPathToTables.entrySet()) {
+      if (field.getKey().getLast().isHidden()) continue; // hidden field not exposed in graphQL queries
 
 //      if (goal == ExecutionGoal.TEST) {
 //        if (!path.getValue().get(0).isTest()) continue;
 //      } else {
 //        if (path.getValue().get(0).isTest()) continue;
 //      }
-      Optional<GraphQLObjectType> graphQLObjectType = generateObject(path.getValue(),
-          objectPathToTables.getOrDefault(path.getKey(), List.of()));
+      Optional<GraphQLObjectType> graphQLObjectType = generateObject(field.getValue(), // list of table functions de field is in
+          objectPathToTables.getOrDefault(field.getKey(), List.of())); // List of table functions relationships
       graphQLObjectType.map(objectTypes::add);
     }
 
     GraphQLObjectType queryType = createQueryType(goal, objectPathToTables.get(NamePath.ROOT));
 
+    // cleaning of invalid objectTypes
     postProcess();
 
     if (queryFields.isEmpty()) {
@@ -129,6 +129,9 @@ public Optional<GraphQLSchema> generate(ExecutionGoal goal) {
 
     GraphQLSchema.Builder builder = GraphQLSchema.newSchema()
         .query(queryType);
+    if (extendedScalarTypes) { // use the plural parameter name in place of only bigInteger to avoid having a conf parameter of each special type mapping feature in the future
+      builder.additionalTypes(Set.of(CustomScalars.GRAPHQL_BIGINTEGER));
+    }
     if (goal != ExecutionGoal.TEST) {
       if (logManager.hasLogEngine() && System.getenv().get("ENABLE_SUBSCRIPTIONS") != null) {
         Optional<GraphQLObjectType.Builder> subscriptions = createSubscriptionTypes(schema);
@@ -137,7 +140,7 @@ public Optional<GraphQLSchema> generate(ExecutionGoal goal) {
       Optional<GraphQLObjectType.Builder> mutations = createMutationTypes(schema);
       mutations.map(builder::mutation);
     }
-    builder.additionalTypes(new LinkedHashSet<>(objectTypes));
+    builder.additionalTypes(new LinkedHashSet<>(objectTypes)); // the inferred types
 
     if (queryType.getFields().isEmpty()) {
       if (goal == ExecutionGoal.TEST) {
@@ -174,7 +177,7 @@ private Optional<Builder> createMutationTypes(SqrlSchema schema) {
       RelDataType type = mutation.getRelDataType();
 
       // Create the 'event' argument which should mirror the structure of the type
-      GraphQLInputType inputType = GraphqlSchemaUtil.createInputTypeForRelDataType(type, NamePath.of(name), seen).orElseThrow(
+      GraphQLInputType inputType = GraphqlSchemaUtil.createInputTypeForRelDataType(type, NamePath.of(name), seen, extendedScalarTypes).orElseThrow(
           () -> new IllegalArgumentException("Could not create input type for mutation: " + mutation.getName()));
 
       GraphQLArgument inputArgument = GraphQLArgument.newArgument()
@@ -185,7 +188,7 @@ private Optional<Builder> createMutationTypes(SqrlSchema schema) {
       GraphQLFieldDefinition subscriptionField = GraphQLFieldDefinition.newFieldDefinition()
           .name(name)
           .argument(inputArgument)
-          .type(createOutputTypeForRelDataType(type, NamePath.of(name), seen).get())
+          .type(createOutputTypeForRelDataType(type, NamePath.of(name), seen, extendedScalarTypes).get())
           .build();
 
       builder.field(subscriptionField);
@@ -226,7 +229,7 @@ public Optional<Builder> createSubscriptionTypes(SqrlSchema schema) {
 
       GraphQLFieldDefinition subscriptionField = GraphQLFieldDefinition.newFieldDefinition()
           .name(tableName)
-          .type(createOutputTypeForRelDataType(table.getRowType(), NamePath.of(tableName), seen).get())
+          .type(createOutputTypeForRelDataType(table.getRowType(), NamePath.of(tableName), seen, extendedScalarTypes).get())
           .build();
 
       subscriptionFields.add(subscriptionField);
@@ -258,7 +261,7 @@ private GraphQLObjectType createQueryType(ExecutionGoal goal, List<SqrlTableMacr
 
       GraphQLFieldDefinition field = GraphQLFieldDefinition.newFieldDefinition()
           .name(rel.getAbsolutePath().getDisplay())
-          .type(wrap(createTypeName(rel), rel.getMultiplicity()))
+          .type(GraphqlSchemaUtil.wrapMultiplicity(createTypeName(rel), rel.getMultiplicity()))
           .arguments(createArguments(rel))
           .build();
       fields.add(field);
@@ -280,22 +283,23 @@ private Optional<GraphQLObjectType> generateObject(List<SqrlTableMacro> tableMac
     //Todo check: The multiple table macros should all point to the same relnode type
 
     Map<Name, List<SqrlTableMacro>> relByName = relationships.stream()
-        .collect(Collectors.groupingBy(g -> g.getFullPath().getLast(),
+        .collect(Collectors.groupingBy(g -> g.getFullPath().getLast(), //map from relationship field to list of table functions
             LinkedHashMap::new, Collectors.toList()));
 
-    SqrlTableMacro first = tableMacros.get(0);
-    RelDataType rowType = first.getRowType();
+    SqrlTableMacro first = tableMacros.get(0); // take the first table function the field is in
+    RelDataType rowType = first.getRowType(); // and extract its calcite schema.
     //todo: check that all table macros are compatible
 //    for (int i = 1; i < tableMacros.size(); i++) {
 //    }
 
+    // create the graphQL fields for non-relationship fields
     List<GraphQLFieldDefinition> fields = new ArrayList<>();
     for (RelDataTypeField field : rowType.getFieldList()) {
       if (!relByName.containsKey(Name.system(field.getName()))) {
         createRelationshipField(field).map(fields::add);
       }
     }
-
+    // create the graphQL fields for relationship fields
     for (Map.Entry<Name, List<SqrlTableMacro>> rel : relByName.entrySet()) {
       createRelationshipField(rel.getValue()).map(fields::add);
     }
@@ -338,7 +342,7 @@ private Optional<GraphQLFieldDefinition> createRelationshipField(List<SqrlTableM
 
     GraphQLFieldDefinition field = GraphQLFieldDefinition.newFieldDefinition()
         .name(name)
-        .type(wrap(createTypeName(sqrlTableMacro), sqrlTableMacro.getMultiplicity()))
+        .type(GraphqlSchemaUtil.wrapMultiplicity(createTypeName(sqrlTableMacro), sqrlTableMacro.getMultiplicity()))
         .arguments(createArguments(sqrlTableMacro))
         .build();
 
@@ -367,10 +371,10 @@ private List<GraphQLArgument> createArguments(SqrlTableMacro field) {
     } else {
       return parameters.stream()
           .filter(p->!((SqrlFunctionParameter)p).isInternal())
-          .filter(p->getInputType(p.getType(null), NamePath.of(p.getName()), seen).isPresent())
+          .filter(p->getInputType(p.getType(null), NamePath.of(p.getName()), seen, extendedScalarTypes).isPresent())
           .map(parameter -> GraphQLArgument.newArgument()
               .name(((SqrlFunctionParameter)parameter).getVariableName())
-              .type(nonNull(getInputType(parameter.getType(null), NamePath.of(parameter.getName()), seen).get()))
+              .type(nonNull(getInputType(parameter.getType(null), NamePath.of(parameter.getName()), seen, extendedScalarTypes).get()))
               .build()).collect(Collectors.toList());
     }
   }
@@ -417,12 +421,12 @@ private List<GraphQLArgument> generatePermuted(SqrlTableMacro macro) {
 
     return primaryKeys
         .stream()
-        .filter(f -> getInputType(f.getType(), NamePath.of(tableName), seen).isPresent())
+        .filter(f -> getInputType(f.getType(), NamePath.of(tableName), seen, extendedScalarTypes).isPresent())
         .filter(f -> isValidGraphQLName(f.getName()))
         .filter(this::isVisible)
         .map(f -> GraphQLArgument.newArgument()
             .name(f.getName())
-            .type(getInputType(f.getType(), NamePath.of(f.getName()), seen).get())
+            .type(getInputType(f.getType(), NamePath.of(f.getName()), seen, extendedScalarTypes).get())
             .build())
         .collect(Collectors.toList());
   }
@@ -438,12 +442,12 @@ private GraphQLOutputType createTypeName(SqrlTableMacro sqrlTableMacro) {
   }
 
   private Optional<GraphQLFieldDefinition> createRelationshipField(RelDataTypeField field) {
-    return getOutputType(field.getType(), NamePath.of(field.getName()), seen)
+    return getOutputType(field.getType(), NamePath.of(field.getName()), seen, extendedScalarTypes)
         .filter(f->isValidGraphQLName(field.getName()))
         .filter(f->isVisible(field))
         .map(t -> GraphQLFieldDefinition.newFieldDefinition()
             .name(field.getName())
-            .type(wrap(t, field.getType())).build());
+            .type(wrapNullable(t, field.getType())).build());
   }
 
   public void postProcess() {