From 5d9e942f8f0171603c61326184dd1a8e9543eab0 Mon Sep 17 00:00:00 2001 From: Andre Masella Date: Thu, 21 Dec 2023 09:57:46 -0500 Subject: [PATCH] Redesign optional handling in `Group` clauses Removes the `OnlyIf` collector and replace it with a more sophisticated set of collectors to handle optional types. --- changes/change_group_optional.md | 1 + language.md | 44 ++- .../compiler/ExpressionNodeOptionalOf.java | 279 ++++-------------- .../oicr/gsi/shesmu/compiler/GroupNode.java | 44 ++- .../gsi/shesmu/compiler/GroupNodeOnlyIf.java | 86 ------ .../compiler/GroupNodeOptionalUnpack.java | 119 ++++++++ .../gsi/shesmu/compiler/LayerUnNester.java | 48 +++ .../OptionalCaptureCompilerServices.java | 88 ++++++ .../compiler/OptionalFlattenOutput.java | 23 ++ .../shesmu/compiler/OptionalGroupUnpack.java | 253 ++++++++++++++++ .../compiler/RegroupVariablesBuilder.java | 228 +++++++------- .../oicr/gsi/shesmu/compiler/Regrouper.java | 32 +- .../shesmu/compiler/UnboxableExpression.java | 81 +++++ .../test/resources/run/group-onlyif.shesmu | 6 - .../run/group-optional-flatten.shesmu | 7 + .../run/group-optional-flatten2.shesmu | 10 + .../run/group-optional-onlyif-all.shesmu | 7 + .../run/group-optional-onlyif-all2.shesmu | 7 + .../run/group-optional-onlyif-any.shesmu | 7 + .../group-optional-require-all-fail.shesmu | 7 + .../run/group-optional-require-all.shesmu | 7 + .../group-optional-require-any-fail.shesmu | 7 + .../run/group-optional-require-any.shesmu | 7 + 23 files changed, 957 insertions(+), 441 deletions(-) create mode 100644 changes/change_group_optional.md delete mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOnlyIf.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOptionalUnpack.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/LayerUnNester.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalCaptureCompilerServices.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalFlattenOutput.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalGroupUnpack.java create mode 100644 shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/UnboxableExpression.java delete mode 100644 shesmu-server/src/test/resources/run/group-onlyif.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-flatten.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-flatten2.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-onlyif-all.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-onlyif-all2.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-onlyif-any.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-require-all-fail.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-require-all.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-require-any-fail.shesmu create mode 100644 shesmu-server/src/test/resources/run/group-optional-require-any.shesmu diff --git a/changes/change_group_optional.md b/changes/change_group_optional.md new file mode 100644 index 000000000..d1b4946fc --- /dev/null +++ b/changes/change_group_optional.md @@ -0,0 +1 @@ +* Replace `OnlyIf` in `Group` clause with more sophisticated syntax for handling optionals diff --git a/language.md b/language.md index d2c8335e4..a96d55214 100644 --- a/language.md +++ b/language.md @@ -173,7 +173,7 @@ Replace the contents of a database with the output from the olive. Each record the olive emits is another "row" sent to the database. How the refiller interprets the data and its behaviour is defined by the refiller. - + ## Dynamic Tags Tags can be attached to an action based on the data in the olive. They can be any string. Duplicate tags are removed. @@ -430,12 +430,6 @@ Collect the smallest; if none are collected, the group is rejected. Check if _expr_ is false for all rows. If none are collected, the result is true. -- `OnlyIf` _expr_ - -Take an optional value and extract it. Ignore any empty optionals. If multiple -different values are found, reject the group. If only one unique value is -found, use it as the result and use this value. - - `PartitionCount` _expr_ Collect a counter of the number of times _expr_ was true and the number of @@ -475,6 +469,38 @@ Performs multiple collections at once and converts the results into an object. This can be very useful to share a `Where` condition while collecting multiple pieces of information. +- [_behaviour_] `` ` `` _collector_ `` ` `` + +This allows using optional values in other collectors. For instance, suppose +there is an optional number and the minimum is desired, one could write: `` ` +Min x? ` ``. + +There are special _behaviours_ for how to handle records with missing data: + +- the unspecified behaviour is to simply drop empty optionals and proceed with + the collector as normal. This is equivalent to ``Where x != ` ` Min x Default + 1234 ``; since `x` will never have the empty optional, the default will never + be used. +- `OnlyIf All` requires that no empty input makes it to the collector; if any + optional input is found, the output from the collector is replaced by the + empty optional. So, if ``v = OnlyIf All `Min x?` `` was given `` `3` `` and + `` ` ` ``, it would produce `` v == ` ` ``. +- `OnlyIf Any` requires that one non-empty input makes it to the collector; if + no optional input is found, the output from the collector is replaced by the + empty optional. So, if ``v = OnlyIf Any `Min x?` `` was given `` `3` `` and + `` ` ` ``, it would produce `` v == `3` ``, but `` ` ` `` and `` ` ` `` would + produce ``v == ` ` ``. +- `Require All` requires that no empty input makes it to the collector; if any + optional input is found, the group will be rejected. So, if + ``v = Require All `Min x?` `` was given `` `3` `` and `` ` ` ``, the group + would not be present in the output. +- `Require Any` requires that one non-empty input makes it to the collector; if + no optional input is found, the group will be rejected. So, if + ``v = Require Any `Min x?` `` was given `` `3` `` and `` ` ` ``, it would + produce `v == 3`, but if the input were `` ` ` `` and `` ` ` ``, the group + would not be present in the output. + + ## Expressions Shesmu has the following expressions, for lowest precedence to highest precedence. @@ -1349,7 +1375,7 @@ For details on optional values, see [the Mandatory Guide to Optional Values](optionalguide.md). - + ## Types There are a small number of types in the language, listed below. Each has syntax as it appears in the language and a descriptor that is used for @@ -1421,7 +1447,7 @@ to a more readable form: Mixing the two representations is fine (_e.g._, `["qb", "s"]` is equivalent to `[{"optional", "inner": "b"}, "s"]` or `t2qbs`). - + ## Regular Expression Flags Regular expressions can have modified behaviour. Any combination of the following flags can be used after a regular expression: diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/ExpressionNodeOptionalOf.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/ExpressionNodeOptionalOf.java index 094a27988..6633d8e27 100644 --- a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/ExpressionNodeOptionalOf.java +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/ExpressionNodeOptionalOf.java @@ -3,126 +3,18 @@ import static ca.on.oicr.gsi.shesmu.compiler.TypeUtils.TO_ASM; import ca.on.oicr.gsi.shesmu.compiler.Target.Flavour; -import ca.on.oicr.gsi.shesmu.compiler.definitions.FunctionDefinition; -import ca.on.oicr.gsi.shesmu.compiler.definitions.InputFormatDefinition; import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; import java.nio.file.Path; import java.util.*; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.stream.Collectors; +import org.objectweb.asm.Label; import org.objectweb.asm.Type; import org.objectweb.asm.commons.GeneratorAdapter; import org.objectweb.asm.commons.Method; public class ExpressionNodeOptionalOf extends ExpressionNode { - /** - * For each question mark we encounter, we need to track the question marks inside of it. To do - * this, we divide the operations into layers of nesting. Each layer is defined by an integer - * number that starts at zero and decreases for every inner layer. Processing is done in layer - * order. - * - *

Given foo(x?)? + y?, this will be divided into two layers (plus the base - * expression): - * - *

- * - * The order of the expressions in each layer is arbitrary, but there is no way for them to - * interfere, so it doesn't matter. - */ - private class OptionalCaptureCompilerServices implements ExpressionCompilerServices { - private final Consumer errorHandler; - private final ExpressionCompilerServices expressionCompilerServices; - private final int layer; - - public OptionalCaptureCompilerServices( - ExpressionCompilerServices expressionCompilerServices, - Consumer errorHandler, - int layer) { - this.expressionCompilerServices = expressionCompilerServices; - this.errorHandler = errorHandler; - this.layer = layer; - } - - @Override - public Optional captureOptional( - ExpressionNode expression, int line, int column, Consumer errorHandler) { - if (expression.resolveDefinitions( - new OptionalCaptureCompilerServices( - expressionCompilerServices, this.errorHandler, layer - 1), - this.errorHandler)) { - final var target = new UnboxableExpression(expression); - captures.computeIfAbsent(layer, k -> new ArrayList<>()).add(target); - return Optional.of(target); - } else { - return Optional.empty(); - } - } - - @Override - public FunctionDefinition function(String name) { - return expressionCompilerServices.function(name); - } - - @Override - public InputFormatDefinition inputFormat(String format) { - return expressionCompilerServices.inputFormat(format); - } - - @Override - public Imyhat imyhat(String name) { - return expressionCompilerServices.imyhat(name); - } - - @Override - public InputFormatDefinition inputFormat() { - return expressionCompilerServices.inputFormat(); - } - } - - private static class UnboxableExpression implements TargetWithContext { - private Optional defs = Optional.empty(); - private final ExpressionNode expression; - private final String name; - - public UnboxableExpression(ExpressionNode expression) { - this.expression = expression; - name = String.format("Lift of %d:%d", expression.line(), expression.column()); - } - - @Override - public Flavour flavour() { - return Flavour.LAMBDA; - } - - @Override - public String name() { - return name; - } - - @Override - public void read() { - // Super. Don't care. - } - - @Override - public void setContext(NameDefinitions defs) { - this.defs = Optional.of(defs); - } - - @Override - public Imyhat type() { - return expression.type() instanceof Imyhat.OptionalImyhat - ? ((Imyhat.OptionalImyhat) expression.type()).inner() - : expression.type(); - } - } - private static final Type A_OBJECT_TYPE = Type.getType(Object.class); private static final Type A_OPTIONAL_TYPE = Type.getType(Optional.class); @@ -133,6 +25,45 @@ public Imyhat type() { new Method("isPresent", Type.BOOLEAN_TYPE, new Type[0]); private static final Method METHOD_OPTIONAL__OF = new Method("of", A_OPTIONAL_TYPE, new Type[] {A_OBJECT_TYPE}); + + static void renderLayers( + Renderer renderer, Label empty, Map> captures) { + for (final var layer : captures.values()) { + for (final var capture : layer) { + final var optional = renderer.methodGen().newLocal(A_OPTIONAL_TYPE); + capture.render(renderer); + renderer.methodGen().dup(); + renderer.methodGen().storeLocal(optional); + renderer.methodGen().invokeVirtual(A_OPTIONAL_TYPE, METHOD_OPTIONAL__IS_PRESENT); + renderer.methodGen().ifZCmp(GeneratorAdapter.EQ, empty); + renderer.methodGen().loadLocal(optional); + renderer.methodGen().invokeVirtual(A_OPTIONAL_TYPE, METHOD_OPTIONAL__GET); + final var type = capture.type().apply(TO_ASM); + renderer.methodGen().unbox(type); + final var local = renderer.methodGen().newLocal(type); + renderer.methodGen().storeLocal(local); + renderer.define( + capture.name(), + new LoadableValue() { + @Override + public void accept(Renderer renderer) { + renderer.methodGen().loadLocal(local); + } + + @Override + public String name() { + return capture.name(); + } + + @Override + public Type type() { + return type; + } + }); + } + } + } + private final Map> captures = new TreeMap<>(); private final ExpressionNode item; private Imyhat type = Imyhat.BAD; @@ -146,7 +77,7 @@ public ExpressionNodeOptionalOf(int line, int column, ExpressionNode item) { public void collectFreeVariables(Set names, Predicate predicate) { for (final var layer : captures.values()) { for (final var capture : layer) { - capture.expression.collectFreeVariables(names, predicate); + capture.collectFreeVariables(names, predicate); } } item.collectFreeVariables(names, predicate); @@ -156,64 +87,12 @@ public void collectFreeVariables(Set names, Predicate predicate public void collectPlugins(Set pluginFileNames) { for (final var layer : captures.values()) { for (final var capture : layer) { - capture.expression.collectPlugins(pluginFileNames); + capture.collectPlugins(pluginFileNames); } } item.collectPlugins(pluginFileNames); } - class LayerUnNester implements Consumer { - - private final Iterator> iterator; - private final String output; - - LayerUnNester(Iterator> iterator, String output) { - this.iterator = iterator; - this.output = output; - } - - @Override - public void accept(EcmaScriptRenderer renderer) { - if (iterator.hasNext()) { - renderer.conditional( - iterator.next().stream() - .map( - capture -> { - final var capturedValue = - renderer.newConst(capture.expression.renderEcma(renderer)); - renderer.define( - new EcmaLoadableValue() { - @Override - public String name() { - return capture.name; - } - - @Override - public String get() { - return capturedValue; - } - }); - return capturedValue + " !== null"; - }) - .collect(Collectors.joining(" && ")), - this); - } else { - renderer.statement(String.format("%s = %s", output, item.renderEcma(renderer))); - } - } - } - - @Override - public String renderEcma(EcmaScriptRenderer renderer) { - if (captures.isEmpty()) { - return item.renderEcma(renderer); - } else { - final var result = renderer.newLet("null"); - new LayerUnNester(captures.values().iterator(), result).accept(renderer); - return result; - } - } - @Override public Optional dumpColumnName() { return item.dumpColumnName(); @@ -233,51 +112,31 @@ public void render(Renderer renderer) { // and then wrap it all back up in an optional (if the inner result isn't already) final var end = renderer.methodGen().newLabel(); final var empty = renderer.methodGen().newLabel(); - for (final var layer : captures.values()) { - for (final var capture : layer) { - capture.expression.render(renderer); - renderer.methodGen().dup(); - renderer.methodGen().invokeVirtual(A_OPTIONAL_TYPE, METHOD_OPTIONAL__IS_PRESENT); - renderer.methodGen().ifZCmp(GeneratorAdapter.EQ, empty); - renderer.methodGen().invokeVirtual(A_OPTIONAL_TYPE, METHOD_OPTIONAL__GET); - final var type = capture.type().apply(TO_ASM); - renderer.methodGen().unbox(type); - final var local = renderer.methodGen().newLocal(type); - renderer.methodGen().storeLocal(local); - renderer.define( - capture.name(), - new LoadableValue() { - @Override - public void accept(Renderer renderer) { - renderer.methodGen().loadLocal(local); - } - - @Override - public String name() { - return capture.name(); - } - - @Override - public Type type() { - return type; - } - }); - } - } + renderLayers(renderer, empty, captures); item.render(renderer); - // The value might already be wrapped in an optional; if it ins't do that now. + // The value might already be wrapped in an optional; if it isn't do that now. if (!item.type().isSame(item.type().asOptional())) { renderer.methodGen().box(item.type().apply(TO_ASM)); renderer.methodGen().invokeStatic(A_OPTIONAL_TYPE, METHOD_OPTIONAL__OF); } renderer.methodGen().goTo(end); renderer.methodGen().mark(empty); - renderer.methodGen().pop(); renderer.methodGen().invokeStatic(A_OPTIONAL_TYPE, METHOD_OPTIONAL__EMPTY); renderer.methodGen().mark(end); } } + @Override + public String renderEcma(EcmaScriptRenderer renderer) { + if (captures.isEmpty()) { + return item.renderEcma(renderer); + } else { + final var result = renderer.newLet("null"); + new LayerUnNester(captures.values().iterator(), item, result).accept(renderer); + return result; + } + } + @Override public boolean resolve(NameDefinitions defs, Consumer errorHandler) { // In most of the passes, we evaluate the captured expressions first because they will be @@ -295,13 +154,7 @@ public boolean resolve(NameDefinitions defs, Consumer errorHandler) { && captures.values().stream() .allMatch( layer -> - layer.stream() - .filter( - capture -> - capture.expression.resolve( - capture.defs.map(defs::withShadowContext).orElse(defs), - errorHandler)) - .count() + layer.stream().filter(capture -> capture.resolve(defs, errorHandler)).count() == layer.size()); } @@ -309,7 +162,7 @@ public boolean resolve(NameDefinitions defs, Consumer errorHandler) { public boolean resolveDefinitions( ExpressionCompilerServices expressionCompilerServices, Consumer errorHandler) { return item.resolveDefinitions( - new OptionalCaptureCompilerServices(expressionCompilerServices, errorHandler, 0), + new OptionalCaptureCompilerServices(expressionCompilerServices, errorHandler, captures), errorHandler); } @@ -333,23 +186,7 @@ public boolean typeCheck(Consumer errorHandler) { captures.values().stream() .allMatch( layer -> - layer.stream() - .filter( - capture -> { - final var captureOk = - capture.expression.typeCheck(errorHandler); - if (captureOk - && !capture - .expression - .type() - .isSame(capture.expression.type().asOptional())) { - capture.expression.typeError( - "optional", capture.expression.type(), errorHandler); - return false; - } - return captureOk; - }) - .count() + layer.stream().filter(capture -> capture.typeCheck(errorHandler)).count() == layer.size()) && item.typeCheck(errorHandler); type = item.type(); diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNode.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNode.java index c87f66835..076c507a5 100644 --- a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNode.java +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNode.java @@ -1,6 +1,10 @@ package ca.on.oicr.gsi.shesmu.compiler; +import static ca.on.oicr.gsi.shesmu.compiler.GroupNodeOptionalUnpack.INNER_SUFFIX; + +import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.shesmu.plugin.Parser; +import ca.on.oicr.gsi.shesmu.plugin.Parser.ParseDispatch; import ca.on.oicr.gsi.shesmu.plugin.Parser.Rule; import java.nio.file.Path; import java.util.List; @@ -38,7 +42,6 @@ private interface ParseGroupWithExpressionDefaultable { }); GROUPERS.addKeyword("First", ofWithDefault(GroupNodeFirst::new)); GROUPERS.addKeyword("Flatten", of(GroupNodeFlatten::new)); - GROUPERS.addKeyword("OnlyIf", of(GroupNodeOnlyIf::new)); GROUPERS.addKeyword("List", of(GroupNodeList::new)); GROUPERS.addKeyword( "LexicalConcat", @@ -154,6 +157,26 @@ private interface ParseGroupWithExpressionDefaultable { } return result; }); + for (final var entry : + List.of( + new Pair<>( + "OnlyIf", + List.of( + new Pair<>("All", OptionalGroupUnpack.EMPTY_IF_ALL_EMPTY), + new Pair<>("Any", OptionalGroupUnpack.EMPTY_IF_ANY_EMPTY))), + new Pair<>( + "Require", + List.of( + new Pair<>("All", OptionalGroupUnpack.REJECT_IF_ALL_EMPTY), + new Pair<>("Any", OptionalGroupUnpack.REJECT_IF_ANY_EMPTY))))) { + final var dispatch = new ParseDispatch(); + GROUPERS.addKeyword(entry.first(), (p, o) -> p.whitespace().dispatch(dispatch, o)); + for (final var inner : entry.second()) { + final var parser = parseOptional(inner.second()); + dispatch.addKeyword(inner.first(), (p, o) -> p.whitespace().symbol("`").then(parser, o)); + } + GROUPERS.addSymbol("`", parseOptional(OptionalGroupUnpack.FLATTEN)); + } } private static Rule of(ParseGroupWithExpression maker) { @@ -207,6 +230,25 @@ public static Parser parse(Parser input, Consumer output) { GROUPERS, maker -> output.accept(maker.make(input.line(), input.column(), name.get()))); } + private static Rule parseOptional(OptionalGroupUnpack unpack) { + return ((parser, output) -> + parser + .whitespace() + .dispatch( + GROUPERS, + (pg) -> + output.accept( + (line, column, name) -> + new GroupNodeOptionalUnpack( + parser.line(), + parser.column(), + name, + pg.make(line, column, INNER_SUFFIX), + unpack))) + .symbol("`") + .whitespace()); + } + private final int column; private final int line; diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOnlyIf.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOnlyIf.java deleted file mode 100644 index 9cad32378..000000000 --- a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOnlyIf.java +++ /dev/null @@ -1,86 +0,0 @@ -package ca.on.oicr.gsi.shesmu.compiler; - -import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; -import java.nio.file.Path; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Predicate; - -/** - * A only-if action in a “Group” clause - * - *

Also usable as the variable definition for the result - */ -public final class GroupNodeOnlyIf extends GroupNode { - - private final ExpressionNode expression; - private Imyhat innerType = Imyhat.BAD; - private final String name; - private boolean read; - - public GroupNodeOnlyIf(int line, int column, String name, ExpressionNode expression) { - super(line, column); - this.name = name; - this.expression = expression; - } - - @Override - public void collectFreeVariables(Set freeVariables, Predicate predicate) { - expression.collectFreeVariables(freeVariables, predicate); - } - - @Override - public void collectPlugins(Set pluginFileNames) { - expression.collectPlugins(pluginFileNames); - } - - @Override - public boolean isRead() { - return read; - } - - @Override - public String name() { - return name; - } - - @Override - public void read() { - read = true; - } - - @Override - public void render(Regrouper regroup, RootBuilder rootBuilder) { - regroup.addOnlyIf(innerType, name(), expression::render); - } - - @Override - public boolean resolve( - NameDefinitions defs, NameDefinitions outerDefs, Consumer errorHandler) { - return expression.resolve(defs, errorHandler); - } - - @Override - public boolean resolveDefinitions( - ExpressionCompilerServices expressionCompilerServices, Consumer errorHandler) { - return expression.resolveDefinitions(expressionCompilerServices, errorHandler); - } - - @Override - public Imyhat type() { - return innerType; - } - - @Override - public boolean typeCheck(Consumer errorHandler) { - if (expression.typeCheck(errorHandler)) { - if (expression.type() instanceof Imyhat.OptionalImyhat) { - innerType = ((Imyhat.OptionalImyhat) expression.type()).inner(); - return true; - } else { - expression.typeError("optional", expression.type(), errorHandler); - } - } - return false; - } -} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOptionalUnpack.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOptionalUnpack.java new file mode 100644 index 000000000..e2e24258d --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/GroupNodeOptionalUnpack.java @@ -0,0 +1,119 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; +import java.util.function.Predicate; + +/** + * A only-if action in a “Group” clause + * + *

Also usable as the variable definition for the result + */ +public final class GroupNodeOptionalUnpack extends GroupNode { + + public static final String INNER_SUFFIX = ":inner"; + + static String innerName(String name) { + return name + INNER_SUFFIX; + } + + private final String name; + private final GroupNode inner; + private final OptionalGroupUnpack unpack; + private final Map> captures = new TreeMap<>(); + + public GroupNodeOptionalUnpack( + int line, int column, String name, GroupNode inner, OptionalGroupUnpack unpack) { + super(line, column); + this.name = name; + this.inner = inner; + this.unpack = unpack; + } + + @Override + public void collectFreeVariables(Set freeVariables, Predicate predicate) { + for (final var layer : captures.values()) { + for (final var capture : layer) { + capture.collectFreeVariables(freeVariables, predicate); + } + } + inner.collectFreeVariables(freeVariables, predicate); + } + + @Override + public void collectPlugins(Set pluginFileNames) { + for (final var layer : captures.values()) { + for (final var capture : layer) { + capture.collectPlugins(pluginFileNames); + } + } + inner.collectPlugins(pluginFileNames); + } + + @Override + public boolean isRead() { + return inner.isRead(); + } + + @Override + public String name() { + return name; + } + + @Override + public void read() { + inner.read(); + } + + @Override + public void render(Regrouper regroup, RootBuilder rootBuilder) { + inner.render(regroup.addOnlyIf(name, unpack.consumer(inner.type(), captures)), rootBuilder); + } + + @Override + public boolean resolve( + NameDefinitions defs, NameDefinitions outerDefs, Consumer errorHandler) { + return inner.resolve(defs, outerDefs, errorHandler) + && captures.values().stream() + .allMatch( + layer -> + layer.stream().filter(capture -> capture.resolve(defs, errorHandler)).count() + == layer.size()); + } + + @Override + public boolean resolveDefinitions( + ExpressionCompilerServices expressionCompilerServices, Consumer errorHandler) { + return inner.resolveDefinitions( + new OptionalCaptureCompilerServices(expressionCompilerServices, errorHandler, captures), + errorHandler); + } + + @Override + public Imyhat type() { + return unpack.type(inner.type()); + } + + @Override + public boolean typeCheck(Consumer errorHandler) { + if (captures.isEmpty()) { + errorHandler.accept( + String.format("%d:%d: No optional values are used in this collector.", line(), column())); + return false; + } else { + var ok = + captures.values().stream() + .allMatch( + layer -> + layer.stream().filter(capture -> capture.typeCheck(errorHandler)).count() + == layer.size()) + && inner.typeCheck(errorHandler); + return ok; + } + } +} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/LayerUnNester.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/LayerUnNester.java new file mode 100644 index 000000000..39a0fde54 --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/LayerUnNester.java @@ -0,0 +1,48 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +class LayerUnNester implements Consumer { + + private final Iterator> iterator; + private final ExpressionNode inner; + private final String output; + + LayerUnNester(Iterator> iterator, ExpressionNode inner, String output) { + this.iterator = iterator; + this.inner = inner; + this.output = output; + } + + @Override + public void accept(EcmaScriptRenderer renderer) { + if (iterator.hasNext()) { + renderer.conditional( + iterator.next().stream() + .map( + capture -> { + final var capturedValue = renderer.newConst(capture.renderEcma(renderer)); + renderer.define( + new EcmaLoadableValue() { + @Override + public String name() { + return capture.name(); + } + + @Override + public String get() { + return capturedValue; + } + }); + return capturedValue + " !== null"; + }) + .collect(Collectors.joining(" && ")), + this); + } else { + renderer.statement(String.format("%s = %s", output, inner.renderEcma(renderer))); + } + } +} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalCaptureCompilerServices.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalCaptureCompilerServices.java new file mode 100644 index 000000000..b76e740c0 --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalCaptureCompilerServices.java @@ -0,0 +1,88 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import ca.on.oicr.gsi.shesmu.compiler.definitions.FunctionDefinition; +import ca.on.oicr.gsi.shesmu.compiler.definitions.InputFormatDefinition; +import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; + +/** + * For each question mark we encounter, we need to track the question marks inside of it. To do + * this, we divide the operations into layers of nesting. Each layer is defined by an integer number + * that starts at zero and decreases for every inner layer. Processing is done in layer order. + * + *

Given foo(x?)? + y?, this will be divided into two layers (plus the base + * expression): + * + *

    + *
  • Base expression: $0 + $1 + *
  • Layer 0: $0 = foo($2) and $1 = y + *
  • Layer -1: $2 = x + *
+ * + * The order of the expressions in each layer is arbitrary, but there is no way for them to + * interfere, so it doesn't matter. + */ +class OptionalCaptureCompilerServices implements ExpressionCompilerServices { + + private final Consumer errorHandler; + private final ExpressionCompilerServices expressionCompilerServices; + private final int layer; + private final Map> captures; + + public OptionalCaptureCompilerServices( + ExpressionCompilerServices expressionCompilerServices, + Consumer errorHandler, + Map> captures) { + this(expressionCompilerServices, errorHandler, 0, captures); + } + + private OptionalCaptureCompilerServices( + ExpressionCompilerServices expressionCompilerServices, + Consumer errorHandler, + int layer, + Map> captures) { + this.expressionCompilerServices = expressionCompilerServices; + this.errorHandler = errorHandler; + this.layer = layer; + this.captures = captures; + } + + @Override + public Optional captureOptional( + ExpressionNode expression, int line, int column, Consumer errorHandler) { + if (expression.resolveDefinitions( + new OptionalCaptureCompilerServices( + expressionCompilerServices, this.errorHandler, layer - 1, captures), + this.errorHandler)) { + final var target = new UnboxableExpression(expression); + captures.computeIfAbsent(layer, k -> new ArrayList<>()).add(target); + return Optional.of(target); + } else { + return Optional.empty(); + } + } + + @Override + public FunctionDefinition function(String name) { + return expressionCompilerServices.function(name); + } + + @Override + public InputFormatDefinition inputFormat(String format) { + return expressionCompilerServices.inputFormat(format); + } + + @Override + public Imyhat imyhat(String name) { + return expressionCompilerServices.imyhat(name); + } + + @Override + public InputFormatDefinition inputFormat() { + return expressionCompilerServices.inputFormat(); + } +} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalFlattenOutput.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalFlattenOutput.java new file mode 100644 index 000000000..b1fc1e3da --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalFlattenOutput.java @@ -0,0 +1,23 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; + +public enum OptionalFlattenOutput { + /** Follow the inner collector's behaviour (i.e., if it would reject, we reject) */ + PRESERVE { + @Override + public Imyhat type(Imyhat type) { + return type; + } + }, + /** If the inner collector saw no input, return an optional output */ + WRAP_ON_NO_INPUT { + @Override + public Imyhat type(Imyhat type) { + return type.asOptional(); + } + }, + ; + + public abstract Imyhat type(Imyhat type); +} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalGroupUnpack.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalGroupUnpack.java new file mode 100644 index 000000000..afaa5f993 --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/OptionalGroupUnpack.java @@ -0,0 +1,253 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import static ca.on.oicr.gsi.shesmu.compiler.BaseOliveBuilder.A_OPTIONAL_TYPE; +import static ca.on.oicr.gsi.shesmu.compiler.ExpressionNodeOptionalOf.renderLayers; +import static ca.on.oicr.gsi.shesmu.compiler.GroupNodeOptionalUnpack.innerName; +import static ca.on.oicr.gsi.shesmu.compiler.TypeUtils.TO_ASM; + +import ca.on.oicr.gsi.shesmu.compiler.Regrouper.OnlyIfConsumer; +import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import org.objectweb.asm.Label; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.GeneratorAdapter; +import org.objectweb.asm.commons.Method; + +public enum OptionalGroupUnpack { + FLATTEN { + @Override + public Imyhat type(Imyhat type) { + return type; + } + + @Override + public OnlyIfConsumer consumer(Imyhat type, Map> captures) { + return new OnlyIfConsumer() { + @Override + public void build(Renderer renderer, String fieldName, Type owner, Label failurePath) { + renderLayers(renderer, failurePath, captures); + } + + @Override + public boolean countsRequired() { + return false; + } + + @Override + public void failIfBad( + GeneratorAdapter checker, + String fieldName, + Type owner, + BiConsumer checkInner, + Label failure) { + checkInner.accept(checker, failure); + } + + @Override + public void renderGetter(GeneratorAdapter getter, String fieldName, Type owner) { + getter.loadThis(); + getter.invokeVirtual(owner, new Method(innerName(fieldName), type(), new Type[] {})); + } + + @Override + public Type type() { + return type.apply(TO_ASM); + } + }; + } + }, + EMPTY_IF_ANY_EMPTY { + @Override + public Imyhat type(Imyhat type) { + return type.asOptional(); + } + + @Override + public OnlyIfConsumer consumer(Imyhat type, Map> captures) { + return new AsEmpty(type, captures, Regrouper::goodCount, false); + } + }, + REJECT_IF_ANY_EMPTY { + @Override + public Imyhat type(Imyhat type) { + return type; + } + + @Override + public OnlyIfConsumer consumer(Imyhat type, Map> captures) { + return new Reject(type, captures, Regrouper::goodCount, false); + } + }, + EMPTY_IF_ALL_EMPTY { + @Override + public Imyhat type(Imyhat type) { + return type.asOptional(); + } + + @Override + public OnlyIfConsumer consumer(Imyhat type, Map> captures) { + return new AsEmpty(type, captures, Regrouper::badCount, true); + } + }, + REJECT_IF_ALL_EMPTY { + @Override + public Imyhat type(Imyhat type) { + return type; + } + + @Override + public OnlyIfConsumer consumer(Imyhat type, Map> captures) { + return new Reject(type, captures, Regrouper::badCount, true); + } + }; + + private static final class AsEmpty implements OnlyIfConsumer { + private final Map> captures; + private final Function checkCount; + private final boolean isZero; + private final Imyhat type; + + private AsEmpty( + Imyhat type, + Map> captures, + Function checkCount, + boolean isZero) { + this.type = type; + this.captures = captures; + this.checkCount = checkCount; + this.isZero = isZero; + } + + @Override + public void build(Renderer renderer, String fieldName, Type owner, Label failurePath) { + renderLayers(renderer, failurePath, captures); + } + + @Override + public boolean countsRequired() { + return true; + } + + @Override + public void failIfBad( + GeneratorAdapter checker, + String fieldName, + Type owner, + BiConsumer checkInner, + Label failure) { + final var end = checker.newLabel(); + // First check if we failed by optional standards; if we did , continue, as we will never care + // about the inside + checker.loadThis(); + checker.getField(owner, checkCount.apply(fieldName), Type.INT_TYPE); + checker.push(0); + checker.ifICmp(isZero ? GeneratorAdapter.NE : GeneratorAdapter.EQ, end); + + // If we would emit this value, but it's bad... + final var innerBad = checker.newLabel(); + checkInner.accept(checker, innerBad); + checker.goTo(end); + + // scramble the count so we never try to read it + checker.mark(innerBad); + checker.loadThis(); + checker.push(0); + checker.putField(owner, checkCount.apply(fieldName), Type.INT_TYPE); + + checker.mark(end); + } + + @Override + public void renderGetter(GeneratorAdapter getter, String fieldName, Type owner) { + final var failPath = getter.newLabel(); + final var end = getter.newLabel(); + getter.loadThis(); + getter.getField(owner, checkCount.apply(fieldName), Type.INT_TYPE); + getter.push(0); + getter.ifICmp(isZero ? GeneratorAdapter.NE : GeneratorAdapter.EQ, failPath); + getter.loadThis(); + getter.invokeVirtual( + owner, new Method(innerName(fieldName), type.apply(TO_ASM), new Type[] {})); + getter.valueOf(type.apply(TO_ASM)); + getter.invokeStatic(A_OPTIONAL_TYPE, METHOD_OPTIONAL__OF); + getter.goTo(end); + getter.mark(failPath); + getter.invokeStatic(A_OPTIONAL_TYPE, METHOD_OPTIONAL__EMPTY); + getter.mark(end); + } + + @Override + public Type type() { + return A_OPTIONAL_TYPE; + } + } + + private static final class Reject implements OnlyIfConsumer { + private final Map> captures; + private final Function checkCount; + private final boolean isZero; + private final Imyhat type; + + private Reject( + Imyhat type, + Map> captures, + Function checkCount, + boolean isZero) { + this.type = type; + this.captures = captures; + this.checkCount = checkCount; + this.isZero = isZero; + } + + @Override + public void build(Renderer renderer, String fieldName, Type owner, Label failurePath) { + renderLayers(renderer, failurePath, captures); + } + + @Override + public boolean countsRequired() { + return true; + } + + @Override + public void failIfBad( + GeneratorAdapter checker, + String fieldName, + Type owner, + BiConsumer checkInner, + Label failure) { + // First check if we failed by optional standards; if we did, fail + checker.loadThis(); + checker.getField(owner, checkCount.apply(fieldName), Type.INT_TYPE); + checker.push(0); + checker.ifICmp(isZero ? GeneratorAdapter.NE : GeneratorAdapter.EQ, failure); + + // We would emit this value, so do its check... + checkInner.accept(checker, failure); + } + + @Override + public void renderGetter(GeneratorAdapter getter, String fieldName, Type owner) { + getter.loadThis(); + getter.invokeVirtual(owner, new Method(innerName(fieldName), type(), new Type[] {})); + } + + @Override + public Type type() { + return type.apply(TO_ASM); + } + } + + private static final Method METHOD_OPTIONAL__EMPTY = + new Method("empty", A_OPTIONAL_TYPE, new Type[0]); + private static final Method METHOD_OPTIONAL__OF = + new Method("of", A_OPTIONAL_TYPE, new Type[] {Type.getType(Object.class)}); + + public abstract OnlyIfConsumer consumer( + Imyhat type, Map> captures); + + public abstract Imyhat type(Imyhat type); +} diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/RegroupVariablesBuilder.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/RegroupVariablesBuilder.java index 6496f3719..f071d4095 100644 --- a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/RegroupVariablesBuilder.java +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/RegroupVariablesBuilder.java @@ -1,16 +1,25 @@ package ca.on.oicr.gsi.shesmu.compiler; import static ca.on.oicr.gsi.shesmu.compiler.TypeUtils.TO_ASM; -import static org.objectweb.asm.Type.*; +import static org.objectweb.asm.Type.BOOLEAN_TYPE; +import static org.objectweb.asm.Type.INT_TYPE; +import static org.objectweb.asm.Type.LONG_TYPE; +import static org.objectweb.asm.Type.VOID_TYPE; import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.shesmu.plugin.Tuple; import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; import ca.on.oicr.gsi.shesmu.runtime.PartitionCount; import ca.on.oicr.gsi.shesmu.runtime.UnivaluedGroupAccumulator; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.function.Consumer; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.Label; @@ -67,7 +76,7 @@ public final void addFlatten(Imyhat valueType, String fieldName, Consumer loader, Consumer delimiterLoader) { elements.add(new LexicalConcat(prefix + fieldName, loader, delimiterLoader)); } @@ -85,8 +94,10 @@ public final Regrouper addObject(String fieldName, Stream> } @Override - public final void addOnlyIf(Imyhat valueType, String fieldName, Consumer loader) { - elements.add(new OnlyIf(valueType, prefix + fieldName, loader)); + public final Regrouper addOnlyIf(String fieldName, OnlyIfConsumer consumer) { + final var element = new OnlyIf(prefix + fieldName, consumer); + elements.add(element); + return element; } @Override @@ -111,7 +122,7 @@ public final void addPartitionCount(String fieldName, Consumer conditi } @Override - public void addSum(Imyhat valueType, String fieldName, Consumer loader) { + public final void addSum(Imyhat valueType, String fieldName, Consumer loader) { elements.add(new Sum(valueType, prefix + fieldName, loader)); } @@ -167,14 +178,22 @@ public final Stream constructorType() { } @Override - public final void failIfBad(GeneratorAdapter okMethod) { - elements.forEach(element -> element.failIfBad(okMethod)); + public void failIfBad(GeneratorAdapter okMethod, Label failure) { + failInnerIfBad(okMethod, failure); + } + + protected final void failInnerIfBad(GeneratorAdapter okMethod, Label failure) { + elements.forEach(element -> element.failIfBad(okMethod, failure)); } @Override public final void loadConstructorArgument() { elements.forEach(Element::loadConstructorArgument); } + + protected final String prefix() { + return prefix; + } } private abstract class BaseList extends Element { @@ -228,7 +247,7 @@ public final Stream constructorType() { } @Override - public final void failIfBad(GeneratorAdapter okMethod) { + public final void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -310,7 +329,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Counts are always okay. } @@ -381,7 +400,7 @@ public final Stream constructorType() { } @Override - public final void failIfBad(GeneratorAdapter okMethod) { + public final void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -458,7 +477,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -480,7 +499,7 @@ private abstract static class Element { public abstract Stream constructorType(); - public abstract void failIfBad(GeneratorAdapter okMethod); + public abstract void failIfBad(GeneratorAdapter okMethod, Label failure); public abstract void loadConstructorArgument(); } @@ -539,14 +558,10 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { okMethod.loadThis(); okMethod.getField(self, fieldName + "$ok", BOOLEAN_TYPE); - final var next = okMethod.newLabel(); - okMethod.ifZCmp(GeneratorAdapter.NE, next); - okMethod.push(false); - okMethod.returnValue(); - okMethod.mark(next); + okMethod.ifZCmp(GeneratorAdapter.EQ, failure); } @Override @@ -615,7 +630,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // First with default is always ok } @@ -710,7 +725,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // LexicalConcat with default is always ok } @@ -782,7 +797,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -874,7 +889,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -888,8 +903,7 @@ private class NamedTuple extends BaseComposite { public NamedTuple(String prefix, Stream> fields) { super(prefix + " "); - final var fieldInfo = - fields.sorted(Comparator.comparing(Pair::first)).collect(Collectors.toList()); + final var fieldInfo = fields.sorted(Comparator.comparing(Pair::first)).toList(); final var getMethod = new GeneratorAdapter( Opcodes.ACC_PUBLIC, @@ -925,32 +939,39 @@ public void buildCollect() { } } - private class OnlyIf extends Element { - private final String fieldName; - private final Consumer loader; - private final Imyhat valueType; - - private OnlyIf(Imyhat valueType, String fieldName, Consumer loader) { - super(); - this.valueType = valueType; - this.fieldName = fieldName; - this.loader = loader; - classVisitor - .visitField(Opcodes.ACC_PUBLIC, fieldName, A_SET_TYPE.getDescriptor(), null, null) - .visitEnd(); + private class OnlyIf extends BaseComposite { + private final OnlyIfConsumer consumer; + + private OnlyIf(String fieldName, OnlyIfConsumer consumer) { + super(fieldName); + this.consumer = consumer; + if (consumer.countsRequired()) { + classVisitor + .visitField( + Opcodes.ACC_PUBLIC, + Regrouper.badCount(fieldName), + INT_TYPE.getDescriptor(), + null, + null) + .visitEnd(); + classVisitor + .visitField( + Opcodes.ACC_PUBLIC, + Regrouper.goodCount(fieldName), + INT_TYPE.getDescriptor(), + null, + null) + .visitEnd(); + } final var getMethod = new GeneratorAdapter( Opcodes.ACC_PUBLIC, - new Method(fieldName, valueType.apply(TO_ASM), new Type[] {}), + new Method(fieldName, consumer.type(), new Type[] {}), null, null, classVisitor); getMethod.visitCode(); - getMethod.loadThis(); - getMethod.getField(self, fieldName, A_SET_TYPE); - getMethod.invokeInterface(A_SET_TYPE, SET__ITERATOR); - getMethod.invokeInterface(A_ITERATOR_TYPE, ITERATOR__NEXT); - getMethod.unbox(valueType.apply(TO_ASM)); + consumer.renderGetter(getMethod, fieldName, self); getMethod.returnValue(); getMethod.visitMaxs(0, 0); getMethod.visitEnd(); @@ -958,57 +979,37 @@ private OnlyIf(Imyhat valueType, String fieldName, Consumer loader) { @Override public void buildCollect() { - loader.accept(collectRenderer); - collectRenderer.methodGen().loadArg(collectedSelfArgument); - collectRenderer.methodGen().getField(self, fieldName, A_SET_TYPE); - LambdaBuilder.pushInterface( - collectRenderer, - "add", - LambdaBuilder.consumerErasingReturn(Type.BOOLEAN_TYPE, A_OBJECT_TYPE), - A_SET_TYPE); - collectRenderer.methodGen().invokeVirtual(A_OPTIONAL_TYPE, METHOD_OPTIONAL__IF_PRESENT); - } - - @Override - public int buildConstructor(GeneratorAdapter ctor, int index) { - ctor.loadThis(); - Renderer.loadImyhatInMethod(ctor, valueType.descriptor()); - ctor.invokeVirtual(A_IMYHAT_TYPE, METHOD_IMYHAT__NEW_SET); - ctor.putField(self, fieldName, A_SET_TYPE); - return index; - } - - @Override - public void buildEquals(GeneratorAdapter methodGen, int otherLocal, Label end) { - // OnlyIf are not included in equality. - } - - @Override - public void buildHashCode(GeneratorAdapter hashMethod) { - // OnlyIf are not included in the hash. - } - - @Override - public Stream constructorType() { - return Stream.empty(); - } - - @Override - public void failIfBad(GeneratorAdapter okMethod) { - okMethod.loadThis(); - okMethod.getField(self, fieldName, A_SET_TYPE); - okMethod.invokeInterface(A_SET_TYPE, SET__SIZE); - okMethod.push(1); - final var next = okMethod.newLabel(); - okMethod.ifICmp(GeneratorAdapter.EQ, next); - okMethod.push(0); - okMethod.returnValue(); - okMethod.mark(next); + final var end = collectRenderer.methodGen().newLabel(); + final var failPath = collectRenderer.methodGen().newLabel(); + final var originalRenderer = collectRenderer; + collectRenderer = collectRenderer.duplicate(); + consumer.build(collectRenderer, prefix(), self, failPath); + if (consumer.countsRequired()) { + collectRenderer.methodGen().loadArg(collectedSelfArgument); + collectRenderer.methodGen().dup(); + collectRenderer.methodGen().getField(self, Regrouper.goodCount(prefix()), INT_TYPE); + collectRenderer.methodGen().push(1); + collectRenderer.methodGen().math(GeneratorAdapter.ADD, INT_TYPE); + collectRenderer.methodGen().putField(self, Regrouper.goodCount(prefix()), INT_TYPE); + } + buildInnerCollect(); + collectRenderer.methodGen().goTo(end); + collectRenderer = originalRenderer; + collectRenderer.methodGen().mark(failPath); + if (consumer.countsRequired()) { + collectRenderer.methodGen().loadArg(collectedSelfArgument); + collectRenderer.methodGen().dup(); + collectRenderer.methodGen().getField(self, Regrouper.badCount(prefix()), INT_TYPE); + collectRenderer.methodGen().push(1); + collectRenderer.methodGen().math(GeneratorAdapter.ADD, INT_TYPE); + collectRenderer.methodGen().putField(self, Regrouper.badCount(prefix()), INT_TYPE); + } + collectRenderer.methodGen().mark(end); } @Override - public void loadConstructorArgument() { - // No argument to constructor. + public void failIfBad(GeneratorAdapter okMethod, Label failure) { + consumer.failIfBad(okMethod, prefix(), self, this::failInnerIfBad, failure); } } @@ -1090,14 +1091,10 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { okMethod.loadThis(); okMethod.getField(self, fieldName + "$ok", BOOLEAN_TYPE); - final var next = okMethod.newLabel(); - okMethod.ifZCmp(GeneratorAdapter.NE, next); - okMethod.push(false); - okMethod.returnValue(); - okMethod.mark(next); + okMethod.ifZCmp(GeneratorAdapter.EQ, failure); } @Override @@ -1176,8 +1173,8 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { - // Optima with deafults are always ok. + public void failIfBad(GeneratorAdapter okMethod, Label failure) { + // Optima with defaults are always ok. } @Override @@ -1249,7 +1246,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Do nothing } @@ -1303,7 +1300,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Sum cannot fail } @@ -1379,16 +1376,12 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { okMethod.loadThis(); okMethod.getField(self, fieldName, A_SET_TYPE); okMethod.invokeInterface(A_SET_TYPE, SET__SIZE); okMethod.push(1); - final var next = okMethod.newLabel(); - okMethod.ifICmp(GeneratorAdapter.EQ, next); - okMethod.push(0); - okMethod.returnValue(); - okMethod.mark(next); + okMethod.ifICmp(GeneratorAdapter.NE, failure); } @Override @@ -1480,7 +1473,7 @@ public Stream constructorType() { } @Override - public void failIfBad(GeneratorAdapter okMethod) { + public void failIfBad(GeneratorAdapter okMethod, Label failure) { // Univalued with default is always ok } @@ -1494,7 +1487,6 @@ public void loadConstructorArgument() { private static final Type A_ITERATOR_TYPE = Type.getType(Iterator.class); private static final Type A_MAP_TYPE = Type.getType(Map.class); private static final Type A_OBJECT_TYPE = Type.getType(Object.class); - private static final Type A_OPTIONAL_TYPE = Type.getType(Optional.class); private static final Type A_PARTITION_COUNT_TYPE = Type.getType(PartitionCount.class); private static final Type A_SET_TYPE = Type.getType(Set.class); private static final Type A_STRING_TYPE = Type.getType(String.class); @@ -1541,7 +1533,7 @@ public void loadConstructorArgument() { new Method("iterator", A_ITERATOR_TYPE, new Type[] {}); private static final Method SET__SIZE = new Method("size", INT_TYPE, new Type[] {}); private final ClassVisitor classVisitor; - private final Renderer collectRenderer; + private Renderer collectRenderer; public final int collectedSelfArgument; private final List elements = new ArrayList<>(); @@ -1628,8 +1620,10 @@ public Regrouper addObject(String fieldName, Stream> fields } @Override - public void addOnlyIf(Imyhat valueType, String fieldName, Consumer loader) { - elements.add(new OnlyIf(valueType, fieldName, loader)); + public Regrouper addOnlyIf(String fieldName, OnlyIfConsumer consumer) { + final var element = new OnlyIf(fieldName, consumer); + elements.add(element); + return element; } @Override @@ -1764,9 +1758,13 @@ public void finish() { final var okMethod = new GeneratorAdapter(Opcodes.ACC_PUBLIC, METHOD_IS_OK, null, null, classVisitor); okMethod.visitCode(); - elements.forEach(element -> element.failIfBad(okMethod)); + final var failure = okMethod.newLabel(); + elements.forEach(element -> element.failIfBad(okMethod, failure)); okMethod.push(true); okMethod.returnValue(); + okMethod.mark(failure); + okMethod.push(false); + okMethod.returnValue(); okMethod.visitMaxs(0, 0); okMethod.visitEnd(); diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/Regrouper.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/Regrouper.java index 1d675cbb0..fad5da788 100644 --- a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/Regrouper.java +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/Regrouper.java @@ -2,12 +2,39 @@ import ca.on.oicr.gsi.Pair; import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Stream; +import org.objectweb.asm.Label; import org.objectweb.asm.Type; +import org.objectweb.asm.commons.GeneratorAdapter; /** Build a grouping operation */ public interface Regrouper { + static String badCount(String fieldName) { + return fieldName + " Bad Count"; + } + + static String goodCount(String fieldName) { + return fieldName + " Good Count"; + } + + interface OnlyIfConsumer { + void build(Renderer renderer, String fieldName, Type owner, Label failurePath); + + boolean countsRequired(); + + void failIfBad( + GeneratorAdapter checker, + String fieldName, + Type owner, + BiConsumer checkInner, + Label failure); + + void renderGetter(GeneratorAdapter getter, String fieldName, Type owner); + + Type type(); + } /** * Add a new collection of values slurped during iteration @@ -84,11 +111,10 @@ void addLexicalConcat( * *

The row will be rejected if it has zero or 2 or more items. * - * @param valueType the type of the values in the collection * @param fieldName the name of the output variable - * @param loader a function to load the variable; this must return an optional + * @param consumer the behaviour for checking the optional value */ - void addOnlyIf(Imyhat valueType, String fieldName, Consumer loader); + Regrouper addOnlyIf(String fieldName, OnlyIfConsumer consumer); /** * A single value which is the optima from all input values diff --git a/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/UnboxableExpression.java b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/UnboxableExpression.java new file mode 100644 index 000000000..424b5107f --- /dev/null +++ b/shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/compiler/UnboxableExpression.java @@ -0,0 +1,81 @@ +package ca.on.oicr.gsi.shesmu.compiler; + +import ca.on.oicr.gsi.shesmu.plugin.types.Imyhat; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; + +class UnboxableExpression implements TargetWithContext { + + private Optional defs = Optional.empty(); + private final ExpressionNode expression; + private final String name; + + public UnboxableExpression(ExpressionNode expression) { + this.expression = expression; + name = String.format("Lift of %d:%d", expression.line(), expression.column()); + } + + public void collectFreeVariables(Set names, Predicate predicate) { + expression.collectFreeVariables(names, predicate); + } + + public void collectPlugins(Set pluginFileNames) { + expression.collectPlugins(pluginFileNames); + } + + @Override + public Flavour flavour() { + return Flavour.LAMBDA; + } + + @Override + public String name() { + return name; + } + + @Override + public void read() { + // Super. Don't care. + } + + public void render(Renderer renderer) { + expression.render(renderer); + } + + public String renderEcma(EcmaScriptRenderer renderer) { + return expression.renderEcma(renderer); + } + + public boolean resolve(NameDefinitions defs, Consumer errorHandler) { + return expression.resolve(this.defs.map(defs::withShadowContext).orElse(defs), errorHandler); + } + + public boolean resolveDefinitions( + ExpressionCompilerServices expressionCompilerServices, Consumer errorHandler) { + return expression.resolveDefinitions(expressionCompilerServices, errorHandler); + } + + @Override + public void setContext(NameDefinitions defs) { + this.defs = Optional.of(defs); + } + + @Override + public Imyhat type() { + return expression.type() instanceof Imyhat.OptionalImyhat + ? ((Imyhat.OptionalImyhat) expression.type()).inner() + : expression.type(); + } + + public boolean typeCheck(Consumer errorHandler) { + final var captureOk = expression.typeCheck(errorHandler); + if (captureOk && !expression.type().isSame(expression.type().asOptional())) { + expression.typeError("optional", expression.type(), errorHandler); + return false; + } + return captureOk; + } +} diff --git a/shesmu-server/src/test/resources/run/group-onlyif.shesmu b/shesmu-server/src/test/resources/run/group-onlyif.shesmu deleted file mode 100644 index 1eea51af6..000000000 --- a/shesmu-server/src/test/resources/run/group-onlyif.shesmu +++ /dev/null @@ -1,6 +0,0 @@ -Version 1; -Input test; - -Olive - Group By workflow Into a = OnlyIf `project` - Run ok With ok = a == "the_foo_study"; diff --git a/shesmu-server/src/test/resources/run/group-optional-flatten.shesmu b/shesmu-server/src/test/resources/run/group-optional-flatten.shesmu new file mode 100644 index 000000000..a50760e5b --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-flatten.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = `project` + Group By workflow Into a = `Univalued project?` + Run ok With ok = a == "the_foo_study"; diff --git a/shesmu-server/src/test/resources/run/group-optional-flatten2.shesmu b/shesmu-server/src/test/resources/run/group-optional-flatten2.shesmu new file mode 100644 index 000000000..2d65a48db --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-flatten2.shesmu @@ -0,0 +1,10 @@ +Version 1; +Input test; + +project_info = Dict { + "the_foo_study" = 3 As json +}; + +Olive + Group By workflow Into a = Require Any `Univalued (project_info[project]? As integer)?` + Run ok With ok = a == 3; diff --git a/shesmu-server/src/test/resources/run/group-optional-onlyif-all.shesmu b/shesmu-server/src/test/resources/run/group-optional-onlyif-all.shesmu new file mode 100644 index 000000000..4424b0886 --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-onlyif-all.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = `project` + Group By workflow Into a = OnlyIf All `Univalued project?` + Run ok With ok = a == `"the_foo_study"`; diff --git a/shesmu-server/src/test/resources/run/group-optional-onlyif-all2.shesmu b/shesmu-server/src/test/resources/run/group-optional-onlyif-all2.shesmu new file mode 100644 index 000000000..e39f3ef5d --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-onlyif-all2.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = If library_size == 307 Then `project` Else `` + Group By workflow Into a = OnlyIf All `Univalued project?` + Run ok With ok = a == ``; diff --git a/shesmu-server/src/test/resources/run/group-optional-onlyif-any.shesmu b/shesmu-server/src/test/resources/run/group-optional-onlyif-any.shesmu new file mode 100644 index 000000000..d97b90d2b --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-onlyif-any.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = If library_size == 307 Then `project` Else `` + Group By workflow Into a = OnlyIf Any `Univalued project?` + Run ok With ok = a == `"the_foo_study"`; diff --git a/shesmu-server/src/test/resources/run/group-optional-require-all-fail.shesmu b/shesmu-server/src/test/resources/run/group-optional-require-all-fail.shesmu new file mode 100644 index 000000000..fc8bd9415 --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-require-all-fail.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = If library_size == 307 Then `project` Else `` + Group By workflow Into a = Require All `Univalued project?` + Run ok With ok = a == "the_foo_study"; diff --git a/shesmu-server/src/test/resources/run/group-optional-require-all.shesmu b/shesmu-server/src/test/resources/run/group-optional-require-all.shesmu new file mode 100644 index 000000000..d745c3c91 --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-require-all.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = `project` + Group By workflow Into a = Require All `Univalued project?` + Run ok With ok = a == "the_foo_study"; diff --git a/shesmu-server/src/test/resources/run/group-optional-require-any-fail.shesmu b/shesmu-server/src/test/resources/run/group-optional-require-any-fail.shesmu new file mode 100644 index 000000000..02605bb5f --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-require-any-fail.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = If False Then `"foo"` Else `` + Group By workflow Into a = Require Any `Univalued project?` + Run ok With ok = a == "the_foo_study"; diff --git a/shesmu-server/src/test/resources/run/group-optional-require-any.shesmu b/shesmu-server/src/test/resources/run/group-optional-require-any.shesmu new file mode 100644 index 000000000..c3980a069 --- /dev/null +++ b/shesmu-server/src/test/resources/run/group-optional-require-any.shesmu @@ -0,0 +1,7 @@ +Version 1; +Input test; + +Olive + Let workflow, project = If library_size == 307 Then `project` Else `` + Group By workflow Into a = Require Any `Univalued project?` + Run ok With ok = a == "the_foo_study";