diff --git a/src/sly/parser/parser/Parser.cs b/src/sly/parser/parser/Parser.cs index b133b61b..30c710bd 100644 --- a/src/sly/parser/parser/Parser.cs +++ b/src/sly/parser/parser/Parser.cs @@ -44,7 +44,12 @@ public virtual BuildResult> BuildExpressionParser( var expressionGenerator = new ExpressionRulesGenerator(I18n); exprResult = expressionGenerator.BuildExpressionRules(Configuration, Instance.GetType(), exprResult); Configuration = exprResult.Result; - SyntaxParser.Init(exprResult.Result, startingRule); + if (exprResult.IsOk) + { + // #540 : recompute starting tokens taking account of expression rules + SyntaxParser.Init(exprResult.Result, startingRule); + } + if (startingRule != null) { Configuration.StartingRule = startingRule; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs index 951b821c..54ab25d4 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs @@ -247,6 +247,11 @@ private SyntaxParseResult NoMatchingRuleError(IList> tokens, public virtual void Init(ParserConfiguration configuration, string root) { if (root != null) StartingNonTerminal = root; + // #540 : reset all leading tokens and recompute with the new configuration (expression rules) + foreach (var nonTerminal in configuration.NonTerminals.Values) + { + nonTerminal?.Rules?.ForEach(x => x?.PossibleLeadingTokens?.Clear()); + } InitializeStartingTokens(configuration, StartingNonTerminal); } diff --git a/tests/ParserTests/Issue540/Issue540Parser.cs b/tests/ParserTests/Issue540/Issue540Parser.cs new file mode 100644 index 00000000..ed6603bc --- /dev/null +++ b/tests/ParserTests/Issue540/Issue540Parser.cs @@ -0,0 +1,254 @@ + +using sly.parser.generator; +using sly.parser.parser; +using System.Collections.Generic; +using sly.lexer; + + +namespace ParserTests.Issue540 +{ + public class Issue540Parser + { + + [Production($"NtSection: NtStatement*")] + public object NTSection(List a) => null; + + [Production( + $"NtStatement: [NtSCExpr | NtLoop | NtCond | NtFunction | NtBlock | NtReturnStatement | NtLoopControlStatement]")] + public object NtStatement(object subStatement) => null; + + [Production($"NtSCExpr: NtExpression Semicolon [d]")] + + public object NtScExpr(object expression) => null; + + [Production($"NtReturnStatement: Return [d] NtSCExpr")] + public object NtReturnStatement(object expr) => null; + + [Production($"NtLoopControlStatement: [Break | Continue] NtNestedValueInLoopControl?")] + public object NtLoopControlStatement(Issue540Token op, ValueOption nestedVal) => null; + + [Production($"NtNestedValueInLoopControl: Identifier")] + public object NtNestedValueInLoopControl(Token val) => null; + + [Production($"NtLoop: NtForLoopHeader NtLoopLabel? NtStatement NtElse?")] + [Production($"NtLoop: NtWhileLoopHeader NtLoopLabel? NtStatement NtElse?")] + public object NtLoop(object loopHeader, ValueOption label, object statementExpr, + ValueOption slseExpr) => null; + + [Production($"NtLoopLabel: As [d] Identifier")] + public object NtLoopLabel(Token ident) => null; + + [Production( + $"NtForLoopHeader: For [d] OpenParen [d] NtExpression Semicolon [d] NtExpression Semicolon [d] NtExpression CloseParen [d]")] + public object NtForLoopHeader(object init, object condition, object step) => null; + + [Production($"NtWhileLoopHeader: While [d] OpenParen [d] NtExpression CloseParen [d]")] + public object NtWhileLoopHeader(object condition) => null; + + [Production($"NtCond: [NtSwitch | NtIf]")] + public object NtCond(object _) => null; + + [Production($"NtIf: If [d] OpenParen [d] NtExpression CloseParen [d] NtStatement NtElse?")] + public object NtIf(object a, object b, ValueOption c) => null; + + [Production($"NtElse: Else [d] NtStatement")] + public object NtElse(object _) => null; + + [Production( + $"NtSwitch: Switch [d] OpenParen [d] NtExpression CloseParen [d] OpenCurly [d] NtSwitchBody* CloseCurly [d]")] + public object NtSwitch(object a, List b) => null; + + [Production($"NtSwitchBody: NtExpression Colon [d] NtStatement")] + public object NtSwitchBody(object a, object b) => null; + + [Production($"NtFunction: NtType Identifier OpenParen [d] NtTypeAndIdentifierCSV? CloseParen [d] NtStatement")] + public object NtFunction(object a, Token b, ValueOption c, object d) => + null; + + [Production($"NtTypeAndIdentifierCSV: NtTypeAndIdentifierCSVElement (Comma [d] NtTypeAndIdentifierCSV)*")] + public object NtTypeAndIdentifierCsv(object a, List> b) => null; + + [Production($"NtTypeAndIdentifierCSVElement: NtFunctionArgDeclModifiersCombined? NtType Identifier")] + public object NtTypeAndIdentifierCsvElement(ValueOption a, object b, Token c) => + null; + + [Production($"NtBlock: OpenCurly [d] NtSection CloseCurly [d]")] + public object NtBlock(object a) => null; + + [Production($"NtExpression: NtAliasExpr")] + public object NtExpression(object pass) => pass; + + [Production($"NtAliasExpr: [NtAliasExpr1 | NtAliasExpr2 | NtAliasExpr3 | NtDeclarationExpr]")] + public object NtAliasExpr(object a) => null; + + [Production($"NtAliasExpr1: Identifier As [d] Identifier")] + public object NtAliasExpr1(Token a, Token b) => null; + + [Production($"NtAliasExpr2: Identifier As [d] NtType Identifier")] + public object NtAliasExpr2(Token a, object b, Token v) => null; + + [Production($"NtAliasExpr3: Identifier As [d] NtType")] + public object NtAliasExpr3(Token a, object b) => null; + + [Production($"NtDeclarationExpr: [NtDeclarationExpr1 | NtAssignmentExpr]")] + public object NtDeclarationExpr(object a) => null; + + [Production($"NtDeclarationExpr1: NtDeclarationModifiersCombined? NtType Identifier NtAssignmentPrime?")] + public object NtDeclarationExpr1(ValueOption a, object b, Token c, + ValueOption d) => null; + + [Production($"NtDeclarationModifiersCombined: NtDeclarationModifier+")] + public object NtDeclarationModifiersCombined(List a) => null; + + [Production($"NtDeclarationModifier: [Ref | Readonly | Frozen | Immut]")] + public object NtDeclarationModifier(Token a) => null; + + [Production($"NtFunctionArgDeclModifier: [Ref | Readonly | Frozen | Immut | Copy]")] + public object NtFunctionArgDeclModifier(Token a) => null; + + [Production($"NtFunctionArgDeclModifiersCombined: NtFunctionArgDeclModifier+")] + public object NtFunctionArgDeclModifiersCombined(List a) => null; + + [Production($"NtAssignmentPrime: Equals NtExpression")] + public object NtAssignmentPrime(Token a, object b) => null; + + [Production($"NtAssignmentExpr: NtAssignmentExpr1")] + public object NtAssignmentExpr(object a) => null; + + [Production($"NtAssignmentExpr1: Issue540Parser_expressions")] + public object NtAssignmentExpr1(object a) => null; + + [Operand] + [Production($"NtPrimary: NtLPrimary NtPrimaryPrime?")] + public object NtPrimary(object a, ValueOption b) => null; + + [Production($"NtPrimaryPrime: [NtIndexPrime | NtFunctionCallPrime]")] + public object NtPrimaryPrime(object a) => null; + + [Production($"NtIndexPrime: OpenSquare [d] NtExpression CloseSquare [d]")] + public object NtIndexPrime(object a) => null; + + [Production($"NtLPrimary: [NtNewExpr | NtLPrimary1 | NtLPrimary2 | NtLPrimary3]")] + public object NtLPrimary(object a) => null; + + [Production($"NtLPrimary1: OpenParen [d] NtExpression CloseParen [d]")] + public object NtLPrimary1(object a) => null; + + [Production($"NtLPrimary2: Copy [d] NtExpression")] + public object NtLPrimary2(object a) => null; + + [Production($"NtLPrimary3: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] + public object NtLPrimary3(Token a) => null; + + [Production($"NtNewExpr: New [d] NtType OpenParen [d] NtArgList?")] + public object NtNewExpr(object a, ValueOption b) => null; + + [Production($"NtFunctionCallPrime:OpenParen [d] NtArgList? CloseParen [d]")] + public object NtFunctionCallPrime(ValueOption a) => null; + + [Production($"NtArgList: NtArgListElement NtArgListPrime*")] + public object NtArgList(object a, List b) => null; + + [Production($"NtArgListElement: NtArgumentLabel? NtExpression")] + public object NtArgListElement(ValueOption a, object b) => null; + + [Production($"NtArgListPrime: Comma [d] NtArgListElement")] + public object NtArgListPrime(object a) => null; + + [Production($"NtArgumentLabel: Identifier Colon [d]")] + public object NtArgumentLabel(Token ident) => null; + + [Production($"NtTypeCSV: NtType (Comma [d] NtType)*")] + public object NtTypeCsv(object aType, List> otherTypes) => null; + + [Production($"NtType: [NtBaseType | NtGenericType]")] + public object NtType(object node) => null; + + [Production( + $"NtGenericType: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] NtTypeCSV CloseAngleSquare [d]")] + public object NtGenericType(Token typeToken, object typeArgs) => null; + + [Production( + $"NtBaseType: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] + public object NtBaseType(Token typeToken) => null; + + #region ComparisonExpressions + + const int EqualToPrecedence = NotExprPrecedence + 1; + const int NotEqualToPrecedence = EqualToPrecedence; + const int GreaterThanPrecedence = NotEqualToPrecedence; + const int LessThanPrecedence = GreaterThanPrecedence; + const int GreaterThanOrEqualToPrecedence = LessThanPrecedence; + const int LessThanOrEqualToPrecedence = GreaterThanOrEqualToPrecedence; + + [Infix((int)Issue540Token.EqualTo, Associativity.Right, EqualToPrecedence)] + [Infix((int)Issue540Token.NotEqualTo, Associativity.Right, NotEqualToPrecedence)] + [Infix((int)Issue540Token.GreaterThan, Associativity.Right, GreaterThanPrecedence)] + [Infix((int)Issue540Token.LessThan, Associativity.Right, LessThanPrecedence)] + [Infix((int)Issue540Token.GreaterThanOrEqualTo, Associativity.Right, GreaterThanOrEqualToPrecedence)] + [Infix((int)Issue540Token.LessThanOrEqualTo, Associativity.Right, LessThanOrEqualToPrecedence)] + public object ComparisonExpressions(object a, Token b, object c) => null; + + #endregion + + #region OtherExpressions + + const int AssignmentExprPrecedence = 1; + const int ImpliesExprPrecedence = AssignmentExprPrecedence + 1; + const int OrExprPrecedence = ImpliesExprPrecedence + 1; + + const int XorExprPrecedence = OrExprPrecedence + 1; + + const int AndExprPrecedence = XorExprPrecedence + 1; + + const int NotExprPrecedence = AndExprPrecedence + 1; + const int AdditionPrecedence = EqualToPrecedence + 1; + const int SubtractionPrecedence = AdditionPrecedence; + + const int MultiplicationPrecedence = SubtractionPrecedence + 1; + const int DivisionPrecedence = MultiplicationPrecedence; + + const int PowerPrecedence = DivisionPrecedence + 1; + + const int NegationPrecedence = PowerPrecedence + 1; + + const int FactorialPrecedence = NegationPrecedence + 1; + const int RShiftPrecedence = FactorialPrecedence + 1; + const int LShiftPrecedence = RShiftPrecedence + 1; + const int BitwiseOrExprPrecedence = LShiftPrecedence + 1; + + const int BitwiseXorExprPrecedence = BitwiseOrExprPrecedence + 1; + + const int BitwiseAndExprPrecedence = BitwiseXorExprPrecedence + 1; + + const int BitwiseNotExprPrecedence = BitwiseAndExprPrecedence + 1; + + #endregion + + [Postfix((int)Issue540Token.Factorial, Associativity.Left, FactorialPrecedence)] + public object Factorial(object a, Token b) => null; + + [Prefix((int)Issue540Token.LogicalNot, Associativity.Right, NotExprPrecedence)] + [Prefix((int)Issue540Token.BitwiseNegation, Associativity.Right, BitwiseNotExprPrecedence)] + [Prefix((int)Issue540Token.Subtraction, Associativity.Right, NegationPrecedence)] + public object Prefix(Token a, object b) => null; + + + [Infix((int)Issue540Token.Equals, Associativity.Right, 1)] + [Infix((int)Issue540Token.LogicalOr, Associativity.Left, 1)] + [Infix((int)Issue540Token.LogicalXor, Associativity.Left, 1)] + [Infix((int)Issue540Token.LogicalAnd, Associativity.Left, 1)] + [Infix((int)Issue540Token.Addition, Associativity.Left, 1)] + [Infix((int)Issue540Token.Subtraction, Associativity.Left, 1)] + [Infix((int)Issue540Token.Division, Associativity.Left, 1)] + [Infix((int)Issue540Token.Multiplication, Associativity.Left, 1)] + [Infix((int)Issue540Token.Exponentiation, Associativity.Right, 1)] + [Infix((int)Issue540Token.Subtraction, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseOr, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseXor, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseAnd, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseLeftShift, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseRightShift, Associativity.Left, 1)] + public object BinOp(object left, Token a, object b) => null; + } +} \ No newline at end of file diff --git a/tests/ParserTests/Issue540/Issue540Token.cs b/tests/ParserTests/Issue540/Issue540Token.cs new file mode 100644 index 00000000..7321e481 --- /dev/null +++ b/tests/ParserTests/Issue540/Issue540Token.cs @@ -0,0 +1,154 @@ +using System.ComponentModel.DataAnnotations; +using sly.lexer; + +namespace ParserTests.Issue540 +{ + public enum Issue540Token + { + EOF, + + //should be shared between all sets of valid tokens, since invalid ones will simply be unused + [Lexeme(GenericToken.Int)] [Lexeme(GenericToken.Double)] + Number, + + [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "_A-Za-z", "_0-9A-Za-z")] + Identifier, + + [Lexeme(GenericToken.String)] [Lexeme(GenericToken.String, "'")] + String, + + //Operators + + #region bitwise operator + + [Sugar("&")] BitwiseAnd, + [Sugar("|")] BitwiseOr, + [Sugar("^")] BitwiseXor, + [Sugar("<<")] BitwiseLeftShift, + [Sugar(">>")] BitwiseRightShift, + [Sugar("~")] BitwiseNegation, + + #endregion + + #region arithmetic operator + + [Sugar("+")] Addition, + [Sugar("-")] Subtraction, + [Sugar("*")] Multiplication, + [Sugar("/")] Division, + [Sugar("**")] Exponentiation, + [Sugar("!")] Factorial, + + #endregion + + #region logical operator + + [Keyword("and")] LogicalAnd, + [Keyword("or")] LogicalOr, + [Keyword("not")] LogicalNot, + [Keyword("xor")] LogicalXor, + [Keyword("implies")] LogicalImplies, + + #endregion + + #region Brackets + + [Sugar("(")] OpenParen, + [Sugar(")")] CloseParen, + [Sugar("{")] OpenCurly, + [Sugar("}")] CloseCurly, + [Sugar("[")] OpenSquare, + [Sugar("]")] CloseSquare, + [Sugar("<[")] OpenAngleSquare, + [Sugar("]>")] CloseAngleSquare, + + #endregion + + #region Symbols + + [Sugar(",")] Comma, + [Sugar(":")] Colon, + [Sugar(".")] Dot, + + #endregion + + #region Comparison_Operator + + [Sugar("==")] EqualTo, + [Sugar("!=")] NotEqualTo, + [Sugar(">")] GreaterThan, + [Sugar("<")] LessThan, + [Sugar(">=")] GreaterThanOrEqualTo, + [Sugar("<=")] LessThanOrEqualTo, + [Keyword("is")] ComparisonIs, + [Keyword("in")] In, + [Sugar(";")] Semicolon, + + #endregion + + [Sugar("=")] Equals, + + #region Types + + [Keyword("float")] TypeFloat, + [Keyword("int")] TypeInt, + [Keyword("double")] TypeDouble, + + [Keyword("number")] [Keyword("bigfloat")] + TypeNumber, + [Keyword("long")] TypeLong, + + [Keyword("longint")] [Keyword("bigint")] + TypeLongInt, + [Keyword("byte")] TypeByte, + [Keyword("array")] TypeArray, + [Keyword("list")] TypeList, + [Keyword("set")] TypeSet, + [Keyword("dict")] TypeDict, + [Keyword("short")] TypeShort, + [Keyword("rational")] TypeRational, + [Keyword("string")] TypeString, + [Keyword("char")] TypeChar, + [Keyword("void")] TypeVoid, + + #endregion + + #region control-flow + + [Keyword("return")] Return, + [Keyword("break")] Break, + [Keyword("continue")] Continue, + [Keyword("if")] If, + [Keyword("else")] Else, + [Keyword("switch")] Switch, + + #endregion + + #region Loops + + [Keyword("for")] For, + [Keyword("while")] While, + + #endregion + + [Keyword("as")] As, + + #region FunctionMod + + [Keyword("cascading")] Cascading, + [Keyword("copy")] Copy, + [Keyword("readonly")] Readonly, + [Keyword("frozen")] Frozen, + [Keyword("immut")] Immut, + [Keyword("ref")] Ref, + [Keyword("new")] New, + + #endregion + + [Keyword("true")] TrueLiteral, + [Keyword("false")] FalseLiteral, + [Keyword("collection")] TypeCollection, + [Comment("#", "/*", "*/")] Comment + + } +} \ No newline at end of file diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index b61715bc..f9de9095 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Linq; using NFluent; using ParserTests.Issue218; @@ -7,6 +8,7 @@ using ParserTests.Issue260; using ParserTests.Issue277; using ParserTests.Issue536; +using ParserTests.Issue540; using SlowEOS; using sly.buildresult; using sly.lexer; @@ -135,6 +137,18 @@ public static void Issue536Test() Check.That(equals.TokenID).IsEqualTo(Token536.Equals); var x = Token536.Equals; } + + [Fact] + public static void Issue540Test() + { + ParserBuilder builder = new ParserBuilder(); + var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NtSCExpr"); + Check.That(buildParser).IsOk(); + + var parser = buildParser.Result; + var parsed = parser.Parse("1;"); + Check.That(parsed).IsOkParsing(checkIfResultisNull:false); + } } diff --git a/tests/ParserTests/NFluentParseExtensions.cs b/tests/ParserTests/NFluentParseExtensions.cs index 158429a0..5c2cd6ef 100644 --- a/tests/ParserTests/NFluentParseExtensions.cs +++ b/tests/ParserTests/NFluentParseExtensions.cs @@ -11,11 +11,11 @@ namespace ParserTests { public static class NFluentParseExtensions { - public static ICheckLink>> IsOkParsing(this ICheck> context) where IN : struct + public static ICheckLink>> IsOkParsing(this ICheck> context, bool checkIfResultisNull = true) where IN : struct { ExtensibilityHelper.BeginCheck(context) .FailWhen(sut => sut.IsError, $"parse failed") - .FailWhen(sut => sut.Result == null && !sut.SyntaxTree.IsEpsilon, "parse result is null") + .FailWhen(sut => sut.Result == null && checkIfResultisNull && !sut.SyntaxTree.IsEpsilon, "parse result is null") .OnNegate("parse expected to fail.") .EndCheck(); return ExtensibilityHelper.BuildCheckLink(context);