From d29e482cabf31708c1ff73c7569a4f1832c3e377 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 09:22:27 +0100 Subject: [PATCH 1/7] . --- tests/ParserTests/Issue540/Issue540Parser.cs | 322 +++++++++++++++++++ tests/ParserTests/Issue540/Issue540Token.cs | 6 + 2 files changed, 328 insertions(+) create mode 100644 tests/ParserTests/Issue540/Issue540Parser.cs create mode 100644 tests/ParserTests/Issue540/Issue540Token.cs diff --git a/tests/ParserTests/Issue540/Issue540Parser.cs b/tests/ParserTests/Issue540/Issue540Parser.cs new file mode 100644 index 00000000..9b4bf98d --- /dev/null +++ b/tests/ParserTests/Issue540/Issue540Parser.cs @@ -0,0 +1,322 @@ + +using System.Data; +using Common.AST; +using Common.Tokens; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; +using LyToken = sly.lexer.Token; +using NodeType = Common.AST.DynamicASTNode; +namespace ParserTests.Issue540 +{ + public partial class SmallLangParser + { + void AppendIfNotEmpty(List nodeList, ValueOption Considered) + { + if (GetFromValOp(Considered) is NodeType node) + { + nodeList.Add(node); + } + } + List BuildChildren(params object[] Vals) + { + List rc = new(); + foreach (var val in Vals) + { + if (val is NodeType node) + { + rc.Add(node); + } + else if (val is ValueOption valOp) + { + AppendIfNotEmpty(rc, valOp); + } + else if (val is LyToken TVal) + { + var Token = FromToken(TVal); + if (Token == null) continue; + if (Token.TT == TokenType.Identifier) + { + rc.Add(new NodeType(Token, [], ASTNodeType.Identifier)); + } + else + { + rc.Add(new NodeType(Token, [], ASTNodeType.Terminal)); + } + } + else + { + throw new Exception($"Unexpected type: {val.GetType().Name}"); + } + } + return rc; + } + NodeType? GetFromValOp(ValueOption value) + { + if (value.IsSome) + { + return value.Match(x => x, () => throw new Exception("IsSome was true when no value was found")); + } + return null; + } + IToken? FromToken(LyToken t) => t.IsEmpty ? null : IToken.NewToken(t.TokenID, t.Value, t.Position.Index, null); + [Production($"{nameof(NTSection)}: {nameof(NTStatement)}*")] + public NodeType NTSection(List Statements) + { + return new NodeType(null, Statements, ASTNodeType.Section); + } + [Production($"{nameof(NTStatement)}: [{nameof(NTSCExpr)} | {nameof(NTLoop)} | {nameof(NTCond)} | {nameof(NTFunction)} | {nameof(NTBlock)} | {nameof(NTReturnStatement)} | {nameof(NTLoopControlStatement)}]")] + public NodeType NTStatement(NodeType SubStatement) => SubStatement; + [Production($"{nameof(NTSCExpr)}: {nameof(NTExpression)}")] + + public NodeType NTSCExpr(NodeType Expression) => Expression; + [Production($"{nameof(NTReturnStatement)}: Return [d] {nameof(NTSCExpr)}")] + public NodeType NTReturnStatement(NodeType SCExpr) => SCExpr with { NodeType = ASTNodeType.Return }; + [Production($"{nameof(NTLoopControlStatement)}: [Break | Continue] {nameof(NTNestedValueInLoopControl)}?")] + public NodeType NTLoopControlStatement(LyToken Operator, ValueOption NestedVal) + { + List rc = []; + AppendIfNotEmpty(rc, NestedVal); + return new(FromToken(Operator), rc, ASTNodeType.LoopCTRL); + } + [Production($"{nameof(NTNestedValueInLoopControl)}: Identifier")] + public NodeType NTNestedValueInLoopControl(LyToken val) => new NodeType(FromToken(val), [], ASTNodeType.ValInLCTRL); + + [Production($"{nameof(NTLoop)}: {nameof(NTForLoopHeader)} {nameof(NTLoopLabel)}? {nameof(NTStatement)} {nameof(NTElse)}?")] + [Production($"{nameof(NTLoop)}: {nameof(NTWhileLoopHeader)} {nameof(NTLoopLabel)}? {nameof(NTStatement)} {nameof(NTElse)}?")] + public NodeType NTLoop(NodeType LoopHeader, ValueOption Label, NodeType StatementExpr, ValueOption ElseExpr) + { + var rt = LoopHeader.NodeType; + List rc = LoopHeader.Children; + AppendIfNotEmpty(rc, Label); + rc.Add(StatementExpr); + AppendIfNotEmpty(rc, ElseExpr); + return new NodeType(null, rc, rt); + } + [Production($"{nameof(NTLoopLabel)}: As [d] Identifier")] + public NodeType NTLoopLabel(LyToken ident) => new NodeType(FromToken(ident), [], ASTNodeType.LoopLabel); + [Production($"{nameof(NTForLoopHeader)}: For [d] OpenParen [d] {nameof(NTExpression)} Semicolon [d] {nameof(NTExpression)} Semicolon [d] {nameof(NTExpression)} CloseParen [d]")] + public NodeType NTForLoopHeader(NodeType Init, NodeType Condition, NodeType Step) => new NodeType(null, [Init, Condition, Step], ASTNodeType.For); + [Production($"{nameof(NTWhileLoopHeader)}: While [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d]")] + public NodeType NTWhileLoopHeader(NodeType Condition) => new NodeType(null, [Condition], ASTNodeType.While); + [Production($"{nameof(NTCond)}: [{nameof(NTSwitch)} | {nameof(NTIf)}]")] + public NodeType NTCond(NodeType C) => C; + [Production($"{nameof(NTIf)}: If [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d] {nameof(NTStatement)} {nameof(NTElse)}?")] + public NodeType NTIf(NodeType Cond, NodeType StatementExpr, ValueOption ElseExpr) + { + var ThisCombined = new NodeType(null, [Cond, StatementExpr], ASTNodeType.ExprStatementCombined); + List rc = [ThisCombined]; + if (GetFromValOp(ElseExpr) is NodeType node) + { + if (node.NodeType == ASTNodeType.If) + { + rc.AddRange(node.Children); //should automatically add the remaining elifs and else as well + } + else + { + rc.Add(node); //node is statement or child thereof (the latter is probably what it is but we don't care) + } + } + return new(null, rc, ASTNodeType.If); + } + [Production($"{nameof(NTElse)}: Else [d] {nameof(NTStatement)}")] + public NodeType NTElse(NodeType AStatement) => AStatement; //passthrough + [Production($"{nameof(NTSwitch)}: Switch [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d] OpenCurly [d] {nameof(NTSwitchBody)}* CloseCurly [d]")] + public NodeType NTSwitch(NodeType AExpression, List ASwitchBody) => new NodeType(null, [AExpression, .. ASwitchBody], ASTNodeType.Switch); + + [Production($"{nameof(NTSwitchBody)}: {nameof(NTExpression)} Colon [d] {nameof(NTStatement)}")] + public NodeType NTSwitchBody(NodeType AExpr, NodeType AStatement) => new(null, [AExpr, AStatement], ASTNodeType.ExprStatementCombined); + [Production($"{nameof(NTFunction)}: {nameof(NTType)} Identifier OpenParen [d] {nameof(NTTypeAndIdentifierCSV)}? CloseParen [d] {nameof(NTStatement)}")] + public NodeType NTFunction(NodeType AType, LyToken Ident, ValueOption TICSV, NodeType Statement) => new(FromToken(Ident), BuildChildren(AType, TICSV, Statement), ASTNodeType.Function); + [Production($"{nameof(NTTypeAndIdentifierCSV)}: {nameof(NTTypeAndIdentifierCSVElement)} (Comma [d] {nameof(NTTypeAndIdentifierCSV)})*")] + public NodeType NTTypeAndIdentifierCSV(NodeType Element, List> Prime) => new(null, [Element, .. Prime.Select(x => x.Items.First().Value)], ASTNodeType.TypeAndIdentifierCSV); + [Production($"{nameof(NTTypeAndIdentifierCSVElement)}: {nameof(NTFunctionArgDeclModifiersCombined)}? {nameof(NTType)} Identifier")] + public NodeType NTTypeAndIdentifierCSVElement(ValueOption Modifiers, NodeType AType, LyToken Ident) => new(FromToken(Ident), BuildChildren(Modifiers, AType), ASTNodeType.TypeAndIdentifierCSVElement); + [Production($"{nameof(NTBlock)}: OpenCurly [d] {nameof(NTSection)} CloseCurly [d]")] + public NodeType NTBlock(NodeType ASection) => ASection; + [Production($"{nameof(NTExpression)}: {nameof(NTAliasExpr)}")] + public NodeType NTExpression(NodeType pass) => pass; + [Production($"{nameof(NTAliasExpr)}: [{nameof(NTAliasExpr1)} | {nameof(NTAliasExpr2)} | {nameof(NTAliasExpr3)} | {nameof(NTDeclarationExpr)}]")] + public NodeType NTAliasExpr(NodeType Node) => Node; + [Production($"{nameof(NTAliasExpr1)}: Identifier As [d] Identifier")] + public NodeType NTAliasExpr1(LyToken Ident, LyToken Ident2) => new(FromToken(Ident), BuildChildren(Ident2), ASTNodeType.AliasExpr); + [Production($"{nameof(NTAliasExpr2)}: Identifier As [d] {nameof(NTType)} Identifier")] + public NodeType NTAliasExpr2(LyToken Ident, NodeType AType, LyToken Ident2) => new(FromToken(Ident), BuildChildren(AType, Ident2), ASTNodeType.ReTypingAlias); + [Production($"{nameof(NTAliasExpr3)}: Identifier As [d] {nameof(NTType)}")] + public NodeType NTAliasExpr3(LyToken Ident, NodeType Type) => new(FromToken(Ident), BuildChildren(Type), ASTNodeType.ReTypeOriginal); + [Production($"{nameof(NTDeclarationExpr)}: [{nameof(NTDeclarationExpr1)} | {nameof(NTAssignmentExpr)}]")] + public NodeType NTDeclarationExpr(NodeType Node) => Node; + [Production($"{nameof(NTDeclarationExpr1)}: {nameof(NTDeclarationModifiersCombined)}? {nameof(NTType)} Identifier {nameof(NTAssignmentPrime)}?")] + public NodeType NTDeclarationExpr1(ValueOption Modifiers, NodeType AType, LyToken Ident, ValueOption AAssignmentPrime) => new(FromToken(Ident), BuildChildren(Modifiers, AType, AAssignmentPrime), ASTNodeType.Declaration); + [Production($"{nameof(NTDeclarationModifiersCombined)}: {nameof(NTDeclarationModifier)}+")] + public NodeType NTDeclarationModifiersCombined(List Modifiers) => new(null, Modifiers, ASTNodeType.DeclarationModifiersCombined); + [Production($"{nameof(NTDeclarationModifier)}: [Ref | Readonly | Frozen | Immut]")] + public NodeType NTDeclarationModifier(LyToken Mod) => new(FromToken(Mod), [], ASTNodeType.DeclarationModifier); + [Production($"{nameof(NTFunctionArgDeclModifier)}: [Ref | Readonly | Frozen | Immut | Copy]")] + public NodeType NTFunctionArgDeclModifier(LyToken Mod) => NTDeclarationModifier(Mod) with { NodeType = ASTNodeType.FunctionArgDeclModifiers }; + [Production($"{nameof(NTFunctionArgDeclModifiersCombined)}: {nameof(NTFunctionArgDeclModifier)}+")] + public NodeType NTFunctionArgDeclModifiersCombined(List Modifiers) => new(null, Modifiers, ASTNodeType.FunctionArgDeclModifiersCombined); + [Production($"{nameof(NTAssignmentPrime)}: Equals {nameof(NTExpression)}")] + public NodeType NTAssignmentPrime(LyToken EQ, NodeType Expr) => new(FromToken(EQ), [Expr], ASTNodeType.AssignmentPrime); + [Production($"{nameof(NTAssignmentExpr)}: {nameof(NTAssignmentExpr1)}")] + public NodeType NTAssignmentExpr(NodeType Node) => Node; + [Production($"{nameof(NTAssignmentExpr1)}: SmallLangParser_expressions")] + public NodeType NTAssignmentExpr1(NodeType P1) => P1; + [Operand] + [Production($"{nameof(NTPrimary)}: {nameof(NTLPrimary)} {nameof(NTPrimaryPrime)}?")] + public NodeType NTPrimary(NodeType APrimary, ValueOption Prime) + { + if (GetFromValOp(Prime) is NodeType NodePrime) + { + return new(NodePrime.Data, [APrimary, .. NodePrime.Children], NodePrime.NodeType); + } + else return APrimary; + } + [Production($"{nameof(NTPrimaryPrime)}: [{nameof(NTIndexPrime)} | {nameof(NTFunctionCallPrime)}]")] + public NodeType NTPrimaryPrime(NodeType Node) => Node; + [Production($"{nameof(NTIndexPrime)}: OpenSquare [d] {nameof(NTExpression)} CloseSquare [d]")] + public NodeType NTIndexPrime(NodeType Expr) => new(null, [Expr], ASTNodeType.Index); + [Production($"{nameof(NTLPrimary)}: [{nameof(NTNewExpr)} | {nameof(NTLPrimary1)} | {nameof(NTLPrimary2)} | {nameof(NTLPrimary3)}]")] + public NodeType NTLPrimary(NodeType Node) => Node; + [Production($"{nameof(NTLPrimary1)}: OpenParen [d] {nameof(NTExpression)} CloseParen [d]")] + public NodeType NTLPrimary1(NodeType Expr) => Expr; + [Production($"{nameof(NTLPrimary2)}: Copy [d] {nameof(NTExpression)}")] + public NodeType NTLPrimary2(NodeType Expr) => new(null, [Expr], ASTNodeType.CopyExpr); + [Production($"{nameof(NTLPrimary3)}: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] + public NodeType NTLPrimary3(LyToken Token) + { + IToken oT = FromToken(Token)!; + return new(oT, [], oT.TT == TokenType.Identifier ? ASTNodeType.Identifier : ASTNodeType.Primary); + } + [Production($"{nameof(NTNewExpr)}: New [d] {nameof(NTType)} OpenParen [d] {nameof(NTArgList)}?")] + public NodeType NTNewExpr(NodeType AType, ValueOption AArgList) => new(null, BuildChildren(AType, AArgList), ASTNodeType.NewExpr); + [Production($"{nameof(NTFunctionCallPrime)}:OpenParen [d] {nameof(NTArgList)}? CloseParen [d]")] + public NodeType NTFunctionCallPrime(ValueOption AArgList) => new(null, BuildChildren(AArgList), ASTNodeType.FunctionCall); + [Production($"{nameof(NTArgList)}: {nameof(NTArgListElement)} {nameof(NTArgListPrime)}*")] + public NodeType NTArgList(NodeType Element, List Elements) => new(null, [Element, .. Elements], ASTNodeType.ArgList); + [Production($"{nameof(NTArgListElement)}: {nameof(NTArgumentLabel)}? {nameof(NTExpression)}")] + public NodeType NTArgListElement(ValueOption Label, NodeType Expr) => new(null, BuildChildren(Label, Expr), ASTNodeType.ArgListElement); + [Production($"{nameof(NTArgListPrime)}: Comma [d] {nameof(NTArgListElement)}")] + public NodeType NTArgListPrime(NodeType Element) => Element; + [Production($"{nameof(NTArgumentLabel)}: Identifier Colon [d]")] + public NodeType NTArgumentLabel(LyToken Ident) => new(FromToken(Ident), [], ASTNodeType.Identifier); + [Production($"{nameof(NTTypeCSV)}: {nameof(NTType)} (Comma [d] {nameof(NTType)})*")] + public NodeType NTTypeCSV(NodeType AType, List> OtherTypes) + { + return new(null, [AType, .. OtherTypes.Select(x => x.Items.First().Value)], ASTNodeType.TypeCSV); + } + [Production($"{nameof(NTType)}: [{nameof(NTBaseType)} | {nameof(NTGenericType)}]")] + public NodeType NTType(NodeType Node) => Node; + [Production($"{nameof(NTGenericType)}: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] {nameof(NTTypeCSV)} CloseAngleSquare [d]")] + public NodeType NTGenericType(LyToken TypeToken, NodeType TypeArgs) => new(FromToken(TypeToken), [TypeArgs], ASTNodeType.GenericType); + [Production($"{nameof(NTBaseType)}: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] + public NodeType NTBaseType(LyToken TypeToken) => new(FromToken(TypeToken), [], ASTNodeType.BaseType); + }``` + + ```cs + using Common.Tokens; + using sly.parser.generator; + using LyToken = sly.lexer.Token; + using NodeType = Common.AST.DynamicASTNode; +} + +namespace SmallLang.Parser; +public partial class SmallLangParser +{ + #region ComparisonExpressions + const int EqualToPrecedence = NotExprPrecedence + 1; + const int NotEqualToPrecedence = EqualToPrecedence; + const int GreaterThanPrecedence = NotEqualToPrecedence; + const int LessThanPrecedence = GreaterThanPrecedence; + const int GreaterThanOrEqualToPrecedence = LessThanPrecedence; + const int LessThanOrEqualToPrecedence = GreaterThanOrEqualToPrecedence; + [Infix((int)TokenType.EqualTo, Associativity.Right, EqualToPrecedence)] + [Infix((int)TokenType.NotEqualTo, Associativity.Right, NotEqualToPrecedence)] + [Infix((int)TokenType.GreaterThan, Associativity.Right, GreaterThanPrecedence)] + [Infix((int)TokenType.LessThan, Associativity.Right, LessThanPrecedence)] + [Infix((int)TokenType.GreaterThanOrEqualTo, Associativity.Right, GreaterThanOrEqualToPrecedence)] + [Infix((int)TokenType.LessThanOrEqualTo, Associativity.Right, LessThanOrEqualToPrecedence)] + public NodeType ComparisonExpressions(NodeType Left, LyToken Operator, NodeType Right) + { + var OpToken = FromToken(Operator); + List rc = [Left]; + if (Right.NodeType == ASTNodeType.ComparisionExpression) + { + rc.AddRange([ + new(OpToken, [Right.Children[0]], ASTNodeType.OperatorExpressionPair), + ..Right.Children.Skip(1) + ]); + } + else + { + rc.Add(new(OpToken, [Right], ASTNodeType.OperatorExpressionPair)); + } + return new(null, rc, ASTNodeType.ComparisionExpression); + } + #endregion + #region OtherExpressions + const int AssignmentExprPrecedence = 1; + const int ImpliesExprPrecedence = AssignmentExprPrecedence + 1; + const int OrExprPrecedence = ImpliesExprPrecedence + 1; + + const int XorExprPrecedence = OrExprPrecedence + 1; + + const int AndExprPrecedence = XorExprPrecedence + 1; + + const int NotExprPrecedence = AndExprPrecedence + 1; + const int AdditionPrecedence = EqualToPrecedence + 1; + const int SubtractionPrecedence = AdditionPrecedence; + + const int MultiplicationPrecedence = SubtractionPrecedence + 1; + const int DivisionPrecedence = MultiplicationPrecedence; + + const int PowerPrecedence = DivisionPrecedence + 1; + + const int NegationPrecedence = PowerPrecedence + 1; + + const int FactorialPrecedence = NegationPrecedence + 1; + const int RShiftPrecedence = FactorialPrecedence + 1; + const int LShiftPrecedence = RShiftPrecedence + 1; + const int BitwiseOrExprPrecedence = LShiftPrecedence + 1; + + const int BitwiseXorExprPrecedence = BitwiseOrExprPrecedence + 1; + + const int BitwiseAndExprPrecedence = BitwiseXorExprPrecedence + 1; + + const int BitwiseNotExprPrecedence = BitwiseAndExprPrecedence + 1; + #endregion + + [Postfix((int)TokenType.Factorial, Associativity.Left, FactorialPrecedence)] + public NodeType Factorial(NodeType Left, LyToken Op) + { + if (Left.NodeType == ASTNodeType.FactorialExpression) + { + Left.Children.AddRange(BuildChildren(Op)); + return Left; + } + return new(null, BuildChildren(Op, Left), ASTNodeType.FactorialExpression); + } + [Prefix((int)TokenType.LogicalNot, Associativity.Right, NotExprPrecedence)] + [Prefix((int)TokenType.BitwiseNegation, Associativity.Right, BitwiseNotExprPrecedence)] + [Prefix((int)TokenType.Subtraction, Associativity.Right, NegationPrecedence)] + public NodeType Prefix(LyToken Op, NodeType Right) => new(FromToken(Op), [Right], ASTNodeType.UnaryExpression); + [Infix((int)TokenType.Equals, Associativity.Right, AssignmentExprPrecedence)] + [Infix((int)TokenType.LogicalOr, Associativity.Left, OrExprPrecedence)] + [Infix((int)TokenType.LogicalXor, Associativity.Left, XorExprPrecedence)] + [Infix((int)TokenType.LogicalAnd, Associativity.Left, AndExprPrecedence)] + [Infix((int)TokenType.Addition, Associativity.Left, AdditionPrecedence)] + [Infix((int)TokenType.Subtraction, Associativity.Left, SubtractionPrecedence)] + [Infix((int)TokenType.Division, Associativity.Left, DivisionPrecedence)] + [Infix((int)TokenType.Multiplication, Associativity.Left, MultiplicationPrecedence)] + [Infix((int)TokenType.Exponentiation, Associativity.Right, PowerPrecedence)] + [Infix((int)TokenType.Subtraction, Associativity.Left, NegationPrecedence)] + [Infix((int)TokenType.BitwiseOr, Associativity.Left, BitwiseOrExprPrecedence)] + [Infix((int)TokenType.BitwiseXor, Associativity.Left, BitwiseXorExprPrecedence)] + [Infix((int)TokenType.BitwiseAnd, Associativity.Left, BitwiseAndExprPrecedence)] + [Infix((int)TokenType.BitwiseLeftShift, Associativity.Left, LShiftPrecedence)] + [Infix((int)TokenType.BitwiseRightShift, Associativity.Left, RShiftPrecedence)] + public NodeType BinOp(NodeType left, LyToken Op, NodeType right) + { + return new NodeType(FromToken(Op), [left, right], ASTNodeType.BinaryExpression); + } +} \ No newline at end of file diff --git a/tests/ParserTests/Issue540/Issue540Token.cs b/tests/ParserTests/Issue540/Issue540Token.cs new file mode 100644 index 00000000..846302c1 --- /dev/null +++ b/tests/ParserTests/Issue540/Issue540Token.cs @@ -0,0 +1,6 @@ +namespace ParserTests.Issue540; + +public enum Issue540Token +{ + +} \ No newline at end of file From 8b807a30d571af95741c3ae5408425cb0e0f5dde Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 10:03:20 +0100 Subject: [PATCH 2/7] . --- tests/ParserTests/Issue540/Issue540Parser.cs | 532 +++++++++---------- tests/ParserTests/Issue540/Issue540Token.cs | 154 +++++- tests/ParserTests/IssuesTests.cs | 12 + 3 files changed, 412 insertions(+), 286 deletions(-) diff --git a/tests/ParserTests/Issue540/Issue540Parser.cs b/tests/ParserTests/Issue540/Issue540Parser.cs index 9b4bf98d..b4ffd169 100644 --- a/tests/ParserTests/Issue540/Issue540Parser.cs +++ b/tests/ParserTests/Issue540/Issue540Parser.cs @@ -1,322 +1,288 @@ -using System.Data; -using Common.AST; -using Common.Tokens; -using sly.lexer; +using System; using sly.parser.generator; using sly.parser.parser; -using LyToken = sly.lexer.Token; -using NodeType = Common.AST.DynamicASTNode; +using System.Collections.Generic; +using ParserTests.Issue540; +using sly.lexer; + + namespace ParserTests.Issue540 { - public partial class SmallLangParser + public partial class Issue540Parser { - void AppendIfNotEmpty(List nodeList, ValueOption Considered) + void AppendIfNotEmpty(List nodeList, ValueOption Considered) { - if (GetFromValOp(Considered) is NodeType node) + if (GetFromValOp(Considered) is object node) { nodeList.Add(node); } } - List BuildChildren(params object[] Vals) + + List BuildChildren(params object[] Vals) { - List rc = new(); - foreach (var val in Vals) - { - if (val is NodeType node) - { - rc.Add(node); - } - else if (val is ValueOption valOp) - { - AppendIfNotEmpty(rc, valOp); - } - else if (val is LyToken TVal) - { - var Token = FromToken(TVal); - if (Token == null) continue; - if (Token.TT == TokenType.Identifier) - { - rc.Add(new NodeType(Token, [], ASTNodeType.Identifier)); - } - else - { - rc.Add(new NodeType(Token, [], ASTNodeType.Terminal)); - } - } - else - { - throw new Exception($"Unexpected type: {val.GetType().Name}"); - } - } - return rc; + return new(); + } - NodeType? GetFromValOp(ValueOption value) + + object? GetFromValOp(ValueOption value) { if (value.IsSome) { return value.Match(x => x, () => throw new Exception("IsSome was true when no value was found")); } + return null; } - IToken? FromToken(LyToken t) => t.IsEmpty ? null : IToken.NewToken(t.TokenID, t.Value, t.Position.Index, null); - [Production($"{nameof(NTSection)}: {nameof(NTStatement)}*")] - public NodeType NTSection(List Statements) - { - return new NodeType(null, Statements, ASTNodeType.Section); - } - [Production($"{nameof(NTStatement)}: [{nameof(NTSCExpr)} | {nameof(NTLoop)} | {nameof(NTCond)} | {nameof(NTFunction)} | {nameof(NTBlock)} | {nameof(NTReturnStatement)} | {nameof(NTLoopControlStatement)}]")] - public NodeType NTStatement(NodeType SubStatement) => SubStatement; - [Production($"{nameof(NTSCExpr)}: {nameof(NTExpression)}")] - - public NodeType NTSCExpr(NodeType Expression) => Expression; - [Production($"{nameof(NTReturnStatement)}: Return [d] {nameof(NTSCExpr)}")] - public NodeType NTReturnStatement(NodeType SCExpr) => SCExpr with { NodeType = ASTNodeType.Return }; - [Production($"{nameof(NTLoopControlStatement)}: [Break | Continue] {nameof(NTNestedValueInLoopControl)}?")] - public NodeType NTLoopControlStatement(LyToken Operator, ValueOption NestedVal) - { - List rc = []; - AppendIfNotEmpty(rc, NestedVal); - return new(FromToken(Operator), rc, ASTNodeType.LoopCTRL); - } - [Production($"{nameof(NTNestedValueInLoopControl)}: Identifier")] - public NodeType NTNestedValueInLoopControl(LyToken val) => new NodeType(FromToken(val), [], ASTNodeType.ValInLCTRL); - [Production($"{nameof(NTLoop)}: {nameof(NTForLoopHeader)} {nameof(NTLoopLabel)}? {nameof(NTStatement)} {nameof(NTElse)}?")] - [Production($"{nameof(NTLoop)}: {nameof(NTWhileLoopHeader)} {nameof(NTLoopLabel)}? {nameof(NTStatement)} {nameof(NTElse)}?")] - public NodeType NTLoop(NodeType LoopHeader, ValueOption Label, NodeType StatementExpr, ValueOption ElseExpr) - { - var rt = LoopHeader.NodeType; - List rc = LoopHeader.Children; - AppendIfNotEmpty(rc, Label); - rc.Add(StatementExpr); - AppendIfNotEmpty(rc, ElseExpr); - return new NodeType(null, rc, rt); - } - [Production($"{nameof(NTLoopLabel)}: As [d] Identifier")] - public NodeType NTLoopLabel(LyToken ident) => new NodeType(FromToken(ident), [], ASTNodeType.LoopLabel); - [Production($"{nameof(NTForLoopHeader)}: For [d] OpenParen [d] {nameof(NTExpression)} Semicolon [d] {nameof(NTExpression)} Semicolon [d] {nameof(NTExpression)} CloseParen [d]")] - public NodeType NTForLoopHeader(NodeType Init, NodeType Condition, NodeType Step) => new NodeType(null, [Init, Condition, Step], ASTNodeType.For); - [Production($"{nameof(NTWhileLoopHeader)}: While [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d]")] - public NodeType NTWhileLoopHeader(NodeType Condition) => new NodeType(null, [Condition], ASTNodeType.While); - [Production($"{nameof(NTCond)}: [{nameof(NTSwitch)} | {nameof(NTIf)}]")] - public NodeType NTCond(NodeType C) => C; - [Production($"{nameof(NTIf)}: If [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d] {nameof(NTStatement)} {nameof(NTElse)}?")] - public NodeType NTIf(NodeType Cond, NodeType StatementExpr, ValueOption ElseExpr) + object FromToken(Issue540Token t) => null; + + [Production($"NTSection: NTStatement*")] + public object NTSection(List Statements) { - var ThisCombined = new NodeType(null, [Cond, StatementExpr], ASTNodeType.ExprStatementCombined); - List rc = [ThisCombined]; - if (GetFromValOp(ElseExpr) is NodeType node) - { - if (node.NodeType == ASTNodeType.If) - { - rc.AddRange(node.Children); //should automatically add the remaining elifs and else as well - } - else - { - rc.Add(node); //node is statement or child thereof (the latter is probably what it is but we don't care) - } - } - return new(null, rc, ASTNodeType.If); + return null; } - [Production($"{nameof(NTElse)}: Else [d] {nameof(NTStatement)}")] - public NodeType NTElse(NodeType AStatement) => AStatement; //passthrough - [Production($"{nameof(NTSwitch)}: Switch [d] OpenParen [d] {nameof(NTExpression)} CloseParen [d] OpenCurly [d] {nameof(NTSwitchBody)}* CloseCurly [d]")] - public NodeType NTSwitch(NodeType AExpression, List ASwitchBody) => new NodeType(null, [AExpression, .. ASwitchBody], ASTNodeType.Switch); - - [Production($"{nameof(NTSwitchBody)}: {nameof(NTExpression)} Colon [d] {nameof(NTStatement)}")] - public NodeType NTSwitchBody(NodeType AExpr, NodeType AStatement) => new(null, [AExpr, AStatement], ASTNodeType.ExprStatementCombined); - [Production($"{nameof(NTFunction)}: {nameof(NTType)} Identifier OpenParen [d] {nameof(NTTypeAndIdentifierCSV)}? CloseParen [d] {nameof(NTStatement)}")] - public NodeType NTFunction(NodeType AType, LyToken Ident, ValueOption TICSV, NodeType Statement) => new(FromToken(Ident), BuildChildren(AType, TICSV, Statement), ASTNodeType.Function); - [Production($"{nameof(NTTypeAndIdentifierCSV)}: {nameof(NTTypeAndIdentifierCSVElement)} (Comma [d] {nameof(NTTypeAndIdentifierCSV)})*")] - public NodeType NTTypeAndIdentifierCSV(NodeType Element, List> Prime) => new(null, [Element, .. Prime.Select(x => x.Items.First().Value)], ASTNodeType.TypeAndIdentifierCSV); - [Production($"{nameof(NTTypeAndIdentifierCSVElement)}: {nameof(NTFunctionArgDeclModifiersCombined)}? {nameof(NTType)} Identifier")] - public NodeType NTTypeAndIdentifierCSVElement(ValueOption Modifiers, NodeType AType, LyToken Ident) => new(FromToken(Ident), BuildChildren(Modifiers, AType), ASTNodeType.TypeAndIdentifierCSVElement); - [Production($"{nameof(NTBlock)}: OpenCurly [d] {nameof(NTSection)} CloseCurly [d]")] - public NodeType NTBlock(NodeType ASection) => ASection; - [Production($"{nameof(NTExpression)}: {nameof(NTAliasExpr)}")] - public NodeType NTExpression(NodeType pass) => pass; - [Production($"{nameof(NTAliasExpr)}: [{nameof(NTAliasExpr1)} | {nameof(NTAliasExpr2)} | {nameof(NTAliasExpr3)} | {nameof(NTDeclarationExpr)}]")] - public NodeType NTAliasExpr(NodeType Node) => Node; - [Production($"{nameof(NTAliasExpr1)}: Identifier As [d] Identifier")] - public NodeType NTAliasExpr1(LyToken Ident, LyToken Ident2) => new(FromToken(Ident), BuildChildren(Ident2), ASTNodeType.AliasExpr); - [Production($"{nameof(NTAliasExpr2)}: Identifier As [d] {nameof(NTType)} Identifier")] - public NodeType NTAliasExpr2(LyToken Ident, NodeType AType, LyToken Ident2) => new(FromToken(Ident), BuildChildren(AType, Ident2), ASTNodeType.ReTypingAlias); - [Production($"{nameof(NTAliasExpr3)}: Identifier As [d] {nameof(NTType)}")] - public NodeType NTAliasExpr3(LyToken Ident, NodeType Type) => new(FromToken(Ident), BuildChildren(Type), ASTNodeType.ReTypeOriginal); - [Production($"{nameof(NTDeclarationExpr)}: [{nameof(NTDeclarationExpr1)} | {nameof(NTAssignmentExpr)}]")] - public NodeType NTDeclarationExpr(NodeType Node) => Node; - [Production($"{nameof(NTDeclarationExpr1)}: {nameof(NTDeclarationModifiersCombined)}? {nameof(NTType)} Identifier {nameof(NTAssignmentPrime)}?")] - public NodeType NTDeclarationExpr1(ValueOption Modifiers, NodeType AType, LyToken Ident, ValueOption AAssignmentPrime) => new(FromToken(Ident), BuildChildren(Modifiers, AType, AAssignmentPrime), ASTNodeType.Declaration); - [Production($"{nameof(NTDeclarationModifiersCombined)}: {nameof(NTDeclarationModifier)}+")] - public NodeType NTDeclarationModifiersCombined(List Modifiers) => new(null, Modifiers, ASTNodeType.DeclarationModifiersCombined); - [Production($"{nameof(NTDeclarationModifier)}: [Ref | Readonly | Frozen | Immut]")] - public NodeType NTDeclarationModifier(LyToken Mod) => new(FromToken(Mod), [], ASTNodeType.DeclarationModifier); - [Production($"{nameof(NTFunctionArgDeclModifier)}: [Ref | Readonly | Frozen | Immut | Copy]")] - public NodeType NTFunctionArgDeclModifier(LyToken Mod) => NTDeclarationModifier(Mod) with { NodeType = ASTNodeType.FunctionArgDeclModifiers }; - [Production($"{nameof(NTFunctionArgDeclModifiersCombined)}: {nameof(NTFunctionArgDeclModifier)}+")] - public NodeType NTFunctionArgDeclModifiersCombined(List Modifiers) => new(null, Modifiers, ASTNodeType.FunctionArgDeclModifiersCombined); - [Production($"{nameof(NTAssignmentPrime)}: Equals {nameof(NTExpression)}")] - public NodeType NTAssignmentPrime(LyToken EQ, NodeType Expr) => new(FromToken(EQ), [Expr], ASTNodeType.AssignmentPrime); - [Production($"{nameof(NTAssignmentExpr)}: {nameof(NTAssignmentExpr1)}")] - public NodeType NTAssignmentExpr(NodeType Node) => Node; - [Production($"{nameof(NTAssignmentExpr1)}: SmallLangParser_expressions")] - public NodeType NTAssignmentExpr1(NodeType P1) => P1; + + [Production( + $"NTStatement: [NTSCExpr | NTLoop | NTCond | NTFunction | NTBlock | NTReturnStatement | NTLoopControlStatement]")] + public object NTStatement(object SubStatement) => SubStatement; + + [Production($"NTSCExpr: NTExpression")] + + public object NTSCExpr(object Expression) => Expression; + + [Production($"NTReturnStatement: Return [d] NTSCExpr")] + public object NTReturnStatement(object SCExpr) => null; + + [Production($"NTLoopControlStatement: [Break | Continue] NTNestedValueInLoopControl?")] + public object NTLoopControlStatement(Issue540Token Operator, ValueOption NestedVal) => null; + + [Production($"NTNestedValueInLoopControl: Identifier")] + public object NTNestedValueInLoopControl(Token val) => null; + + [Production($"NTLoop: NTForLoopHeader NTLoopLabel? NTStatement NTElse?")] + [Production($"NTLoop: NTWhileLoopHeader NTLoopLabel? NTStatement NTElse?")] + public object NTLoop(object LoopHeader, ValueOption Label, object StatementExpr, + ValueOption ElseExpr) => null; + + [Production($"NTLoopLabel: As [d] Identifier")] + public object NTLoopLabel(Token ident) => null; + + [Production( + $"NTForLoopHeader: For [d] OpenParen [d] NTExpression Semicolon [d] NTExpression Semicolon [d] NTExpression CloseParen [d]")] + public object NTForLoopHeader(object Init, object Condition, object Step) => null; + + [Production($"NTWhileLoopHeader: While [d] OpenParen [d] NTExpression CloseParen [d]")] + public object NTWhileLoopHeader(object Condition) => null; + + [Production($"NTCond: [NTSwitch | NTIf]")] + public object NTCond(object C) => null; + + [Production($"NTIf: If [d] OpenParen [d] NTExpression CloseParen [d] NTStatement NTElse?")] + public object NTIf(object Cond, object StatementExpr, ValueOption ElseExpr) => null; + + [Production($"NTElse: Else [d] NTStatement")] + public object NTElse(object AStatement) => null; + + [Production( + $"NTSwitch: Switch [d] OpenParen [d] NTExpression CloseParen [d] OpenCurly [d] NTSwitchBody* CloseCurly [d]")] + public object NTSwitch(object AExpression, List ASwitchBody) => null; + + [Production($"NTSwitchBody: NTExpression Colon [d] NTStatement")] + public object NTSwitchBody(object AExpr, object AStatement) => null; + + [Production($"NTFunction: NTType Identifier OpenParen [d] NTTypeAndIdentifierCSV? CloseParen [d] NTStatement")] + public object NTFunction(object AType, Token Ident, ValueOption TICSV, object Statement) => + null; + + [Production($"NTTypeAndIdentifierCSV: NTTypeAndIdentifierCSVElement (Comma [d] NTTypeAndIdentifierCSV)*")] + public object NTTypeAndIdentifierCSV(object Element, List> Prime) => null; + + [Production($"NTTypeAndIdentifierCSVElement: NTFunctionArgDeclModifiersCombined? NTType Identifier")] + public object NTTypeAndIdentifierCSVElement(ValueOption Modifiers, object AType, Token Ident) => + null; + + [Production($"NTBlock: OpenCurly [d] NTSection CloseCurly [d]")] + public object NTBlock(object ASection) => ASection; + + [Production($"NTExpression: NTAliasExpr")] + public object NTExpression(object pass) => pass; + + [Production($"NTAliasExpr: [NTAliasExpr1 | NTAliasExpr2 | NTAliasExpr3 | NTDeclarationExpr]")] + public object NTAliasExpr(object Node) => Node; + + [Production($"NTAliasExpr1: Identifier As [d] Identifier")] + public object NTAliasExpr1(Token Ident, Token Ident2) => null; + + [Production($"NTAliasExpr2: Identifier As [d] NTType Identifier")] + public object NTAliasExpr2(Token Ident, object AType, Token Ident2) => null; + + [Production($"NTAliasExpr3: Identifier As [d] NTType")] + public object NTAliasExpr3(Token Ident, object Type) => null; + + [Production($"NTDeclarationExpr: [NTDeclarationExpr1 | NTAssignmentExpr]")] + public object NTDeclarationExpr(object Node) => Node; + + [Production($"NTDeclarationExpr1: NTDeclarationModifiersCombined? NTType Identifier NTAssignmentPrime?")] + public object NTDeclarationExpr1(ValueOption Modifiers, object AType, Token Ident, + ValueOption AAssignmentPrime) => null; + + [Production($"NTDeclarationModifiersCombined: NTDeclarationModifier+")] + public object NTDeclarationModifiersCombined(List Modifiers) => null; + + [Production($"NTDeclarationModifier: [Ref | Readonly | Frozen | Immut]")] + public object NTDeclarationModifier(Token Mod) => null; + + [Production($"NTFunctionArgDeclModifier: [Ref | Readonly | Frozen | Immut | Copy]")] + public object NTFunctionArgDeclModifier(Token Mod) => null; + + [Production($"NTFunctionArgDeclModifiersCombined: NTFunctionArgDeclModifier+")] + public object NTFunctionArgDeclModifiersCombined(List Modifiers) => null; + + [Production($"NTAssignmentPrime: Equals NTExpression")] + public object NTAssignmentPrime(Token EQ, object Expr) => null; + + [Production($"NTAssignmentExpr: NTAssignmentExpr1")] + public object NTAssignmentExpr(object Node) => null; + + [Production($"NTAssignmentExpr1: Issue540Parser_expressions")] + public object NTAssignmentExpr1(object P1) => null; + [Operand] - [Production($"{nameof(NTPrimary)}: {nameof(NTLPrimary)} {nameof(NTPrimaryPrime)}?")] - public NodeType NTPrimary(NodeType APrimary, ValueOption Prime) - { - if (GetFromValOp(Prime) is NodeType NodePrime) - { - return new(NodePrime.Data, [APrimary, .. NodePrime.Children], NodePrime.NodeType); - } - else return APrimary; - } - [Production($"{nameof(NTPrimaryPrime)}: [{nameof(NTIndexPrime)} | {nameof(NTFunctionCallPrime)}]")] - public NodeType NTPrimaryPrime(NodeType Node) => Node; - [Production($"{nameof(NTIndexPrime)}: OpenSquare [d] {nameof(NTExpression)} CloseSquare [d]")] - public NodeType NTIndexPrime(NodeType Expr) => new(null, [Expr], ASTNodeType.Index); - [Production($"{nameof(NTLPrimary)}: [{nameof(NTNewExpr)} | {nameof(NTLPrimary1)} | {nameof(NTLPrimary2)} | {nameof(NTLPrimary3)}]")] - public NodeType NTLPrimary(NodeType Node) => Node; - [Production($"{nameof(NTLPrimary1)}: OpenParen [d] {nameof(NTExpression)} CloseParen [d]")] - public NodeType NTLPrimary1(NodeType Expr) => Expr; - [Production($"{nameof(NTLPrimary2)}: Copy [d] {nameof(NTExpression)}")] - public NodeType NTLPrimary2(NodeType Expr) => new(null, [Expr], ASTNodeType.CopyExpr); - [Production($"{nameof(NTLPrimary3)}: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] - public NodeType NTLPrimary3(LyToken Token) - { - IToken oT = FromToken(Token)!; - return new(oT, [], oT.TT == TokenType.Identifier ? ASTNodeType.Identifier : ASTNodeType.Primary); - } - [Production($"{nameof(NTNewExpr)}: New [d] {nameof(NTType)} OpenParen [d] {nameof(NTArgList)}?")] - public NodeType NTNewExpr(NodeType AType, ValueOption AArgList) => new(null, BuildChildren(AType, AArgList), ASTNodeType.NewExpr); - [Production($"{nameof(NTFunctionCallPrime)}:OpenParen [d] {nameof(NTArgList)}? CloseParen [d]")] - public NodeType NTFunctionCallPrime(ValueOption AArgList) => new(null, BuildChildren(AArgList), ASTNodeType.FunctionCall); - [Production($"{nameof(NTArgList)}: {nameof(NTArgListElement)} {nameof(NTArgListPrime)}*")] - public NodeType NTArgList(NodeType Element, List Elements) => new(null, [Element, .. Elements], ASTNodeType.ArgList); - [Production($"{nameof(NTArgListElement)}: {nameof(NTArgumentLabel)}? {nameof(NTExpression)}")] - public NodeType NTArgListElement(ValueOption Label, NodeType Expr) => new(null, BuildChildren(Label, Expr), ASTNodeType.ArgListElement); - [Production($"{nameof(NTArgListPrime)}: Comma [d] {nameof(NTArgListElement)}")] - public NodeType NTArgListPrime(NodeType Element) => Element; - [Production($"{nameof(NTArgumentLabel)}: Identifier Colon [d]")] - public NodeType NTArgumentLabel(LyToken Ident) => new(FromToken(Ident), [], ASTNodeType.Identifier); - [Production($"{nameof(NTTypeCSV)}: {nameof(NTType)} (Comma [d] {nameof(NTType)})*")] - public NodeType NTTypeCSV(NodeType AType, List> OtherTypes) - { - return new(null, [AType, .. OtherTypes.Select(x => x.Items.First().Value)], ASTNodeType.TypeCSV); - } - [Production($"{nameof(NTType)}: [{nameof(NTBaseType)} | {nameof(NTGenericType)}]")] - public NodeType NTType(NodeType Node) => Node; - [Production($"{nameof(NTGenericType)}: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] {nameof(NTTypeCSV)} CloseAngleSquare [d]")] - public NodeType NTGenericType(LyToken TypeToken, NodeType TypeArgs) => new(FromToken(TypeToken), [TypeArgs], ASTNodeType.GenericType); - [Production($"{nameof(NTBaseType)}: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] - public NodeType NTBaseType(LyToken TypeToken) => new(FromToken(TypeToken), [], ASTNodeType.BaseType); - }``` - - ```cs - using Common.Tokens; - using sly.parser.generator; - using LyToken = sly.lexer.Token; - using NodeType = Common.AST.DynamicASTNode; -} - -namespace SmallLang.Parser; -public partial class SmallLangParser -{ - #region ComparisonExpressions - const int EqualToPrecedence = NotExprPrecedence + 1; - const int NotEqualToPrecedence = EqualToPrecedence; - const int GreaterThanPrecedence = NotEqualToPrecedence; - const int LessThanPrecedence = GreaterThanPrecedence; - const int GreaterThanOrEqualToPrecedence = LessThanPrecedence; - const int LessThanOrEqualToPrecedence = GreaterThanOrEqualToPrecedence; - [Infix((int)TokenType.EqualTo, Associativity.Right, EqualToPrecedence)] - [Infix((int)TokenType.NotEqualTo, Associativity.Right, NotEqualToPrecedence)] - [Infix((int)TokenType.GreaterThan, Associativity.Right, GreaterThanPrecedence)] - [Infix((int)TokenType.LessThan, Associativity.Right, LessThanPrecedence)] - [Infix((int)TokenType.GreaterThanOrEqualTo, Associativity.Right, GreaterThanOrEqualToPrecedence)] - [Infix((int)TokenType.LessThanOrEqualTo, Associativity.Right, LessThanOrEqualToPrecedence)] - public NodeType ComparisonExpressions(NodeType Left, LyToken Operator, NodeType Right) - { - var OpToken = FromToken(Operator); - List rc = [Left]; - if (Right.NodeType == ASTNodeType.ComparisionExpression) - { - rc.AddRange([ - new(OpToken, [Right.Children[0]], ASTNodeType.OperatorExpressionPair), - ..Right.Children.Skip(1) - ]); - } - else - { - rc.Add(new(OpToken, [Right], ASTNodeType.OperatorExpressionPair)); - } - return new(null, rc, ASTNodeType.ComparisionExpression); - } - #endregion - #region OtherExpressions - const int AssignmentExprPrecedence = 1; - const int ImpliesExprPrecedence = AssignmentExprPrecedence + 1; - const int OrExprPrecedence = ImpliesExprPrecedence + 1; + [Production($"NTPrimary: NTLPrimary NTPrimaryPrime?")] + public object NTPrimary(object APrimary, ValueOption Prime) => null; - const int XorExprPrecedence = OrExprPrecedence + 1; + [Production($"NTPrimaryPrime: [NTIndexPrime | NTFunctionCallPrime]")] + public object NTPrimaryPrime(object Node) => null; - const int AndExprPrecedence = XorExprPrecedence + 1; + [Production($"NTIndexPrime: OpenSquare [d] NTExpression CloseSquare [d]")] + public object NTIndexPrime(object Expr) => null; - const int NotExprPrecedence = AndExprPrecedence + 1; - const int AdditionPrecedence = EqualToPrecedence + 1; - const int SubtractionPrecedence = AdditionPrecedence; + [Production($"NTLPrimary: [NTNewExpr | NTLPrimary1 | NTLPrimary2 | NTLPrimary3]")] + public object NTLPrimary(object Node) => null; - const int MultiplicationPrecedence = SubtractionPrecedence + 1; - const int DivisionPrecedence = MultiplicationPrecedence; + [Production($"NTLPrimary1: OpenParen [d] NTExpression CloseParen [d]")] + public object NTLPrimary1(object Expr) => null; - const int PowerPrecedence = DivisionPrecedence + 1; + [Production($"NTLPrimary2: Copy [d] NTExpression")] + public object NTLPrimary2(object Expr) => null; - const int NegationPrecedence = PowerPrecedence + 1; + [Production($"NTLPrimary3: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] + public object NTLPrimary3(Token Token) => null; - const int FactorialPrecedence = NegationPrecedence + 1; - const int RShiftPrecedence = FactorialPrecedence + 1; - const int LShiftPrecedence = RShiftPrecedence + 1; - const int BitwiseOrExprPrecedence = LShiftPrecedence + 1; + [Production($"NTNewExpr: New [d] NTType OpenParen [d] NTArgList?")] + public object NTNewExpr(object AType, ValueOption AArgList) => null; - const int BitwiseXorExprPrecedence = BitwiseOrExprPrecedence + 1; + [Production($"NTFunctionCallPrime:OpenParen [d] NTArgList? CloseParen [d]")] + public object NTFunctionCallPrime(ValueOption AArgList) => null; - const int BitwiseAndExprPrecedence = BitwiseXorExprPrecedence + 1; + [Production($"NTArgList: NTArgListElement NTArgListPrime*")] + public object NTArgList(object Element, List Elements) => null; - const int BitwiseNotExprPrecedence = BitwiseAndExprPrecedence + 1; - #endregion + [Production($"NTArgListElement: NTArgumentLabel? NTExpression")] + public object NTArgListElement(ValueOption Label, object Expr) => null; - [Postfix((int)TokenType.Factorial, Associativity.Left, FactorialPrecedence)] - public NodeType Factorial(NodeType Left, LyToken Op) - { - if (Left.NodeType == ASTNodeType.FactorialExpression) - { - Left.Children.AddRange(BuildChildren(Op)); - return Left; - } - return new(null, BuildChildren(Op, Left), ASTNodeType.FactorialExpression); + [Production($"NTArgListPrime: Comma [d] NTArgListElement")] + public object NTArgListPrime(object Element) => null; + + [Production($"NTArgumentLabel: Identifier Colon [d]")] + public object NTArgumentLabel(Token Ident) => null; + + [Production($"NTTypeCSV: NTType (Comma [d] NTType)*")] + public object NTTypeCSV(object AType, List> OtherTypes) => null; + + [Production($"NTType: [NTBaseType | NTGenericType]")] + public object NTType(object Node) => null; + + [Production( + $"NTGenericType: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] NTTypeCSV CloseAngleSquare [d]")] + public object NTGenericType(Token TypeToken, object TypeArgs) => null; + + [Production( + $"NTBaseType: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] + public object NTBaseType(Token TypeToken) => null; } - [Prefix((int)TokenType.LogicalNot, Associativity.Right, NotExprPrecedence)] - [Prefix((int)TokenType.BitwiseNegation, Associativity.Right, BitwiseNotExprPrecedence)] - [Prefix((int)TokenType.Subtraction, Associativity.Right, NegationPrecedence)] - public NodeType Prefix(LyToken Op, NodeType Right) => new(FromToken(Op), [Right], ASTNodeType.UnaryExpression); - [Infix((int)TokenType.Equals, Associativity.Right, AssignmentExprPrecedence)] - [Infix((int)TokenType.LogicalOr, Associativity.Left, OrExprPrecedence)] - [Infix((int)TokenType.LogicalXor, Associativity.Left, XorExprPrecedence)] - [Infix((int)TokenType.LogicalAnd, Associativity.Left, AndExprPrecedence)] - [Infix((int)TokenType.Addition, Associativity.Left, AdditionPrecedence)] - [Infix((int)TokenType.Subtraction, Associativity.Left, SubtractionPrecedence)] - [Infix((int)TokenType.Division, Associativity.Left, DivisionPrecedence)] - [Infix((int)TokenType.Multiplication, Associativity.Left, MultiplicationPrecedence)] - [Infix((int)TokenType.Exponentiation, Associativity.Right, PowerPrecedence)] - [Infix((int)TokenType.Subtraction, Associativity.Left, NegationPrecedence)] - [Infix((int)TokenType.BitwiseOr, Associativity.Left, BitwiseOrExprPrecedence)] - [Infix((int)TokenType.BitwiseXor, Associativity.Left, BitwiseXorExprPrecedence)] - [Infix((int)TokenType.BitwiseAnd, Associativity.Left, BitwiseAndExprPrecedence)] - [Infix((int)TokenType.BitwiseLeftShift, Associativity.Left, LShiftPrecedence)] - [Infix((int)TokenType.BitwiseRightShift, Associativity.Left, RShiftPrecedence)] - public NodeType BinOp(NodeType left, LyToken Op, NodeType right) + + + public partial class Issue540Parser { - return new NodeType(FromToken(Op), [left, right], ASTNodeType.BinaryExpression); + #region ComparisonExpressions + + const int EqualToPrecedence = NotExprPrecedence + 1; + const int NotEqualToPrecedence = EqualToPrecedence; + const int GreaterThanPrecedence = NotEqualToPrecedence; + const int LessThanPrecedence = GreaterThanPrecedence; + const int GreaterThanOrEqualToPrecedence = LessThanPrecedence; + const int LessThanOrEqualToPrecedence = GreaterThanOrEqualToPrecedence; + + [Infix((int)Issue540Token.EqualTo, Associativity.Right, EqualToPrecedence)] + [Infix((int)Issue540Token.NotEqualTo, Associativity.Right, NotEqualToPrecedence)] + [Infix((int)Issue540Token.GreaterThan, Associativity.Right, GreaterThanPrecedence)] + [Infix((int)Issue540Token.LessThan, Associativity.Right, LessThanPrecedence)] + [Infix((int)Issue540Token.GreaterThanOrEqualTo, Associativity.Right, GreaterThanOrEqualToPrecedence)] + [Infix((int)Issue540Token.LessThanOrEqualTo, Associativity.Right, LessThanOrEqualToPrecedence)] + public object ComparisonExpressions(object Left, Token Operator, object Right) => null; + + #endregion + + #region OtherExpressions + + const int AssignmentExprPrecedence = 1; + const int ImpliesExprPrecedence = AssignmentExprPrecedence + 1; + const int OrExprPrecedence = ImpliesExprPrecedence + 1; + + const int XorExprPrecedence = OrExprPrecedence + 1; + + const int AndExprPrecedence = XorExprPrecedence + 1; + + const int NotExprPrecedence = AndExprPrecedence + 1; + const int AdditionPrecedence = EqualToPrecedence + 1; + const int SubtractionPrecedence = AdditionPrecedence; + + const int MultiplicationPrecedence = SubtractionPrecedence + 1; + const int DivisionPrecedence = MultiplicationPrecedence; + + const int PowerPrecedence = DivisionPrecedence + 1; + + const int NegationPrecedence = PowerPrecedence + 1; + + const int FactorialPrecedence = NegationPrecedence + 1; + const int RShiftPrecedence = FactorialPrecedence + 1; + const int LShiftPrecedence = RShiftPrecedence + 1; + const int BitwiseOrExprPrecedence = LShiftPrecedence + 1; + + const int BitwiseXorExprPrecedence = BitwiseOrExprPrecedence + 1; + + const int BitwiseAndExprPrecedence = BitwiseXorExprPrecedence + 1; + + const int BitwiseNotExprPrecedence = BitwiseAndExprPrecedence + 1; + + #endregion + + [Postfix((int)Issue540Token.Factorial, Associativity.Left, FactorialPrecedence)] + public object Factorial(object Left, Token Op) => null; + + [Prefix((int)Issue540Token.LogicalNot, Associativity.Right, NotExprPrecedence)] + [Prefix((int)Issue540Token.BitwiseNegation, Associativity.Right, BitwiseNotExprPrecedence)] + [Prefix((int)Issue540Token.Subtraction, Associativity.Right, NegationPrecedence)] + public object Prefix(Token Op, object Right) => null; + + + [Infix((int)Issue540Token.Equals, Associativity.Right, 1)] + [Infix((int)Issue540Token.LogicalOr, Associativity.Left, 1)] + [Infix((int)Issue540Token.LogicalXor, Associativity.Left, 1)] + [Infix((int)Issue540Token.LogicalAnd, Associativity.Left, 1)] + [Infix((int)Issue540Token.Addition, Associativity.Left, 1)] + [Infix((int)Issue540Token.Subtraction, Associativity.Left, 1)] + [Infix((int)Issue540Token.Division, Associativity.Left, 1)] + [Infix((int)Issue540Token.Multiplication, Associativity.Left, 1)] + [Infix((int)Issue540Token.Exponentiation, Associativity.Right, 1)] + [Infix((int)Issue540Token.Subtraction, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseOr, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseXor, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseAnd, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseLeftShift, Associativity.Left, 1)] + [Infix((int)Issue540Token.BitwiseRightShift, Associativity.Left, 1)] + public object BinOp(object left, Token Op, object right) => null; } } \ No newline at end of file diff --git a/tests/ParserTests/Issue540/Issue540Token.cs b/tests/ParserTests/Issue540/Issue540Token.cs index 846302c1..7321e481 100644 --- a/tests/ParserTests/Issue540/Issue540Token.cs +++ b/tests/ParserTests/Issue540/Issue540Token.cs @@ -1,6 +1,154 @@ -namespace ParserTests.Issue540; +using System.ComponentModel.DataAnnotations; +using sly.lexer; -public enum Issue540Token +namespace ParserTests.Issue540 { - + public enum Issue540Token + { + EOF, + + //should be shared between all sets of valid tokens, since invalid ones will simply be unused + [Lexeme(GenericToken.Int)] [Lexeme(GenericToken.Double)] + Number, + + [Lexeme(GenericToken.Identifier, IdentifierType.Custom, "_A-Za-z", "_0-9A-Za-z")] + Identifier, + + [Lexeme(GenericToken.String)] [Lexeme(GenericToken.String, "'")] + String, + + //Operators + + #region bitwise operator + + [Sugar("&")] BitwiseAnd, + [Sugar("|")] BitwiseOr, + [Sugar("^")] BitwiseXor, + [Sugar("<<")] BitwiseLeftShift, + [Sugar(">>")] BitwiseRightShift, + [Sugar("~")] BitwiseNegation, + + #endregion + + #region arithmetic operator + + [Sugar("+")] Addition, + [Sugar("-")] Subtraction, + [Sugar("*")] Multiplication, + [Sugar("/")] Division, + [Sugar("**")] Exponentiation, + [Sugar("!")] Factorial, + + #endregion + + #region logical operator + + [Keyword("and")] LogicalAnd, + [Keyword("or")] LogicalOr, + [Keyword("not")] LogicalNot, + [Keyword("xor")] LogicalXor, + [Keyword("implies")] LogicalImplies, + + #endregion + + #region Brackets + + [Sugar("(")] OpenParen, + [Sugar(")")] CloseParen, + [Sugar("{")] OpenCurly, + [Sugar("}")] CloseCurly, + [Sugar("[")] OpenSquare, + [Sugar("]")] CloseSquare, + [Sugar("<[")] OpenAngleSquare, + [Sugar("]>")] CloseAngleSquare, + + #endregion + + #region Symbols + + [Sugar(",")] Comma, + [Sugar(":")] Colon, + [Sugar(".")] Dot, + + #endregion + + #region Comparison_Operator + + [Sugar("==")] EqualTo, + [Sugar("!=")] NotEqualTo, + [Sugar(">")] GreaterThan, + [Sugar("<")] LessThan, + [Sugar(">=")] GreaterThanOrEqualTo, + [Sugar("<=")] LessThanOrEqualTo, + [Keyword("is")] ComparisonIs, + [Keyword("in")] In, + [Sugar(";")] Semicolon, + + #endregion + + [Sugar("=")] Equals, + + #region Types + + [Keyword("float")] TypeFloat, + [Keyword("int")] TypeInt, + [Keyword("double")] TypeDouble, + + [Keyword("number")] [Keyword("bigfloat")] + TypeNumber, + [Keyword("long")] TypeLong, + + [Keyword("longint")] [Keyword("bigint")] + TypeLongInt, + [Keyword("byte")] TypeByte, + [Keyword("array")] TypeArray, + [Keyword("list")] TypeList, + [Keyword("set")] TypeSet, + [Keyword("dict")] TypeDict, + [Keyword("short")] TypeShort, + [Keyword("rational")] TypeRational, + [Keyword("string")] TypeString, + [Keyword("char")] TypeChar, + [Keyword("void")] TypeVoid, + + #endregion + + #region control-flow + + [Keyword("return")] Return, + [Keyword("break")] Break, + [Keyword("continue")] Continue, + [Keyword("if")] If, + [Keyword("else")] Else, + [Keyword("switch")] Switch, + + #endregion + + #region Loops + + [Keyword("for")] For, + [Keyword("while")] While, + + #endregion + + [Keyword("as")] As, + + #region FunctionMod + + [Keyword("cascading")] Cascading, + [Keyword("copy")] Copy, + [Keyword("readonly")] Readonly, + [Keyword("frozen")] Frozen, + [Keyword("immut")] Immut, + [Keyword("ref")] Ref, + [Keyword("new")] New, + + #endregion + + [Keyword("true")] TrueLiteral, + [Keyword("false")] FalseLiteral, + [Keyword("collection")] TypeCollection, + [Comment("#", "/*", "*/")] Comment + + } } \ No newline at end of file diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index b61715bc..5257ce5a 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -7,6 +7,7 @@ using ParserTests.Issue260; using ParserTests.Issue277; using ParserTests.Issue536; +using ParserTests.Issue540; using SlowEOS; using sly.buildresult; using sly.lexer; @@ -135,6 +136,17 @@ public static void Issue536Test() Check.That(equals.TokenID).IsEqualTo(Token536.Equals); var x = Token536.Equals; } + + [Fact] + public static void Issue540Test() + { + ParserBuilder builder = new ParserBuilder(); + var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NTSCExpr"); + Check.That(buildParser).IsOk(); + var parser = buildParser.Result; + var parsed = parser.Parse("1"); + Check.That(parsed).IsOkParsing(); + } } From 484a0cad0f1290053493bbce8bf7f311f02c6948 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 13:11:05 +0100 Subject: [PATCH 3/7] #540 reproducing test case --- tests/ParserTests/Issue540/Issue540Parser.cs | 246 ++++++++----------- tests/ParserTests/IssuesTests.cs | 4 +- 2 files changed, 110 insertions(+), 140 deletions(-) diff --git a/tests/ParserTests/Issue540/Issue540Parser.cs b/tests/ParserTests/Issue540/Issue540Parser.cs index b4ffd169..3a9a4d04 100644 --- a/tests/ParserTests/Issue540/Issue540Parser.cs +++ b/tests/ParserTests/Issue540/Issue540Parser.cs @@ -1,9 +1,7 @@ -using System; using sly.parser.generator; using sly.parser.parser; using System.Collections.Generic; -using ParserTests.Issue540; using sly.lexer; @@ -11,196 +9,168 @@ namespace ParserTests.Issue540 { public partial class Issue540Parser { - void AppendIfNotEmpty(List nodeList, ValueOption Considered) - { - if (GetFromValOp(Considered) is object node) - { - nodeList.Add(node); - } - } - - List BuildChildren(params object[] Vals) - { - return new(); - - } - - object? GetFromValOp(ValueOption value) - { - if (value.IsSome) - { - return value.Match(x => x, () => throw new Exception("IsSome was true when no value was found")); - } - - return null; - } - - object FromToken(Issue540Token t) => null; - - [Production($"NTSection: NTStatement*")] - public object NTSection(List Statements) - { - return null; - } + + [Production($"NtSection: NtStatement*")] + public object NTSection(List a) => null; [Production( - $"NTStatement: [NTSCExpr | NTLoop | NTCond | NTFunction | NTBlock | NTReturnStatement | NTLoopControlStatement]")] - public object NTStatement(object SubStatement) => SubStatement; + $"NtStatement: [NtSCExpr | NtLoop | NtCond | NtFunction | NtBlock | NtReturnStatement | NtLoopControlStatement]")] + public object NtStatement(object subStatement) => null; - [Production($"NTSCExpr: NTExpression")] + [Production($"NtSCExpr: NtExpression Semicolon [d]")] - public object NTSCExpr(object Expression) => Expression; + public object NtScExpr(object expression) => null; - [Production($"NTReturnStatement: Return [d] NTSCExpr")] - public object NTReturnStatement(object SCExpr) => null; + [Production($"NtReturnStatement: Return [d] NtSCExpr")] + public object NtReturnStatement(object expr) => null; - [Production($"NTLoopControlStatement: [Break | Continue] NTNestedValueInLoopControl?")] - public object NTLoopControlStatement(Issue540Token Operator, ValueOption NestedVal) => null; + [Production($"NtLoopControlStatement: [Break | Continue] NtNestedValueInLoopControl?")] + public object NtLoopControlStatement(Issue540Token op, ValueOption nestedVal) => null; - [Production($"NTNestedValueInLoopControl: Identifier")] - public object NTNestedValueInLoopControl(Token val) => null; + [Production($"NtNestedValueInLoopControl: Identifier")] + public object NtNestedValueInLoopControl(Token val) => null; - [Production($"NTLoop: NTForLoopHeader NTLoopLabel? NTStatement NTElse?")] - [Production($"NTLoop: NTWhileLoopHeader NTLoopLabel? NTStatement NTElse?")] - public object NTLoop(object LoopHeader, ValueOption Label, object StatementExpr, - ValueOption ElseExpr) => null; + [Production($"NtLoop: NtForLoopHeader NtLoopLabel? NtStatement NtElse?")] + [Production($"NtLoop: NtWhileLoopHeader NtLoopLabel? NtStatement NtElse?")] + public object NtLoop(object loopHeader, ValueOption label, object statementExpr, + ValueOption slseExpr) => null; - [Production($"NTLoopLabel: As [d] Identifier")] - public object NTLoopLabel(Token ident) => null; + [Production($"NtLoopLabel: As [d] Identifier")] + public object NtLoopLabel(Token ident) => null; [Production( - $"NTForLoopHeader: For [d] OpenParen [d] NTExpression Semicolon [d] NTExpression Semicolon [d] NTExpression CloseParen [d]")] - public object NTForLoopHeader(object Init, object Condition, object Step) => null; + $"NtForLoopHeader: For [d] OpenParen [d] NtExpression Semicolon [d] NtExpression Semicolon [d] NtExpression CloseParen [d]")] + public object NtForLoopHeader(object init, object condition, object step) => null; - [Production($"NTWhileLoopHeader: While [d] OpenParen [d] NTExpression CloseParen [d]")] - public object NTWhileLoopHeader(object Condition) => null; + [Production($"NtWhileLoopHeader: While [d] OpenParen [d] NtExpression CloseParen [d]")] + public object NtWhileLoopHeader(object condition) => null; - [Production($"NTCond: [NTSwitch | NTIf]")] - public object NTCond(object C) => null; + [Production($"NtCond: [NtSwitch | NtIf]")] + public object NtCond(object _) => null; - [Production($"NTIf: If [d] OpenParen [d] NTExpression CloseParen [d] NTStatement NTElse?")] - public object NTIf(object Cond, object StatementExpr, ValueOption ElseExpr) => null; + [Production($"NtIf: If [d] OpenParen [d] NtExpression CloseParen [d] NtStatement NtElse?")] + public object NtIf(object a, object b, ValueOption c) => null; - [Production($"NTElse: Else [d] NTStatement")] - public object NTElse(object AStatement) => null; + [Production($"NtElse: Else [d] NtStatement")] + public object NtElse(object _) => null; [Production( - $"NTSwitch: Switch [d] OpenParen [d] NTExpression CloseParen [d] OpenCurly [d] NTSwitchBody* CloseCurly [d]")] - public object NTSwitch(object AExpression, List ASwitchBody) => null; + $"NtSwitch: Switch [d] OpenParen [d] NtExpression CloseParen [d] OpenCurly [d] NtSwitchBody* CloseCurly [d]")] + public object NtSwitch(object a, List b) => null; - [Production($"NTSwitchBody: NTExpression Colon [d] NTStatement")] - public object NTSwitchBody(object AExpr, object AStatement) => null; + [Production($"NtSwitchBody: NtExpression Colon [d] NtStatement")] + public object NtSwitchBody(object a, object b) => null; - [Production($"NTFunction: NTType Identifier OpenParen [d] NTTypeAndIdentifierCSV? CloseParen [d] NTStatement")] - public object NTFunction(object AType, Token Ident, ValueOption TICSV, object Statement) => + [Production($"NtFunction: NtType Identifier OpenParen [d] NtTypeAndIdentifierCSV? CloseParen [d] NtStatement")] + public object NtFunction(object a, Token b, ValueOption c, object d) => null; - [Production($"NTTypeAndIdentifierCSV: NTTypeAndIdentifierCSVElement (Comma [d] NTTypeAndIdentifierCSV)*")] - public object NTTypeAndIdentifierCSV(object Element, List> Prime) => null; + [Production($"NtTypeAndIdentifierCSV: NtTypeAndIdentifierCSVElement (Comma [d] NtTypeAndIdentifierCSV)*")] + public object NtTypeAndIdentifierCsv(object a, List> b) => null; - [Production($"NTTypeAndIdentifierCSVElement: NTFunctionArgDeclModifiersCombined? NTType Identifier")] - public object NTTypeAndIdentifierCSVElement(ValueOption Modifiers, object AType, Token Ident) => + [Production($"NtTypeAndIdentifierCSVElement: NtFunctionArgDeclModifiersCombined? NtType Identifier")] + public object NtTypeAndIdentifierCsvElement(ValueOption a, object b, Token c) => null; - [Production($"NTBlock: OpenCurly [d] NTSection CloseCurly [d]")] - public object NTBlock(object ASection) => ASection; + [Production($"NtBlock: OpenCurly [d] NtSection CloseCurly [d]")] + public object NtBlock(object a) => null; - [Production($"NTExpression: NTAliasExpr")] - public object NTExpression(object pass) => pass; + [Production($"NtExpression: NtAliasExpr")] + public object NtExpression(object pass) => pass; - [Production($"NTAliasExpr: [NTAliasExpr1 | NTAliasExpr2 | NTAliasExpr3 | NTDeclarationExpr]")] - public object NTAliasExpr(object Node) => Node; + [Production($"NtAliasExpr: [NtAliasExpr1 | NtAliasExpr2 | NtAliasExpr3 | NtDeclarationExpr]")] + public object NtAliasExpr(object a) => null; - [Production($"NTAliasExpr1: Identifier As [d] Identifier")] - public object NTAliasExpr1(Token Ident, Token Ident2) => null; + [Production($"NtAliasExpr1: Identifier As [d] Identifier")] + public object NtAliasExpr1(Token a, Token b) => null; - [Production($"NTAliasExpr2: Identifier As [d] NTType Identifier")] - public object NTAliasExpr2(Token Ident, object AType, Token Ident2) => null; + [Production($"NtAliasExpr2: Identifier As [d] NtType Identifier")] + public object NtAliasExpr2(Token a, object b, Token v) => null; - [Production($"NTAliasExpr3: Identifier As [d] NTType")] - public object NTAliasExpr3(Token Ident, object Type) => null; + [Production($"NtAliasExpr3: Identifier As [d] NtType")] + public object NtAliasExpr3(Token a, object b) => null; - [Production($"NTDeclarationExpr: [NTDeclarationExpr1 | NTAssignmentExpr]")] - public object NTDeclarationExpr(object Node) => Node; + [Production($"NtDeclarationExpr: [NtDeclarationExpr1 | NtAssignmentExpr]")] + public object NtDeclarationExpr(object a) => null; - [Production($"NTDeclarationExpr1: NTDeclarationModifiersCombined? NTType Identifier NTAssignmentPrime?")] - public object NTDeclarationExpr1(ValueOption Modifiers, object AType, Token Ident, - ValueOption AAssignmentPrime) => null; + [Production($"NtDeclarationExpr1: NtDeclarationModifiersCombined? NtType Identifier NtAssignmentPrime?")] + public object NtDeclarationExpr1(ValueOption a, object b, Token c, + ValueOption d) => null; - [Production($"NTDeclarationModifiersCombined: NTDeclarationModifier+")] - public object NTDeclarationModifiersCombined(List Modifiers) => null; + [Production($"NtDeclarationModifiersCombined: NtDeclarationModifier+")] + public object NtDeclarationModifiersCombined(List a) => null; - [Production($"NTDeclarationModifier: [Ref | Readonly | Frozen | Immut]")] - public object NTDeclarationModifier(Token Mod) => null; + [Production($"NtDeclarationModifier: [Ref | Readonly | Frozen | Immut]")] + public object NtDeclarationModifier(Token a) => null; - [Production($"NTFunctionArgDeclModifier: [Ref | Readonly | Frozen | Immut | Copy]")] - public object NTFunctionArgDeclModifier(Token Mod) => null; + [Production($"NtFunctionArgDeclModifier: [Ref | Readonly | Frozen | Immut | Copy]")] + public object NtFunctionArgDeclModifier(Token a) => null; - [Production($"NTFunctionArgDeclModifiersCombined: NTFunctionArgDeclModifier+")] - public object NTFunctionArgDeclModifiersCombined(List Modifiers) => null; + [Production($"NtFunctionArgDeclModifiersCombined: NtFunctionArgDeclModifier+")] + public object NtFunctionArgDeclModifiersCombined(List a) => null; - [Production($"NTAssignmentPrime: Equals NTExpression")] - public object NTAssignmentPrime(Token EQ, object Expr) => null; + [Production($"NtAssignmentPrime: Equals NtExpression")] + public object NtAssignmentPrime(Token a, object b) => null; - [Production($"NTAssignmentExpr: NTAssignmentExpr1")] - public object NTAssignmentExpr(object Node) => null; + [Production($"NtAssignmentExpr: NtAssignmentExpr1")] + public object NtAssignmentExpr(object a) => null; - [Production($"NTAssignmentExpr1: Issue540Parser_expressions")] - public object NTAssignmentExpr1(object P1) => null; + [Production($"NtAssignmentExpr1: Issue540Parser_expressions")] + public object NtAssignmentExpr1(object a) => null; [Operand] - [Production($"NTPrimary: NTLPrimary NTPrimaryPrime?")] - public object NTPrimary(object APrimary, ValueOption Prime) => null; + [Production($"NtPrimary: NtLPrimary NtPrimaryPrime?")] + public object NtPrimary(object a, ValueOption b) => null; - [Production($"NTPrimaryPrime: [NTIndexPrime | NTFunctionCallPrime]")] - public object NTPrimaryPrime(object Node) => null; + [Production($"NtPrimaryPrime: [NtIndexPrime | NtFunctionCallPrime]")] + public object NtPrimaryPrime(object a) => null; - [Production($"NTIndexPrime: OpenSquare [d] NTExpression CloseSquare [d]")] - public object NTIndexPrime(object Expr) => null; + [Production($"NtIndexPrime: OpenSquare [d] NtExpression CloseSquare [d]")] + public object NtIndexPrime(object a) => null; - [Production($"NTLPrimary: [NTNewExpr | NTLPrimary1 | NTLPrimary2 | NTLPrimary3]")] - public object NTLPrimary(object Node) => null; + [Production($"NtLPrimary: [NtNewExpr | NtLPrimary1 | NtLPrimary2 | NtLPrimary3]")] + public object NtLPrimary(object a) => null; - [Production($"NTLPrimary1: OpenParen [d] NTExpression CloseParen [d]")] - public object NTLPrimary1(object Expr) => null; + [Production($"NtLPrimary1: OpenParen [d] NtExpression CloseParen [d]")] + public object NtLPrimary1(object a) => null; - [Production($"NTLPrimary2: Copy [d] NTExpression")] - public object NTLPrimary2(object Expr) => null; + [Production($"NtLPrimary2: Copy [d] NtExpression")] + public object NtLPrimary2(object a) => null; - [Production($"NTLPrimary3: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] - public object NTLPrimary3(Token Token) => null; + [Production($"NtLPrimary3: [Identifier | Number | String | TrueLiteral | FalseLiteral]")] + public object NtLPrimary3(Token a) => null; - [Production($"NTNewExpr: New [d] NTType OpenParen [d] NTArgList?")] - public object NTNewExpr(object AType, ValueOption AArgList) => null; + [Production($"NtNewExpr: New [d] NtType OpenParen [d] NtArgList?")] + public object NtNewExpr(object a, ValueOption b) => null; - [Production($"NTFunctionCallPrime:OpenParen [d] NTArgList? CloseParen [d]")] - public object NTFunctionCallPrime(ValueOption AArgList) => null; + [Production($"NtFunctionCallPrime:OpenParen [d] NtArgList? CloseParen [d]")] + public object NtFunctionCallPrime(ValueOption a) => null; - [Production($"NTArgList: NTArgListElement NTArgListPrime*")] - public object NTArgList(object Element, List Elements) => null; + [Production($"NtArgList: NtArgListElement NtArgListPrime*")] + public object NtArgList(object a, List b) => null; - [Production($"NTArgListElement: NTArgumentLabel? NTExpression")] - public object NTArgListElement(ValueOption Label, object Expr) => null; + [Production($"NtArgListElement: NtArgumentLabel? NtExpression")] + public object NtArgListElement(ValueOption a, object b) => null; - [Production($"NTArgListPrime: Comma [d] NTArgListElement")] - public object NTArgListPrime(object Element) => null; + [Production($"NtArgListPrime: Comma [d] NtArgListElement")] + public object NtArgListPrime(object a) => null; - [Production($"NTArgumentLabel: Identifier Colon [d]")] - public object NTArgumentLabel(Token Ident) => null; + [Production($"NtArgumentLabel: Identifier Colon [d]")] + public object NtArgumentLabel(Token ident) => null; - [Production($"NTTypeCSV: NTType (Comma [d] NTType)*")] - public object NTTypeCSV(object AType, List> OtherTypes) => null; + [Production($"NtTypeCSV: NtType (Comma [d] NtType)*")] + public object NtTypeCsv(object aType, List> otherTypes) => null; - [Production($"NTType: [NTBaseType | NTGenericType]")] - public object NTType(object Node) => null; + [Production($"NtType: [NtBaseType | NtGenericType]")] + public object NtType(object node) => null; [Production( - $"NTGenericType: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] NTTypeCSV CloseAngleSquare [d]")] - public object NTGenericType(Token TypeToken, object TypeArgs) => null; + $"NtGenericType: [TypeArray | TypeList | TypeSet | TypeDict | TypeCollection] OpenAngleSquare [d] NtTypeCSV CloseAngleSquare [d]")] + public object NtGenericType(Token typeToken, object typeArgs) => null; [Production( - $"NTBaseType: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] - public object NTBaseType(Token TypeToken) => null; + $"NtBaseType: [TypeByte | TypeShort | TypeInt | TypeLong | TypeLongInt | TypeFloat | TypeDouble | TypeRational | TypeNumber | TypeString | TypeChar | TypeVoid]")] + public object NtBaseType(Token typeToken) => null; } @@ -221,7 +191,7 @@ public partial class Issue540Parser [Infix((int)Issue540Token.LessThan, Associativity.Right, LessThanPrecedence)] [Infix((int)Issue540Token.GreaterThanOrEqualTo, Associativity.Right, GreaterThanOrEqualToPrecedence)] [Infix((int)Issue540Token.LessThanOrEqualTo, Associativity.Right, LessThanOrEqualToPrecedence)] - public object ComparisonExpressions(object Left, Token Operator, object Right) => null; + public object ComparisonExpressions(object a, Token b, object c) => null; #endregion @@ -260,12 +230,12 @@ public partial class Issue540Parser #endregion [Postfix((int)Issue540Token.Factorial, Associativity.Left, FactorialPrecedence)] - public object Factorial(object Left, Token Op) => null; + public object Factorial(object a, Token b) => null; [Prefix((int)Issue540Token.LogicalNot, Associativity.Right, NotExprPrecedence)] [Prefix((int)Issue540Token.BitwiseNegation, Associativity.Right, BitwiseNotExprPrecedence)] [Prefix((int)Issue540Token.Subtraction, Associativity.Right, NegationPrecedence)] - public object Prefix(Token Op, object Right) => null; + public object Prefix(Token a, object b) => null; [Infix((int)Issue540Token.Equals, Associativity.Right, 1)] @@ -283,6 +253,6 @@ public partial class Issue540Parser [Infix((int)Issue540Token.BitwiseAnd, Associativity.Left, 1)] [Infix((int)Issue540Token.BitwiseLeftShift, Associativity.Left, 1)] [Infix((int)Issue540Token.BitwiseRightShift, Associativity.Left, 1)] - public object BinOp(object left, Token Op, object right) => null; + public object BinOp(object left, Token a, object b) => null; } } \ No newline at end of file diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index 5257ce5a..481f6a0c 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -141,11 +141,11 @@ public static void Issue536Test() public static void Issue540Test() { ParserBuilder builder = new ParserBuilder(); - var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NTSCExpr"); + var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NtSCExpr"); Check.That(buildParser).IsOk(); var parser = buildParser.Result; var parsed = parser.Parse("1"); - Check.That(parsed).IsOkParsing(); + Check.That(parsed.IsOk).IsTrue(); } } From 4b161c20f70a54bdca224090711294cc757d3c56 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 13:17:00 +0100 Subject: [PATCH 4/7] . --- tests/ParserTests/Issue540/Issue540Parser.cs | 6 +----- tests/ParserTests/IssuesTests.cs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/ParserTests/Issue540/Issue540Parser.cs b/tests/ParserTests/Issue540/Issue540Parser.cs index 3a9a4d04..ed6603bc 100644 --- a/tests/ParserTests/Issue540/Issue540Parser.cs +++ b/tests/ParserTests/Issue540/Issue540Parser.cs @@ -7,7 +7,7 @@ namespace ParserTests.Issue540 { - public partial class Issue540Parser + public class Issue540Parser { [Production($"NtSection: NtStatement*")] @@ -171,11 +171,7 @@ public object NtDeclarationExpr1(ValueOption a, object b, Token typeToken) => null; - } - - public partial class Issue540Parser - { #region ComparisonExpressions const int EqualToPrecedence = NotExprPrecedence + 1; diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index 481f6a0c..a1f278cf 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -145,7 +145,7 @@ public static void Issue540Test() Check.That(buildParser).IsOk(); var parser = buildParser.Result; var parsed = parser.Parse("1"); - Check.That(parsed.IsOk).IsTrue(); + Check.That(parsed).IsOkParsing(); } } From 7ee306aca6421696f7c1a3348d00a03baa532efb Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 14:42:06 +0100 Subject: [PATCH 5/7] testing and debugging --- tests/ParserTests/IssuesTests.cs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index a1f278cf..9fc6fe43 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Linq; using NFluent; using ParserTests.Issue218; @@ -137,12 +138,41 @@ public static void Issue536Test() var x = Token536.Equals; } + private static void PrintLeadings(ParserConfiguration configuration, string nonterminal) + { + var nt = configuration.NonTerminals[nonterminal]; + var leading = $">>{nonterminal}<< {string.Join(", ", nt.GetPossibleLeadingTokens())}"; + Console.WriteLine(leading); + File.AppendAllLines("c:/tmp/leadings.txt", new []{leading,""}); + } + [Fact] public static void Issue540Test() { ParserBuilder builder = new ParserBuilder(); var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NtSCExpr"); Check.That(buildParser).IsOk(); + + var configuration = buildParser.Result.Configuration; + + PrintLeadings(configuration, "NtAssignmentExpr1"); + + PrintLeadings(configuration, "NtAssignmentExpr"); + + PrintLeadings(configuration, "NtDeclarationExpr"); + + PrintLeadings(configuration, "NtAliasExpr"); + + PrintLeadings(configuration, "NtExpression"); + + PrintLeadings(configuration, "NtSCExpr"); + + PrintLeadings(configuration, "NtStatement"); + + PrintLeadings(configuration, "NtSection"); + + + var parser = buildParser.Result; var parsed = parser.Parse("1"); Check.That(parsed).IsOkParsing(); From 8b24d982177e89a7a8d8121bb8793debb6847504 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 16:41:53 +0100 Subject: [PATCH 6/7] fix #540 --- src/sly/parser/parser/Parser.cs | 7 ++++++- .../parser/llparser/bnf/RecursiveDescentSyntaxParser.cs | 5 +++++ tests/ParserTests/IssuesTests.cs | 4 ++-- tests/ParserTests/NFluentParseExtensions.cs | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/sly/parser/parser/Parser.cs b/src/sly/parser/parser/Parser.cs index b133b61b..30c710bd 100644 --- a/src/sly/parser/parser/Parser.cs +++ b/src/sly/parser/parser/Parser.cs @@ -44,7 +44,12 @@ public virtual BuildResult> BuildExpressionParser( var expressionGenerator = new ExpressionRulesGenerator(I18n); exprResult = expressionGenerator.BuildExpressionRules(Configuration, Instance.GetType(), exprResult); Configuration = exprResult.Result; - SyntaxParser.Init(exprResult.Result, startingRule); + if (exprResult.IsOk) + { + // #540 : recompute starting tokens taking account of expression rules + SyntaxParser.Init(exprResult.Result, startingRule); + } + if (startingRule != null) { Configuration.StartingRule = startingRule; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs index 951b821c..54ab25d4 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs @@ -247,6 +247,11 @@ private SyntaxParseResult NoMatchingRuleError(IList> tokens, public virtual void Init(ParserConfiguration configuration, string root) { if (root != null) StartingNonTerminal = root; + // #540 : reset all leading tokens and recompute with the new configuration (expression rules) + foreach (var nonTerminal in configuration.NonTerminals.Values) + { + nonTerminal?.Rules?.ForEach(x => x?.PossibleLeadingTokens?.Clear()); + } InitializeStartingTokens(configuration, StartingNonTerminal); } diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index 9fc6fe43..52ea0f1c 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -174,8 +174,8 @@ public static void Issue540Test() var parser = buildParser.Result; - var parsed = parser.Parse("1"); - Check.That(parsed).IsOkParsing(); + var parsed = parser.Parse("1;"); + Check.That(parsed).IsOkParsing(checkIfResultisNull:false); } } diff --git a/tests/ParserTests/NFluentParseExtensions.cs b/tests/ParserTests/NFluentParseExtensions.cs index 158429a0..5c2cd6ef 100644 --- a/tests/ParserTests/NFluentParseExtensions.cs +++ b/tests/ParserTests/NFluentParseExtensions.cs @@ -11,11 +11,11 @@ namespace ParserTests { public static class NFluentParseExtensions { - public static ICheckLink>> IsOkParsing(this ICheck> context) where IN : struct + public static ICheckLink>> IsOkParsing(this ICheck> context, bool checkIfResultisNull = true) where IN : struct { ExtensibilityHelper.BeginCheck(context) .FailWhen(sut => sut.IsError, $"parse failed") - .FailWhen(sut => sut.Result == null && !sut.SyntaxTree.IsEpsilon, "parse result is null") + .FailWhen(sut => sut.Result == null && checkIfResultisNull && !sut.SyntaxTree.IsEpsilon, "parse result is null") .OnNegate("parse expected to fail.") .EndCheck(); return ExtensibilityHelper.BuildCheckLink(context); From 867958808a3b855f99075d3434441ae93e59c548 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Fri, 21 Feb 2025 16:45:27 +0100 Subject: [PATCH 7/7] cleaning --- tests/ParserTests/IssuesTests.cs | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/tests/ParserTests/IssuesTests.cs b/tests/ParserTests/IssuesTests.cs index 52ea0f1c..f9de9095 100644 --- a/tests/ParserTests/IssuesTests.cs +++ b/tests/ParserTests/IssuesTests.cs @@ -137,41 +137,13 @@ public static void Issue536Test() Check.That(equals.TokenID).IsEqualTo(Token536.Equals); var x = Token536.Equals; } - - private static void PrintLeadings(ParserConfiguration configuration, string nonterminal) - { - var nt = configuration.NonTerminals[nonterminal]; - var leading = $">>{nonterminal}<< {string.Join(", ", nt.GetPossibleLeadingTokens())}"; - Console.WriteLine(leading); - File.AppendAllLines("c:/tmp/leadings.txt", new []{leading,""}); - } - + [Fact] public static void Issue540Test() { ParserBuilder builder = new ParserBuilder(); var buildParser = builder.BuildParser(new Issue540Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "NtSCExpr"); Check.That(buildParser).IsOk(); - - var configuration = buildParser.Result.Configuration; - - PrintLeadings(configuration, "NtAssignmentExpr1"); - - PrintLeadings(configuration, "NtAssignmentExpr"); - - PrintLeadings(configuration, "NtDeclarationExpr"); - - PrintLeadings(configuration, "NtAliasExpr"); - - PrintLeadings(configuration, "NtExpression"); - - PrintLeadings(configuration, "NtSCExpr"); - - PrintLeadings(configuration, "NtStatement"); - - PrintLeadings(configuration, "NtSection"); - - var parser = buildParser.Result; var parsed = parser.Parse("1;");