From bc9d38da0feeaaceea42090089391181d4594a49 Mon Sep 17 00:00:00 2001 From: bhandfast Date: Tue, 4 Oct 2016 18:08:20 +0200 Subject: [PATCH] GRpc options (#61) / fully implement options Fully implement (and otherwise fix) option statements and option annotations on messages, fields, maps, services, rpc, etc,, including alternate syntax. Now fully supports gRPC option annotations. Squashed the following commits: * Attempt to parse proto3 option maps. * Got simple parsing working. * More unit-tests. * Unit-tests for grpc with options are green. * Cleanup * Changed aggregate options as per feedback. * Implemented aggregated option parser. * Added tests and support for recursive literals. * Handle empty literals. * Refactor options parsing to better handle empty statements. * Minor refactor of RPC parser to improve readability. * Mark skipEmpty as internal. * Remove superfluous empty_ws. * Refactor to eliminate duplication via new manyBetweenCurlySkippingEmpty, add unit test for Service options, and fixed bugs related to parsing options. * Support options on map field, add more option unit tests, move/rename unit tests for consistency. --- Froto.sln | 3 +- Parser.Test/TestClassModel.fs | 9 +- Parser.Test/TestParser.fs | 249 ++++++++++++++++++++++++++++- Parser/Ast.fs | 9 +- Parser/Parser.fs | 129 +++++++++------ TypeProvider/Generation/TypeGen.fs | 2 +- test/FooOptions.proto | 2 +- test/grpc.proto | 83 ++++++++++ 8 files changed, 425 insertions(+), 61 deletions(-) create mode 100644 test/grpc.proto diff --git a/Froto.sln b/Froto.sln index d92aa0b..8a70d44 100644 --- a/Froto.sln +++ b/Froto.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 14 -VisualStudioVersion = 14.0.24720.0 +VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Froto.Roslyn", "Roslyn\Froto.Roslyn.fsproj", "{49D025ED-F804-4D97-9027-461671B333B6}" EndProject @@ -24,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{992FDC44-0 test\addressbook.proto = test\addressbook.proto test\addressbook1.proto = test\addressbook1.proto test\FooOptions.proto = test\FooOptions.proto + test\grpc.proto = test\grpc.proto test\javatutorial.proto = test\javatutorial.proto test\nwind.proto = test\nwind.proto test\person.proto = test\person.proto diff --git a/Parser.Test/TestClassModel.fs b/Parser.Test/TestClassModel.fs index da863c2..0f22d79 100644 --- a/Parser.Test/TestClassModel.fs +++ b/Parser.Test/TestClassModel.fs @@ -119,4 +119,11 @@ let ``can parse riak proto`` () = let ``can parse enum proto`` () = let proto = getTestFile "protoenum.proto" |> parseFile 1 |> should equal proto.Enums.Length - 3 |> should equal proto.Enums.[0].Items.Length \ No newline at end of file + 3 |> should equal proto.Enums.[0].Items.Length + +[] +// from https://github.com/googleapis/googleapis/blob/master/google/pubsub/v1/pubsub.proto +let ``can parse PubSub proto`` () = + let proto = getTestFile "grpc.proto" |> parseFile + 10 |> should equal proto.Sections.Length + "Subscriber" |> should equal proto.Services.[0].Name \ No newline at end of file diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 450721c..c044f37 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -266,6 +266,112 @@ module OptionStatement = Parse.fromStringWithParser pOptionStatement @"option (test).field.more = true;" |> should equal (TOption ("test.field.more", TBoolLit true)) + [] + let ``Alternate option syntax parses`` () = + let option = """option (google.api.http) = { put: "/v1/{name=projects/*/subscriptions/*}" body: "*" };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + [ ("put", TStrLit "/v1/{name=projects/*/subscriptions/*}"); + ("body", TStrLit "*") ] + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + + [] + let ``Numeric option parses`` () = + let option = """option (google.api.http) = { put: "/v1/{name=projects/*/subscriptions/*}" body: 1 };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + [ ("put", TStrLit "/v1/{name=projects/*/subscriptions/*}"); + ("body", TIntLit 1) ] + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + + [] + let ``Recursive option parses`` () = + let option = """option (google.api.http) = { + get: "/v1/messages/{message_id}" + additional_bindings { + get: "/v1/users/{user_id}/messages/{message_id}" + } + };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + [ ("get", TStrLit "/v1/messages/{message_id}"); + ("additional_bindings", TAggregateOptionsLit [ ("get", TStrLit "/v1/users/{user_id}/messages/{message_id}") ]) ] + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + + [] + let ``Two recursive options parses`` () = + let option = """option (google.api.http) = { + get: "/v1/messages/{message_id}" + additional_bindings { + get: "/v1/users/{user_id}/messages/{message_id}" + } + additional_bindings { + put: "/v1/users/{user_id}/messages/{message_id}" + } + };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + [ ("get", TStrLit "/v1/messages/{message_id}"); + ("additional_bindings", TAggregateOptionsLit [ ("get", TStrLit "/v1/users/{user_id}/messages/{message_id}") ]) + ("additional_bindings", TAggregateOptionsLit [ ("put", TStrLit "/v1/users/{user_id}/messages/{message_id}") ]) ] + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + + [] + let ``Two recursive options with empty statements parses`` () = + let option = """option (google.api.http) = { + ; + get: "/v1/messages/{message_id}" + ; + additional_bindings { + get: "/v1/users/{user_id}/messages/{message_id}" + } + ; + ; + additional_bindings { + put: "/v1/users/{user_id}/messages/{message_id}" + }; + };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + [ ("get", TStrLit "/v1/messages/{message_id}"); + ("additional_bindings", TAggregateOptionsLit [ ("get", TStrLit "/v1/users/{user_id}/messages/{message_id}") ]) + ("additional_bindings", TAggregateOptionsLit [ ("put", TStrLit "/v1/users/{user_id}/messages/{message_id}") ]) ] + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + + [] + let ``Option with only empty statement parses`` () = + let option = """option (google.api.http) = { + ; + };""" + let result = Parse.fromStringWithParser Parse.Parsers.pOptionStatement option + + let expectedResults = + List.Empty + let expectedResult = TOption( "google.api.http", PConstant.TAggregateOptionsLit expectedResults ) + + result + |> should equal (expectedResult) + [] module Message = @@ -292,6 +398,17 @@ module Message = [ ("packed", TBoolLit(true)); ("custom.option", TIntLit(2)) ] ) + [] + let ``Field with alt syntax options parses`` () = + Parse.fromStringWithParser pField @"repeated sint32 samples=2 [(custom) = { option:2 another:false }];" + |> should equal + (TField( "samples", + TRepeated, + TSInt32, + 2u, + [ "custom",TAggregateOptionsLit( [ ("option",TIntLit 2); ("another",TBoolLit false) ] ) ] + )) + [] let ``Parse simple message`` () = Parse.fromStringWithParser (ws >>. pMessage .>> ws) @@ -305,6 +422,21 @@ module Message = TField( "blob", TRequired, TBytes, 2u, [ ("myopt",TStrLit("yes")) ]) ]) + [] + let ``Parse simple message with option`` () = + Parse.fromStringWithParser (ws >>. pMessage .>> ws) + """message Echo { + option (foo) = "bar"; + required string msg=1; + required bytes blob=2 [(myopt)="yes"] ; + }""" + |> should equal + <| TMessage ("Echo", + [ TMessageOption( "foo", TStrLit "bar") + TField( "msg", TRequired, TString, 1u, [] ) + TField( "blob", TRequired, TBytes, 2u, [ ("myopt",TStrLit("yes")) ]) + ]) + [] let ``Parse group`` () = Parse.fromStringWithParser (ws >>. pGroup) """ @@ -316,6 +448,20 @@ module Message = [ TField( "abc", TOptional, TString, 42u, [] ) ] ) + [] + let ``Parse group with option`` () = + Parse.fromStringWithParser (ws >>. pGroup) """ + optional group MyGroup = 41 { + optional string abc = 42; + option (foo) = "bar"; + }""" + |> should equal + <| TGroup( "MyGroup", TOptional, 41u, + [ TField( "abc", TOptional, TString, 42u, [] ) + TMessageOption( "foo", TStrLit "bar" ) + ] + ) + [] let ``Parse oneof`` () = Parse.fromStringWithParser (ws >>. pOneOf) """ @@ -325,12 +471,28 @@ module Message = |> should equal <| TOneOf("MyOneof", [ TOneOfField("name",TString,1u,[]) ]) + [] + let ``Parse oneof with field option`` () = + Parse.fromStringWithParser (ws >>. pOneOf) """ + oneof MyOneof { + string name = 1 [(foo)="bar"]; + }""" + |> should equal + <| TOneOf("MyOneof", [ TOneOfField("name",TString,1u,[("foo",TStrLit("bar"))]) ]) + [] let ``Parse map`` () = Parse.fromStringWithParser (ws >>. pMap) """ map projects = 3;""" |> should equal - <| TMap ("projects", TKString, TIdent("Project"), 3u) + <| TMap ("projects", TKString, TIdent("Project"), 3u, []) + + [] + let ``Parse map with option`` () = + Parse.fromStringWithParser (ws >>. pMap) """ + map projects = 3 [(foo)="bar"];""" + |> should equal + <| TMap ("projects", TKString, TIdent("Project"), 3u, ["foo",TStrLit("bar")]) [] let ``Parse extensions`` () = @@ -366,7 +528,7 @@ module Message = "bar" ] [] - let ``Parse enum`` () = + let ``Parse enum with option`` () = Parse.fromStringWithParser (ws >>. pMessageEnum) """ enum EnumAllowingAlias { option allow_alias = true; @@ -420,6 +582,23 @@ module Service = rpc TestMethod (outer) returns (foo);""" |> should equal (TRpc ("TestMethod", "outer", false, "foo", false, [])) + [] + let ``Parse rpc with optional option syntax`` () = + let expectedResults = + [ ("get", TStrLit "/v1/{name=projects/*/subscriptions/*}") ] + + let actual = + Parse.fromStringWithParser pRpc (""" + rpc TestMethod (outer) returns (foo) { + option (google.api.http) = { get: "/v1/{name=projects/*/subscriptions/*}" }; + } + """.Trim()) + + let expected = TRpc ("TestMethod", "outer", false, "foo", false, + [ "google.api.http", PConstant.TAggregateOptionsLit expectedResults ]) + + actual |> should equal expected + [] let ``Parse service`` () = Parse.fromStringWithParser (ws >>. pService) """ @@ -432,6 +611,37 @@ module Service = TRpc ("TestMethod", "outer", false, "foo", false, []) ])) + [] + let ``Parse service with rpc, options, and empty statements`` () = + Parse.fromStringWithParser (ws >>. pService) """ + service TestService { + ; // empty + option (foo).bar = "fee"; + rpc TestMethod (outer) returns (foo) { + ; // empty + option (foo.baz) = "fie"; + ; // empty + option foo = { bat : "foe" qux : "foo" } + ; // empty + } + ; // empty + }""" + |> should equal ( + TService ("TestService", + [ + TServiceOption ( "foo.bar", TStrLit "fee") + TRpc ("TestMethod", "outer", false, "foo", false, + [ + "foo.baz", TStrLit "fie" + "foo", TAggregateOptionsLit [ + "bat", TStrLit "foe" + "qux", TStrLit "foo" + ] + ]) + ])) + + + [] module Proto = @@ -499,7 +709,7 @@ module Proto = TMessageMessage ( "inner", [ TField ("ival", TRequired, TInt64, 1u, [] ) ]) TField ("inner_message", TRepeated, TIdent "inner", 2u, []) TField ("enum_field", TOptional, TIdent "EnumAllowingAlias", 3u, []) - TMap ("my_map", TKInt32, TString, 4u) + TMap ("my_map", TKInt32, TString, 4u, []) TExtensions ( [ (20u,Some(30u)) ]) ]) ] @@ -583,7 +793,7 @@ module Proto = TMessageMessage ( "inner", [ TField ("ival", TRequired, TInt64, 1u, [] ) ]) TField ("inner_message", TRepeated, TIdent "inner", 2u, []) TField ("enum_field", TOptional, TIdent "EnumAllowingAlias", 3u, []) - TMap ("my_map", TKInt32, TString, 4u) + TMap ("my_map", TKInt32, TString, 4u, []) TExtensions ( [ (20u,Some(30u)) ]) ]) TMessage ( "foo", @@ -600,6 +810,35 @@ module Proto = ] ) + [] + let ``Parse proto with optional option syntax`` () = + let expectedResults = + [ ("put", TStrLit "/v1/{name=projects/*/subscriptions/*}"); + ("body", TStrLit "*") ] + + Parse.fromStringWithParser pProto """ + syntax = "proto3"; + + service TestService { + rpc TestMethod (outer) returns (foo) { + option (google.api.http) = { put: "/v1/{name=projects/*/subscriptions/*}" body: "*" }; + } + } + """ + |> should equal ( + [ + TSyntax TProto3 + TService ("TestService", + [ + TRpc ("TestMethod", "outer", false, "foo", false, + [ + "google.api.http", PConstant.TAggregateOptionsLit expectedResults + ]) + ]) + ] + ) + + open System.IO /// gets the path for a test file based on the relative path from the executing assembly @@ -717,6 +956,7 @@ module Proto3 = """ |> ignore |> should throw typeof + [] module Proto2 = @@ -803,4 +1043,3 @@ module Proto2 = } """ |> ignore |> should not' (throw typeof) - diff --git a/Parser/Ast.fs b/Parser/Ast.fs index 31b0832..33ad65d 100644 --- a/Parser/Ast.fs +++ b/Parser/Ast.fs @@ -48,12 +48,15 @@ and PVisibility = // TOption and POption = TIdent * PConstant + +// and PConstant = | TIntLit of int32 | TFloatLit of float | TBoolLit of bool | TStrLit of string | TEnumLit of TIdent + | TAggregateOptionsLit of POption list with override x.ToString() = match x with @@ -62,11 +65,13 @@ and PConstant = | TBoolLit b -> sprintf "%s" (if b then "true" else "false") | TStrLit s -> sprintf "\"%s\"" s | TEnumLit s -> sprintf "%s" s + | TAggregateOptionsLit s -> sprintf "%A" s + // TMessage and PMessageStatement = | TField of TIdent * PLabel * PType * FieldNum * POption list - | TMap of TIdent * PKeyType * PType * FieldNum + | TMap of TIdent * PKeyType * PType * FieldNum * POption list | TGroup of TIdent * PLabel * FieldNum * PMessageStatement list // proto2 | TExtensions of TRange list // proto2 | TReservedRanges of TRange list @@ -81,7 +86,7 @@ and PMessageStatement = override x.ToString() = match x with | TField (id, lbl, vt, num, opts) -> sprintf "TField (%s,%A,%A,%u,[%A])" id lbl vt num opts - | TMap (id, kt, vt, num) -> sprintf "TMap (%s,%A,%A,%u)" id kt vt num + | TMap (id, kt, vt, num, opts) -> sprintf "TMap (%s,%A,%A,%u,[%A])" id kt vt num opts | TGroup (id, lbl, num, xs) -> sprintf "TGroup (%s,%A,%u,%A)" id lbl num xs | TExtensions (es) -> sprintf "TExtensions %A" es | TReservedRanges (rs) -> sprintf "TReservedRanges %A" rs diff --git a/Parser/Parser.fs b/Parser/Parser.fs index 9c59a14..946a7fe 100644 --- a/Parser/Parser.fs +++ b/Parser/Parser.fs @@ -57,6 +57,17 @@ module Parse = /// Skip at least 1 space followed by whitespace or comment let ws1 = spaces1 >>. ws + /// Parse end-of-statement + let internal eostm = pstring ";" + + /// Parse end-of-statement followed by zero or more whitespace + let internal eostm_ws = eostm .>> ws + + /// Skip empty statements followed by zero or more whitespace + let internal skipEmptyStmts = + skipMany eostm_ws + + /// Alias for pstring let str = pstring @@ -78,6 +89,9 @@ module Parse = let inline betweenParens p = between (str_ws "(") (str_ws ")") p + /// Parse many between curly braces, skipping empty statements + let manyBetweenCurlySkippingEmpty p = + betweenCurly (skipEmptyStmts >>. many (p .>> skipEmptyStmts)) // Run parser on string between quote (") or single-quote (') let inline betweenQuotes p = @@ -237,13 +251,6 @@ module Parse = pFullyQualifiedIdent |>> TEnumLit - /// Parser for constant: (boolLit | strLit | intLit | floatLit | Ident) - let pConstant = pBoolLit <|> pStrLit <|> pNumLit <|> pEnumLit - let pConstant_ws = pConstant .>> ws - - // Parser for end-of-statement - let internal eostm = str ";" .>> ws // note the implicit ws - (*--- Statement Parsers ---*) /// Parser for syntax: "syntax" "=" quote ("proto2" | "proto3") quote ";" @@ -251,7 +258,7 @@ module Parse = let pSyntax = str_ws "syntax" >>. str_ws "=" >>. betweenQuotes ( (str_ws "proto2" .>> setProto2) <|> (str_ws "proto3" .>> setProto3) ) - .>> eostm + .>> eostm_ws |>> function | "proto2" -> TSyntax TProto2 | "proto3" -> TSyntax TProto3 @@ -259,15 +266,15 @@ module Parse = // Import parsers let internal import = - str_ws1 "import" >>. strLit_ws .>> eostm + str_ws1 "import" >>. strLit_ws .>> eostm_ws |>> fun s -> (s,TNormal) let internal importPublic = - str_ws1 "import public" >>. strLit_ws .>> eostm + str_ws1 "import public" >>. strLit_ws .>> eostm_ws |>> fun s -> (s,TPublic) let internal importWeak = - str_ws1 "import weak" >>. strLit_ws .>> eostm + str_ws1 "import weak" >>. strLit_ws .>> eostm_ws |>> fun s -> (s,TWeak) /// Parser for import: "import" [ "public" | "weak" ] strLit ";" @@ -280,9 +287,36 @@ module Parse = /// Parser for package: "package" fullIdent ";" let pPackage = - str_ws1 "package" >>. pFullIdent_ws .>> eostm + str_ws1 "package" >>. pFullIdent_ws .>> eostm_ws |>> TPackage + (* Parsers for optionClause + + optionStatement :="option" fullIdent "=" optionClause ";" + optionClause := ( literal | aggregateBlock ) + literal := boolLit | strLit | numLit | enumLit + aggregateBlock := "{" [ aggregateLit | recursiveLit | emptyStatement ] "}" + aggregateLit := ident ":" literal + recursiveLit := ident aggregateBlock + emptyStatement := ";" + *) + + let internal pAggregateBlock_ws, internal pAggregateBlockR = createParserForwardedToRef() + + let internal pLiteral = pBoolLit <|> pStrLit <|> pNumLit <|> pEnumLit + let internal pLiteral_ws = pLiteral .>> ws + + let internal pAggregateLit_ws = pIdent_ws .>> str_ws ":" .>>. pLiteral_ws + let internal pRecursiveLit_ws = pIdent_ws .>>. pAggregateBlock_ws + + do pAggregateBlockR := + manyBetweenCurlySkippingEmpty + (attempt pAggregateLit_ws <|> pRecursiveLit_ws) + |>> TAggregateOptionsLit + + /// Parser for optionClause + let pOptionClause_ws = pLiteral_ws <|> pAggregateBlock_ws + /// Parser for optionName: (ident | "(" fullIdent ")") {"." ident} let pOptionName = let pIdentCustom_ws = @@ -297,13 +331,13 @@ module Parse = /// Parser for optionName + ws let pOptionName_ws = pOptionName .>> ws - /// Parser for optionClause: optionName "=" constant + /// Parser for optionName "=" optionClause let pOption_ws = - pOptionName_ws .>> str_ws "=" .>>. pConstant_ws + pOptionName_ws .>> str_ws "=" .>>. pOptionClause_ws /// Parser for option: "option" optionClause ";" let pOptionStatement = - str_ws1 "option" >>. pOption_ws .>> eostm + str_ws1 "option" >>. pOption_ws .>> eostm_ws |>> TOption let pLabel : Parser = @@ -379,7 +413,7 @@ module Parse = (opt pFieldOption_ws |>> defArg []) (fun lbl typ ident num opts -> (ident,lbl,typ,num,opts) ) - .>> eostm + .>> eostm_ws /// Parser for field: proto2: ("required" | "optional" | "repeated") ident "=" intLit [ "[" { fieldOptions } "]" ] ";" let pField = @@ -396,9 +430,6 @@ module Parse = // Top Level Statements - let internal skipEmptyStmts = - skipMany eostm - let internal pEnumCommon = let pEnumOption_ws = @@ -415,11 +446,10 @@ module Parse = ) let pEnumStatement = - ( pEnumOption_ws <|> pEnumField_ws ) .>> eostm - .>> skipEmptyStmts + ( pEnumOption_ws <|> pEnumField_ws ) .>> eostm_ws let pEnumBody = - betweenCurly (skipEmptyStmts >>. many pEnumStatement) + manyBetweenCurlySkippingEmpty pEnumStatement str_ws1 "enum" >>. pEnumName_ws .>>. pEnumBody @@ -442,8 +472,7 @@ module Parse = str_ws1 "message" >>. pMessageName_ws .>>. pMessageBody // Parse messageBody - and internal pMessageBody = - betweenCurly (many (skipEmptyStmts >>. pMessageStatement .>> skipEmptyStmts)) + and internal pMessageBody = manyBetweenCurlySkippingEmpty pMessageStatement // Message statement: field | enum | etc. and internal pMessageStatement = @@ -465,7 +494,7 @@ module Parse = /// Parse message option: "option" (ident | "(" fullIdent ")" { "." ident } and pMessageOption = - str_ws1 "option" >>. pOption_ws .>> eostm + str_ws1 "option" >>. pOption_ws .>> eostm_ws |>> TMessageOption /// Parse top-level group: label "group" groupName "=" fieldNumber messageBody @@ -490,7 +519,7 @@ module Parse = /// oneOfField: type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";' and pOneOf = (str_ws1 "oneof" >>. pOneOfName_ws) .>>. - (betweenCurly (many (skipEmptyStmts >>. pOneOfField .>> skipEmptyStmts)) ) + (manyBetweenCurlySkippingEmpty pOneOfField) |>> TOneOf and internal pOneOfField = @@ -498,14 +527,9 @@ module Parse = pType_ws pFieldName_ws pEq_FieldNum_ws - (opt pFieldOption_ws) - (fun typ ident num opts-> - let opts = - match opts with - | None -> [] - | Some(os) -> os - TOneOfField (ident,typ,num,opts) ) - .>> eostm + (opt pFieldOption_ws |>> defArg List.empty) + (fun typ ident num opts-> TOneOfField (ident,typ,num,opts) ) + .>> eostm_ws /// Parse map: "map" "<" keyType "," type ">" manName "=" fieldNumber [ "[" fieldOptions "]" ] ";' and pMap = @@ -533,19 +557,20 @@ module Parse = ] ) .>> ws |>> toKType - pipe4 + pipe5 (str_ws "map" >>. str_ws "<" >>. pKeyType_ws) (str_ws "," >>. pType_ws .>> str_ws ">") pMapName_ws pEq_FieldNum_ws - (fun ktype vtype name num -> TMap(name,ktype,vtype,num) ) - .>> eostm + (opt pFieldOption_ws |>> defArg List.empty) + (fun ktype vtype name num opts -> TMap(name,ktype,vtype,num,opts) ) + .>> eostm_ws /// Parse extensions: "extensions" ranges ";" /// ranges: range { "," range } /// range: intLit | "to" ( intLit | "max" ) ] and pExtensions = - str_ws1 "extensions" >>. pRanges .>> str_ws ";" + str_ws1 "extensions" >>. pRanges .>> eostm_ws |>> (fun xs -> TExtensions xs) /// Parse reserved: "reserved" ranges ";" @@ -560,7 +585,7 @@ module Parse = sepBy1 (betweenQuotes pFieldName_ws) (str_ws ",") |>> (fun xs -> TReservedNames xs) - str_ws1 "reserved" >>. (pResRanges <|> pResNames) .>> eostm + str_ws1 "reserved" >>. (pResRanges <|> pResNames) .>> eostm_ws // Parse list of ranges and internal pRanges = @@ -589,20 +614,20 @@ module Parse = and internal pExtendCommon = (str_ws1 "extend" >>. pMessageType_ws) - .>>. betweenCurly (many (skipEmptyStmts >>. (attempt pExtendField <|> pExtendGroup) .>> skipEmptyStmts)) + .>>. manyBetweenCurlySkippingEmpty (attempt pExtendField <|> pExtendGroup) /// Parse service: "service" serviceName "{" { option | rpc | emptyStatement } "}" and pService : Parser = - str_ws1 "service" >>. pServiceName_ws .>>. - (betweenCurly (many (skipEmptyStmts >>. (attempt pServiceOption <|> pRpc) .>> skipEmptyStmts))) + str_ws1 "service" >>. pServiceName_ws + .>>. manyBetweenCurlySkippingEmpty (attempt pServiceOption <|> pRpc) |>> TService /// Parse service options and pServiceOption = - pOption_ws |>> TServiceOption + str_ws1 "option" >>. pOption_ws |>> TServiceOption - /// Parse rpc: proto2: "rpc" rpcName "{" messageType ")" "returns" "(" messageType ")" [ "{" { option | emptyStatement } "}" ] ";" - /// Parse rpc: proto3: "rpc" rpcName "{" ["stream"] messageType ")" "returns" "(" ["stream"] messageType ")" [ "{" { option | emptyStatement } "}" ] ";" + /// Parse rpc: proto2: "rpc" rpcName "(" messageType ")" "returns" "(" messageType ")" [ "{" { option | emptyStatement } "}" ] ";" + /// Parse rpc: proto3: "rpc" rpcName "(" ["stream"] messageType ")" "returns" "(" ["stream"] messageType ")" [ "{" { option | emptyStatement } "}" ] ";" and pRpc = let pStrMessageType_P2 = betweenParens pMessageType_ws @@ -618,20 +643,24 @@ module Parse = <|> (isProto3 >>. pStrMessageType_P3) + let optionStatment = + str_ws1 "option" >>. pOption_ws .>> eostm_ws + let pRpcOptions = - betweenCurly - (sepBy pOption_ws (str_ws ",")) + manyBetweenCurlySkippingEmpty optionStatment + + /// RPC with options don't have an eostm... + let pRpcOptionalOptions = + choice [ pRpcOptions; eostm_ws |>> (fun _ -> List.empty) ] pipe4 (str_ws1 "rpc" >>. pRpcName_ws) pStrMessageType (str_ws "returns" >>. pStrMessageType) - (opt pRpcOptions |>> defArg []) + pRpcOptionalOptions (fun ident (bsReq,req) (bsResp,resp) opts -> TRpc( ident, req, bsReq, resp, bsResp, opts) ) - .>> eostm - /// Parse protobuf: proto2: syntax | import | package | option | message | enum | extend | service | emptyStatement /// Parse protobuf: proto2: syntax | import | package | option | message | enum | service | emptyStatement diff --git a/TypeProvider/Generation/TypeGen.fs b/TypeProvider/Generation/TypeGen.fs index 6db4024..860eef3 100644 --- a/TypeProvider/Generation/TypeGen.fs +++ b/TypeProvider/Generation/TypeGen.fs @@ -185,7 +185,7 @@ let rec createType scope (lookup: TypesLookup) (message: ProtoMessage) = message.Parts |> Seq.choose (fun x -> match x with - | TMap(name, keyTy, valueTy, position) -> Some <| createMap nestedScope lookup name keyTy valueTy (int position) + | TMap(name, keyTy, valueTy, position, _) -> Some <| createMap nestedScope lookup name keyTy valueTy (int position) | _ -> None) |> List.ofSeq diff --git a/test/FooOptions.proto b/test/FooOptions.proto index 4a04cc5..dec796f 100644 --- a/test/FooOptions.proto +++ b/test/FooOptions.proto @@ -11,5 +11,5 @@ extend google.protobuf.FieldOptions { message Bar { optional int32 a = 1 [(foo_options.opt1) = 123, (foo_options.opt2) = "baz"]; // alternative aggregate syntax (uses TextFormat): - //optional int32 b = 2 [(foo_options) = { opt1: 123 opt2: "baz" }]; + optional int32 b = 2 [(foo_options) = { opt1: 123 opt2: "baz" }]; } \ No newline at end of file diff --git a/test/grpc.proto b/test/grpc.proto new file mode 100644 index 0000000..cbdf147 --- /dev/null +++ b/test/grpc.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package google.pubsub.v1; + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_outer_classname = "PubsubProto"; +option java_package = "com.google.pubsub.v1"; + +option go_package = "google.golang.org/genproto/googleapis/pubsub/v1"; + +// The service that an application uses to manipulate subscriptions and to +// consume messages from a subscription via the `Pull` method. +service Subscriber { + // Creates a subscription to a given topic for a given subscriber. + // If the subscription already exists, returns `ALREADY_EXISTS`. + // If the corresponding topic doesn't exist, returns `NOT_FOUND`. + // + // If the name is not provided in the request, the server will assign a random + // name for this subscription on the same project as the topic. + rpc CreateSubscription(Subscription) returns (Subscription) { + option (google.api.http) = { put: "/v1/{name=projects/*/subscriptions/*}" body: "*" }; + } + + // Gets the configuration details of a subscription. + rpc GetSubscription(GetSubscriptionRequest) returns (Subscription) { + option (google.api.http) = { get: "/v1/{subscription=projects/*/subscriptions/*}" }; + } + + // Lists matching subscriptions. + rpc ListSubscriptions(ListSubscriptionsRequest) returns (ListSubscriptionsResponse) { + option (google.api.http) = { get: "/v1/{project=projects/*}/subscriptions" }; + } + + // Deletes an existing subscription. All pending messages in the subscription + // are immediately dropped. Calls to `Pull` after deletion will return + // `NOT_FOUND`. After a subscription is deleted, a new one may be created with + // the same name, but the new one has no association with the old + // subscription, or its topic unless the same topic is specified. + rpc DeleteSubscription(DeleteSubscriptionRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { delete: "/v1/{subscription=projects/*/subscriptions/*}" }; + } + + // Modifies the ack deadline for a specific message. This method is useful + // to indicate that more time is needed to process a message by the + // subscriber, or to make the message available for redelivery if the + // processing was interrupted. + rpc ModifyAckDeadline(ModifyAckDeadlineRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{subscription=projects/*/subscriptions/*}:modifyAckDeadline" body: "*" }; + } + + // Acknowledges the messages associated with the `ack_ids` in the + // `AcknowledgeRequest`. The Pub/Sub system can remove the relevant messages + // from the subscription. + // + // Acknowledging a message whose ack deadline has expired may succeed, + // but such a message may be redelivered later. Acknowledging a message more + // than once will not result in an error. + rpc Acknowledge(AcknowledgeRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{subscription=projects/*/subscriptions/*}:acknowledge" body: "*" }; + } + + // Pulls messages from the server. Returns an empty list if there are no + // messages available in the backlog. The server may return `UNAVAILABLE` if + // there are too many concurrent pull requests pending for the given + // subscription. + rpc Pull(PullRequest) returns (PullResponse) { + option (google.api.http) = { post: "/v1/{subscription=projects/*/subscriptions/*}:pull" body: "*" }; + } + + // Modifies the `PushConfig` for a specified subscription. + // + // This may be used to change a push subscription to a pull one (signified by + // an empty `PushConfig`) or vice versa, or change the endpoint URL and other + // attributes of a push subscription. Messages will accumulate for delivery + // continuously through the call regardless of changes to the `PushConfig`. + rpc ModifyPushConfig(ModifyPushConfigRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { post: "/v1/{subscription=projects/*/subscriptions/*}:modifyPushConfig" body: "*" }; + } +} \ No newline at end of file