diff --git a/src/DocumentStore.zig b/src/DocumentStore.zig index d68df7cf49..9c171fd736 100644 --- a/src/DocumentStore.zig +++ b/src/DocumentStore.zig @@ -13,6 +13,7 @@ const Config = @import("Config.zig"); const ZigVersionWrapper = @import("ZigVersionWrapper.zig"); const translate_c = @import("translate_c.zig"); const ComptimeInterpreter = @import("ComptimeInterpreter.zig"); +const Parser = @import("stage2/Ast.zig"); const AstGen = @import("stage2/AstGen.zig"); const Zir = @import("stage2/Zir.zig"); const InternPool = @import("analyser/InternPool.zig"); @@ -754,8 +755,16 @@ fn createDocument(self: *DocumentStore, uri: Uri, text: [:0]const u8, open: bool var duped_uri = try self.allocator.dupe(u8, uri); errdefer self.allocator.free(duped_uri); - var tree = try Ast.parse(self.allocator, text, .zig); - errdefer tree.deinit(self.allocator); + // var tree = try Ast.parse(self.allocator, text, .zig); + var zls_ast = try Parser.parse(self.allocator, text, .zig); + errdefer zls_ast.deinit(self.allocator); + var tree = Ast{ + .source = zls_ast.source, + .tokens = zls_ast.tokens, + .nodes = zls_ast.nodes, + .extra_data = zls_ast.extra_data, + .errors = zls_ast.errors, + }; // remove unused capacity var nodes = tree.nodes.toMultiArrayList(); diff --git a/src/stage2/Ast.zig b/src/stage2/Ast.zig new file mode 100644 index 0000000000..4c98918111 --- /dev/null +++ b/src/stage2/Ast.zig @@ -0,0 +1,3533 @@ +//! Abstract Syntax Tree for Zig source code. +//! For Zig syntax, the root node is at nodes[0] and contains the list of +//! sub-nodes. +//! For Zon syntax, the root node is at nodes[0] and contains lhs as the node +//! index of the main expression. + +/// Reference to externally-owned data. +source: [:0]const u8, + +tokens: std.zig.Ast.TokenList.Slice, //TokenList.Slice, +/// The root AST node is assumed to be index 0. Since there can be no +/// references to the root node, this means 0 is available to indicate null. +nodes: std.zig.Ast.NodeList.Slice, +extra_data: []Node.Index, + +errors: []const std.zig.Ast.Error, + +pub const TokenIndex = u32; +pub const ByteOffset = u32; + +// pub const TokenList = std.MultiArrayList(struct { +// tag: Token.Tag, +// start: ByteOffset, +// }); +// pub const NodeList = std.MultiArrayList(Node); + +pub const Location = struct { + line: usize, + column: usize, + line_start: usize, + line_end: usize, +}; + +pub fn deinit(tree: *Ast, gpa: Allocator) void { + tree.tokens.deinit(gpa); + tree.nodes.deinit(gpa); + gpa.free(tree.extra_data); + gpa.free(tree.errors); + tree.* = undefined; +} + +pub const RenderError = error{ + /// Ran out of memory allocating call stack frames to complete rendering, or + /// ran out of memory allocating space in the output buffer. + OutOfMemory, +}; + +pub const Mode = enum { zig, zon }; + +/// Result should be freed with tree.deinit() when there are +/// no more references to any of the tokens or nodes. +pub fn parse(gpa: Allocator, source: [:0]const u8, mode: Mode) Allocator.Error!Ast { + var tokens = std.zig.Ast.TokenList{}; + defer tokens.deinit(gpa); + + // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count. + const estimated_token_count = source.len / 8; + try tokens.ensureTotalCapacity(gpa, estimated_token_count); + + var tokenizer = std.zig.Tokenizer.init(source); + while (true) { + const token = tokenizer.next(); + try tokens.append(gpa, .{ + .tag = token.tag, + .start = @as(u32, @intCast(token.loc.start)), + }); + if (token.tag == .eof) break; + } + + var parser: Parse = .{ + .source = source, + .gpa = gpa, + .token_tags = tokens.items(.tag), + .token_starts = tokens.items(.start), + .errors = .{}, + .nodes = .{}, + .extra_data = .{}, + .scratch = .{}, + .tok_i = 0, + }; + defer parser.errors.deinit(gpa); + defer parser.nodes.deinit(gpa); + defer parser.extra_data.deinit(gpa); + defer parser.scratch.deinit(gpa); + + // Empirically, Zig source code has a 2:1 ratio of tokens to AST nodes. + // Make sure at least 1 so we can use appendAssumeCapacity on the root node below. + const estimated_node_count = (tokens.len + 2) / 2; + try parser.nodes.ensureTotalCapacity(gpa, estimated_node_count); + + switch (mode) { + .zig => try parser.parseRoot(), + .zon => try parser.parseZon(), + } + + // TODO experiment with compacting the MultiArrayList slices here + return Ast{ + .source = source, + .tokens = tokens.toOwnedSlice(), + .nodes = parser.nodes.toOwnedSlice(), + .extra_data = try parser.extra_data.toOwnedSlice(gpa), + .errors = try parser.errors.toOwnedSlice(gpa), + }; +} + +/// `gpa` is used for allocating the resulting formatted source code, as well as +/// for allocating extra stack memory if needed, because this function utilizes recursion. +/// Note: that's not actually true yet, see https://github.com/ziglang/zig/issues/1006. +/// Caller owns the returned slice of bytes, allocated with `gpa`. +pub fn render(tree: Ast, gpa: Allocator) RenderError![]u8 { + var buffer = std.ArrayList(u8).init(gpa); + defer buffer.deinit(); + + try tree.renderToArrayList(&buffer); + return buffer.toOwnedSlice(); +} + +pub fn renderToArrayList(tree: Ast, buffer: *std.ArrayList(u8)) RenderError!void { + return @import("./render.zig").renderTree(buffer, tree); +} + +/// Returns an extra offset for column and byte offset of errors that +/// should point after the token in the error message. +pub fn errorOffset(tree: Ast, parse_error: Error) u32 { + return if (parse_error.token_is_prev) + @as(u32, @intCast(tree.tokenSlice(parse_error.token).len)) + else + 0; +} + +pub fn tokenLocation(self: Ast, start_offset: ByteOffset, token_index: TokenIndex) Location { + var loc = Location{ + .line = 0, + .column = 0, + .line_start = start_offset, + .line_end = self.source.len, + }; + const token_start = self.tokens.items(.start)[token_index]; + for (self.source[start_offset..], 0..) |c, i| { + if (i + start_offset == token_start) { + loc.line_end = i + start_offset; + while (loc.line_end < self.source.len and self.source[loc.line_end] != '\n') { + loc.line_end += 1; + } + return loc; + } + if (c == '\n') { + loc.line += 1; + loc.column = 0; + loc.line_start = i + 1; + } else { + loc.column += 1; + } + } + return loc; +} + +pub fn tokenSlice(tree: Ast, token_index: TokenIndex) []const u8 { + const token_starts = tree.tokens.items(.start); + const token_tags = tree.tokens.items(.tag); + const token_tag = token_tags[token_index]; + + // Many tokens can be determined entirely by their tag. + if (token_tag.lexeme()) |lexeme| { + return lexeme; + } + + // For some tokens, re-tokenization is needed to find the end. + var tokenizer: std.zig.Tokenizer = .{ + .buffer = tree.source, + .index = token_starts[token_index], + .pending_invalid_token = null, + }; + const token = tokenizer.findTagAtCurrentIndex(token_tag); + assert(token.tag == token_tag); + return tree.source[token.loc.start..token.loc.end]; +} + +pub fn extraData(tree: Ast, index: usize, comptime T: type) T { + const fields = std.meta.fields(T); + var result: T = undefined; + inline for (fields, 0..) |field, i| { + comptime assert(field.type == Node.Index); + @field(result, field.name) = tree.extra_data[index + i]; + } + return result; +} + +pub fn rootDecls(tree: Ast) []const Node.Index { + // Root is always index 0. + const nodes_data = tree.nodes.items(.data); + return tree.extra_data[nodes_data[0].lhs..nodes_data[0].rhs]; +} + +pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void { + const token_tags = tree.tokens.items(.tag); + switch (parse_error.tag) { + .asterisk_after_ptr_deref => { + // Note that the token will point at the `.*` but ideally the source + // location would point to the `*` after the `.*`. + return stream.writeAll("'.*' cannot be followed by '*'. Are you missing a space?"); + }, + .chained_comparison_operators => { + return stream.writeAll("comparison operators cannot be chained"); + }, + .decl_between_fields => { + return stream.writeAll("declarations are not allowed between container fields"); + }, + .expected_block => { + return stream.print("expected block, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_block_or_assignment => { + return stream.print("expected block or assignment, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_block_or_expr => { + return stream.print("expected block or expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_block_or_field => { + return stream.print("expected block or field, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_container_members => { + return stream.print("expected test, comptime, var decl, or container field, found '{s}'", .{ + token_tags[parse_error.token].symbol(), + }); + }, + .expected_expr => { + return stream.print("expected expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_expr_or_assignment => { + return stream.print("expected expression or assignment, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_expr_or_var_decl => { + return stream.print("expected expression or var decl, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_fn => { + return stream.print("expected function, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_inlinable => { + return stream.print("expected 'while' or 'for', found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_labelable => { + return stream.print("expected 'while', 'for', 'inline', or '{{', found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_param_list => { + return stream.print("expected parameter list, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_prefix_expr => { + return stream.print("expected prefix expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_primary_type_expr => { + return stream.print("expected primary type expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_pub_item => { + return stream.writeAll("expected function or variable declaration after pub"); + }, + .expected_return_type => { + return stream.print("expected return type expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_semi_or_else => { + return stream.writeAll("expected ';' or 'else' after statement"); + }, + .expected_semi_or_lbrace => { + return stream.writeAll("expected ';' or block after function prototype"); + }, + .expected_statement => { + return stream.print("expected statement, found '{s}'", .{ + token_tags[parse_error.token].symbol(), + }); + }, + .expected_suffix_op => { + return stream.print("expected pointer dereference, optional unwrap, or field access, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_type_expr => { + return stream.print("expected type expression, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_var_decl => { + return stream.print("expected variable declaration, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_var_decl_or_fn => { + return stream.print("expected variable declaration or function, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_loop_payload => { + return stream.print("expected loop payload, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .expected_container => { + return stream.print("expected a struct, enum or union, found '{s}'", .{ + token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)].symbol(), + }); + }, + .extern_fn_body => { + return stream.writeAll("extern functions have no body"); + }, + .extra_addrspace_qualifier => { + return stream.writeAll("extra addrspace qualifier"); + }, + .extra_align_qualifier => { + return stream.writeAll("extra align qualifier"); + }, + .extra_allowzero_qualifier => { + return stream.writeAll("extra allowzero qualifier"); + }, + .extra_const_qualifier => { + return stream.writeAll("extra const qualifier"); + }, + .extra_volatile_qualifier => { + return stream.writeAll("extra volatile qualifier"); + }, + .ptr_mod_on_array_child_type => { + return stream.print("pointer modifier '{s}' not allowed on array child type", .{ + token_tags[parse_error.token].symbol(), + }); + }, + .invalid_bit_range => { + return stream.writeAll("bit range not allowed on slices and arrays"); + }, + .same_line_doc_comment => { + return stream.writeAll("same line documentation comment"); + }, + .unattached_doc_comment => { + return stream.writeAll("unattached documentation comment"); + }, + .test_doc_comment => { + return stream.writeAll("documentation comments cannot be attached to tests"); + }, + .comptime_doc_comment => { + return stream.writeAll("documentation comments cannot be attached to comptime blocks"); + }, + .varargs_nonfinal => { + return stream.writeAll("function prototype has parameter after varargs"); + }, + .expected_continue_expr => { + return stream.writeAll("expected ':' before while continue expression"); + }, + + .expected_semi_after_decl => { + return stream.writeAll("expected ';' after declaration"); + }, + .expected_semi_after_stmt => { + return stream.writeAll("expected ';' after statement"); + }, + .expected_comma_after_field => { + return stream.writeAll("expected ',' after field"); + }, + .expected_comma_after_arg => { + return stream.writeAll("expected ',' after argument"); + }, + .expected_comma_after_param => { + return stream.writeAll("expected ',' after parameter"); + }, + .expected_comma_after_initializer => { + return stream.writeAll("expected ',' after initializer"); + }, + .expected_comma_after_switch_prong => { + return stream.writeAll("expected ',' after switch prong"); + }, + .expected_comma_after_for_operand => { + return stream.writeAll("expected ',' after for operand"); + }, + .expected_comma_after_capture => { + return stream.writeAll("expected ',' after for capture"); + }, + .expected_initializer => { + return stream.writeAll("expected field initializer"); + }, + .mismatched_binary_op_whitespace => { + return stream.print("binary operator `{s}` has whitespace on one side, but not the other.", .{token_tags[parse_error.token].lexeme().?}); + }, + .invalid_ampersand_ampersand => { + return stream.writeAll("ambiguous use of '&&'; use 'and' for logical AND, or change whitespace to ' & &' for bitwise AND"); + }, + .c_style_container => { + return stream.print("'{s} {s}' is invalid", .{ + parse_error.extra.expected_tag.symbol(), tree.tokenSlice(parse_error.token), + }); + }, + .zig_style_container => { + return stream.print("to declare a container do 'const {s} = {s}'", .{ + tree.tokenSlice(parse_error.token), parse_error.extra.expected_tag.symbol(), + }); + }, + .previous_field => { + return stream.writeAll("field before declarations here"); + }, + .next_field => { + return stream.writeAll("field after declarations here"); + }, + .expected_var_const => { + return stream.writeAll("expected 'var' or 'const' before variable declaration"); + }, + .wrong_equal_var_decl => { + return stream.writeAll("variable initialized with '==' instead of '='"); + }, + .var_const_decl => { + return stream.writeAll("use 'var' or 'const' to declare variable"); + }, + .extra_for_capture => { + return stream.writeAll("extra capture in for loop"); + }, + .for_input_not_captured => { + return stream.writeAll("for input is not captured"); + }, + + .expected_token => { + const found_tag = token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)]; + const expected_symbol = parse_error.extra.expected_tag.symbol(); + switch (found_tag) { + .invalid => return stream.print("expected '{s}', found invalid bytes", .{ + expected_symbol, + }), + else => return stream.print("expected '{s}', found '{s}'", .{ + expected_symbol, found_tag.symbol(), + }), + } + }, + } +} + +pub fn firstToken(tree: Ast, node: Node.Index) TokenIndex { + const tags = tree.nodes.items(.tag); + const datas = tree.nodes.items(.data); + const main_tokens = tree.nodes.items(.main_token); + const token_tags = tree.tokens.items(.tag); + var end_offset: TokenIndex = 0; + var n = node; + while (true) switch (tags[n]) { + .root => return 0, + + .test_decl, + .@"errdefer", + .@"defer", + .bool_not, + .negation, + .bit_not, + .negation_wrap, + .address_of, + .@"try", + .@"await", + .optional_type, + .@"switch", + .switch_comma, + .if_simple, + .@"if", + .@"suspend", + .@"resume", + .@"continue", + .@"break", + .@"return", + .anyframe_type, + .identifier, + .anyframe_literal, + .char_literal, + .number_literal, + .unreachable_literal, + .string_literal, + .multiline_string_literal, + .grouped_expression, + .builtin_call_two, + .builtin_call_two_comma, + .builtin_call, + .builtin_call_comma, + .error_set_decl, + .@"comptime", + .@"nosuspend", + .asm_simple, + .@"asm", + .array_type, + .array_type_sentinel, + .error_value, + => return main_tokens[n] - end_offset, + + .array_init_dot, + .array_init_dot_comma, + .array_init_dot_two, + .array_init_dot_two_comma, + .struct_init_dot, + .struct_init_dot_comma, + .struct_init_dot_two, + .struct_init_dot_two_comma, + .enum_literal, + => return main_tokens[n] - 1 - end_offset, + + .@"catch", + .field_access, + .unwrap_optional, + .equal_equal, + .bang_equal, + .less_than, + .greater_than, + .less_or_equal, + .greater_or_equal, + .assign_mul, + .assign_div, + .assign_mod, + .assign_add, + .assign_sub, + .assign_shl, + .assign_shl_sat, + .assign_shr, + .assign_bit_and, + .assign_bit_xor, + .assign_bit_or, + .assign_mul_wrap, + .assign_add_wrap, + .assign_sub_wrap, + .assign_mul_sat, + .assign_add_sat, + .assign_sub_sat, + .assign, + .merge_error_sets, + .mul, + .div, + .mod, + .array_mult, + .mul_wrap, + .mul_sat, + .add, + .sub, + .array_cat, + .add_wrap, + .sub_wrap, + .add_sat, + .sub_sat, + .shl, + .shl_sat, + .shr, + .bit_and, + .bit_xor, + .bit_or, + .@"orelse", + .bool_and, + .bool_or, + .slice_open, + .slice, + .slice_sentinel, + .deref, + .array_access, + .array_init_one, + .array_init_one_comma, + .array_init, + .array_init_comma, + .struct_init_one, + .struct_init_one_comma, + .struct_init, + .struct_init_comma, + .call_one, + .call_one_comma, + .call, + .call_comma, + .switch_range, + .for_range, + .error_union, + => n = datas[n].lhs, + + .assign_destructure => { + const extra_idx = datas[n].lhs; + const lhs_len = tree.extra_data[extra_idx]; + assert(lhs_len > 0); + n = tree.extra_data[extra_idx + 1]; + }, + + .fn_decl, + .fn_proto_simple, + .fn_proto_multi, + .fn_proto_one, + .fn_proto, + => { + var i = main_tokens[n]; // fn token + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, + .keyword_export, + .keyword_pub, + .keyword_inline, + .keyword_noinline, + .string_literal, + => continue, + + else => return i + 1 - end_offset, + } + } + return i - end_offset; + }, + + .@"usingnamespace" => { + const main_token = main_tokens[n]; + if (main_token > 0 and token_tags[main_token - 1] == .keyword_pub) { + end_offset += 1; + } + return main_token - end_offset; + }, + + .async_call_one, + .async_call_one_comma, + .async_call, + .async_call_comma, + => { + end_offset += 1; // async token + n = datas[n].lhs; + }, + + .container_field_init, + .container_field_align, + .container_field, + => { + const name_token = main_tokens[n]; + if (token_tags[name_token] != .keyword_comptime and name_token > 0 and token_tags[name_token - 1] == .keyword_comptime) { + end_offset += 1; + } + return name_token - end_offset; + }, + + .global_var_decl, + .local_var_decl, + .simple_var_decl, + .aligned_var_decl, + => { + var i = main_tokens[n]; // mut token + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, + .keyword_export, + .keyword_comptime, + .keyword_pub, + .keyword_threadlocal, + .string_literal, + => continue, + + else => return i + 1 - end_offset, + } + } + return i - end_offset; + }, + + .block, + .block_semicolon, + .block_two, + .block_two_semicolon, + => { + // Look for a label. + const lbrace = main_tokens[n]; + if (token_tags[lbrace - 1] == .colon and + token_tags[lbrace - 2] == .identifier) + { + end_offset += 2; + } + return lbrace - end_offset; + }, + + .container_decl, + .container_decl_trailing, + .container_decl_two, + .container_decl_two_trailing, + .container_decl_arg, + .container_decl_arg_trailing, + .tagged_union, + .tagged_union_trailing, + .tagged_union_two, + .tagged_union_two_trailing, + .tagged_union_enum_tag, + .tagged_union_enum_tag_trailing, + => { + const main_token = main_tokens[n]; + switch (token_tags[main_token -| 1]) { + .keyword_packed, .keyword_extern => end_offset += 1, + else => {}, + } + return main_token - end_offset; + }, + + .ptr_type_aligned, + .ptr_type_sentinel, + .ptr_type, + .ptr_type_bit_range, + => { + const main_token = main_tokens[n]; + return switch (token_tags[main_token]) { + .asterisk, + .asterisk_asterisk, + => switch (token_tags[main_token -| 1]) { + .l_bracket => main_token -| 1, + else => main_token, + }, + .l_bracket => main_token, + else => unreachable, + } - end_offset; + }, + + .switch_case_one => { + if (datas[n].lhs == 0) { + return main_tokens[n] - 1 - end_offset; // else token + } else { + n = datas[n].lhs; + } + }, + .switch_case_inline_one => { + if (datas[n].lhs == 0) { + return main_tokens[n] - 2 - end_offset; // else token + } else { + return firstToken(tree, datas[n].lhs) - 1; + } + }, + .switch_case => { + const extra = tree.extraData(datas[n].lhs, Node.SubRange); + assert(extra.end - extra.start > 0); + n = tree.extra_data[extra.start]; + }, + .switch_case_inline => { + const extra = tree.extraData(datas[n].lhs, Node.SubRange); + assert(extra.end - extra.start > 0); + return firstToken(tree, tree.extra_data[extra.start]) - 1; + }, + + .asm_output, .asm_input => { + assert(token_tags[main_tokens[n] - 1] == .l_bracket); + return main_tokens[n] - 1 - end_offset; + }, + + .while_simple, + .while_cont, + .@"while", + .for_simple, + .@"for", + => { + // Look for a label and inline. + const main_token = main_tokens[n]; + var result = main_token; + if (token_tags[result -| 1] == .keyword_inline) { + result -= 1; + } + if (token_tags[result -| 1] == .colon) { + result -|= 2; + } + return result - end_offset; + }, + }; +} + +pub fn lastToken(tree: Ast, node: Node.Index) TokenIndex { + const tags = tree.nodes.items(.tag); + const datas = tree.nodes.items(.data); + const main_tokens = tree.nodes.items(.main_token); + const token_starts = tree.tokens.items(.start); + const token_tags = tree.tokens.items(.tag); + var n = node; + var end_offset: TokenIndex = 0; + while (true) switch (tags[n]) { + .root => return @as(TokenIndex, @intCast(tree.tokens.len - 1)), + + .@"usingnamespace", + .bool_not, + .negation, + .bit_not, + .negation_wrap, + .address_of, + .@"try", + .@"await", + .optional_type, + .@"resume", + .@"nosuspend", + .@"comptime", + => n = datas[n].lhs, + + .test_decl, + .@"errdefer", + .@"defer", + .@"catch", + .equal_equal, + .bang_equal, + .less_than, + .greater_than, + .less_or_equal, + .greater_or_equal, + .assign_mul, + .assign_div, + .assign_mod, + .assign_add, + .assign_sub, + .assign_shl, + .assign_shl_sat, + .assign_shr, + .assign_bit_and, + .assign_bit_xor, + .assign_bit_or, + .assign_mul_wrap, + .assign_add_wrap, + .assign_sub_wrap, + .assign_mul_sat, + .assign_add_sat, + .assign_sub_sat, + .assign, + .assign_destructure, + .merge_error_sets, + .mul, + .div, + .mod, + .array_mult, + .mul_wrap, + .mul_sat, + .add, + .sub, + .array_cat, + .add_wrap, + .sub_wrap, + .add_sat, + .sub_sat, + .shl, + .shl_sat, + .shr, + .bit_and, + .bit_xor, + .bit_or, + .@"orelse", + .bool_and, + .bool_or, + .anyframe_type, + .error_union, + .if_simple, + .while_simple, + .for_simple, + .fn_proto_simple, + .fn_proto_multi, + .ptr_type_aligned, + .ptr_type_sentinel, + .ptr_type, + .ptr_type_bit_range, + .array_type, + .switch_case_one, + .switch_case_inline_one, + .switch_case, + .switch_case_inline, + .switch_range, + => n = datas[n].rhs, + + .for_range => if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else { + return main_tokens[n] + end_offset; + }, + + .field_access, + .unwrap_optional, + .grouped_expression, + .multiline_string_literal, + .error_set_decl, + .asm_simple, + .asm_output, + .asm_input, + .error_value, + => return datas[n].rhs + end_offset, + + .anyframe_literal, + .char_literal, + .number_literal, + .unreachable_literal, + .identifier, + .deref, + .enum_literal, + .string_literal, + => return main_tokens[n] + end_offset, + + .@"return" => if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + return main_tokens[n] + end_offset; + }, + + .call, .async_call => { + end_offset += 1; // for the rparen + const params = tree.extraData(datas[n].rhs, Node.SubRange); + if (params.end - params.start == 0) { + return main_tokens[n] + end_offset; + } + n = tree.extra_data[params.end - 1]; // last parameter + }, + .tagged_union_enum_tag => { + const members = tree.extraData(datas[n].rhs, Node.SubRange); + if (members.end - members.start == 0) { + end_offset += 4; // for the rparen + rparen + lbrace + rbrace + n = datas[n].lhs; + } else { + end_offset += 1; // for the rbrace + n = tree.extra_data[members.end - 1]; // last parameter + } + }, + .call_comma, + .async_call_comma, + .tagged_union_enum_tag_trailing, + => { + end_offset += 2; // for the comma/semicolon + rparen/rbrace + const params = tree.extraData(datas[n].rhs, Node.SubRange); + assert(params.end > params.start); + n = tree.extra_data[params.end - 1]; // last parameter + }, + .@"switch" => { + const cases = tree.extraData(datas[n].rhs, Node.SubRange); + if (cases.end - cases.start == 0) { + end_offset += 3; // rparen, lbrace, rbrace + n = datas[n].lhs; // condition expression + } else { + end_offset += 1; // for the rbrace + n = tree.extra_data[cases.end - 1]; // last case + } + }, + .container_decl_arg => { + const members = tree.extraData(datas[n].rhs, Node.SubRange); + if (members.end - members.start == 0) { + end_offset += 3; // for the rparen + lbrace + rbrace + n = datas[n].lhs; + } else { + end_offset += 1; // for the rbrace + n = tree.extra_data[members.end - 1]; // last parameter + } + }, + .@"asm" => { + const extra = tree.extraData(datas[n].rhs, Node.Asm); + return extra.rparen + end_offset; + }, + .array_init, + .struct_init, + => { + const elements = tree.extraData(datas[n].rhs, Node.SubRange); + assert(elements.end - elements.start > 0); + end_offset += 1; // for the rbrace + n = tree.extra_data[elements.end - 1]; // last element + }, + .array_init_comma, + .struct_init_comma, + .container_decl_arg_trailing, + .switch_comma, + => { + const members = tree.extraData(datas[n].rhs, Node.SubRange); + assert(members.end - members.start > 0); + end_offset += 2; // for the comma + rbrace + n = tree.extra_data[members.end - 1]; // last parameter + }, + .array_init_dot, + .struct_init_dot, + .block, + .container_decl, + .tagged_union, + .builtin_call, + => { + assert(datas[n].rhs - datas[n].lhs > 0); + end_offset += 1; // for the rbrace + n = tree.extra_data[datas[n].rhs - 1]; // last statement + }, + .array_init_dot_comma, + .struct_init_dot_comma, + .block_semicolon, + .container_decl_trailing, + .tagged_union_trailing, + .builtin_call_comma, + => { + assert(datas[n].rhs - datas[n].lhs > 0); + end_offset += 2; // for the comma/semicolon + rbrace/rparen + n = tree.extra_data[datas[n].rhs - 1]; // last member + }, + .call_one, + .async_call_one, + .array_access, + => { + end_offset += 1; // for the rparen/rbracket + if (datas[n].rhs == 0) { + return main_tokens[n] + end_offset; + } + n = datas[n].rhs; + }, + .array_init_dot_two, + .block_two, + .builtin_call_two, + .struct_init_dot_two, + .container_decl_two, + .tagged_union_two, + => { + if (datas[n].rhs != 0) { + end_offset += 1; // for the rparen/rbrace + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + end_offset += 1; // for the rparen/rbrace + n = datas[n].lhs; + } else { + switch (tags[n]) { + .array_init_dot_two, + .block_two, + .struct_init_dot_two, + => end_offset += 1, // rbrace + .builtin_call_two => end_offset += 2, // lparen/lbrace + rparen/rbrace + .container_decl_two => { + var i: u32 = 2; // lbrace + rbrace + while (token_tags[main_tokens[n] + i] == .container_doc_comment) i += 1; + end_offset += i; + }, + .tagged_union_two => { + var i: u32 = 5; // (enum) {} + while (token_tags[main_tokens[n] + i] == .container_doc_comment) i += 1; + end_offset += i; + }, + else => unreachable, + } + return main_tokens[n] + end_offset; + } + }, + .array_init_dot_two_comma, + .builtin_call_two_comma, + .block_two_semicolon, + .struct_init_dot_two_comma, + .container_decl_two_trailing, + .tagged_union_two_trailing, + => { + end_offset += 2; // for the comma/semicolon + rbrace/rparen + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + unreachable; + } + }, + .simple_var_decl => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + end_offset += 1; // from mut token to name + return main_tokens[n] + end_offset; + } + }, + .aligned_var_decl => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + end_offset += 1; // for the rparen + n = datas[n].lhs; + } else { + end_offset += 1; // from mut token to name + return main_tokens[n] + end_offset; + } + }, + .global_var_decl => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else { + const extra = tree.extraData(datas[n].lhs, Node.GlobalVarDecl); + if (extra.section_node != 0) { + end_offset += 1; // for the rparen + n = extra.section_node; + } else if (extra.align_node != 0) { + end_offset += 1; // for the rparen + n = extra.align_node; + } else if (extra.type_node != 0) { + n = extra.type_node; + } else { + end_offset += 1; // from mut token to name + return main_tokens[n] + end_offset; + } + } + }, + .local_var_decl => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else { + const extra = tree.extraData(datas[n].lhs, Node.LocalVarDecl); + if (extra.align_node != 0) { + end_offset += 1; // for the rparen + n = extra.align_node; + } else if (extra.type_node != 0) { + n = extra.type_node; + } else { + end_offset += 1; // from mut token to name + return main_tokens[n] + end_offset; + } + } + }, + .container_field_init => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + return main_tokens[n] + end_offset; + } + }, + .container_field_align => { + if (datas[n].rhs != 0) { + end_offset += 1; // for the rparen + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + return main_tokens[n] + end_offset; + } + }, + .container_field => { + const extra = tree.extraData(datas[n].rhs, Node.ContainerField); + if (extra.value_expr != 0) { + n = extra.value_expr; + } else if (extra.align_expr != 0) { + end_offset += 1; // for the rparen + n = extra.align_expr; + } else if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + return main_tokens[n] + end_offset; + } + }, + + .array_init_one, + .struct_init_one, + => { + end_offset += 1; // rbrace + if (datas[n].rhs == 0) { + return main_tokens[n] + end_offset; + } else { + n = datas[n].rhs; + } + }, + .slice_open, + .call_one_comma, + .async_call_one_comma, + .array_init_one_comma, + .struct_init_one_comma, + => { + end_offset += 2; // ellipsis2 + rbracket, or comma + rparen + n = datas[n].rhs; + assert(n != 0); + }, + .slice => { + const extra = tree.extraData(datas[n].rhs, Node.Slice); + assert(extra.end != 0); // should have used slice_open + end_offset += 1; // rbracket + n = extra.end; + }, + .slice_sentinel => { + const extra = tree.extraData(datas[n].rhs, Node.SliceSentinel); + assert(extra.sentinel != 0); // should have used slice + end_offset += 1; // rbracket + n = extra.sentinel; + }, + + .@"continue" => { + if (datas[n].lhs != 0) { + return datas[n].lhs + end_offset; + } else { + return main_tokens[n] + end_offset; + } + }, + .@"break" => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else if (datas[n].lhs != 0) { + return datas[n].lhs + end_offset; + } else { + return main_tokens[n] + end_offset; + } + }, + .fn_decl => { + if (datas[n].rhs != 0) { + n = datas[n].rhs; + } else { + n = datas[n].lhs; + } + }, + .fn_proto_one => { + const extra = tree.extraData(datas[n].lhs, Node.FnProtoOne); + // addrspace, linksection, callconv, align can appear in any order, so we + // find the last one here. + var max_node: Node.Index = datas[n].rhs; + var max_start = token_starts[main_tokens[max_node]]; + var max_offset: TokenIndex = 0; + if (extra.align_expr != 0) { + const start = token_starts[main_tokens[extra.align_expr]]; + if (start > max_start) { + max_node = extra.align_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.addrspace_expr != 0) { + const start = token_starts[main_tokens[extra.addrspace_expr]]; + if (start > max_start) { + max_node = extra.addrspace_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.section_expr != 0) { + const start = token_starts[main_tokens[extra.section_expr]]; + if (start > max_start) { + max_node = extra.section_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.callconv_expr != 0) { + const start = token_starts[main_tokens[extra.callconv_expr]]; + if (start > max_start) { + max_node = extra.callconv_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + n = max_node; + end_offset += max_offset; + }, + .fn_proto => { + const extra = tree.extraData(datas[n].lhs, Node.FnProto); + // addrspace, linksection, callconv, align can appear in any order, so we + // find the last one here. + var max_node: Node.Index = datas[n].rhs; + var max_start = token_starts[main_tokens[max_node]]; + var max_offset: TokenIndex = 0; + if (extra.align_expr != 0) { + const start = token_starts[main_tokens[extra.align_expr]]; + if (start > max_start) { + max_node = extra.align_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.addrspace_expr != 0) { + const start = token_starts[main_tokens[extra.addrspace_expr]]; + if (start > max_start) { + max_node = extra.addrspace_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.section_expr != 0) { + const start = token_starts[main_tokens[extra.section_expr]]; + if (start > max_start) { + max_node = extra.section_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + if (extra.callconv_expr != 0) { + const start = token_starts[main_tokens[extra.callconv_expr]]; + if (start > max_start) { + max_node = extra.callconv_expr; + max_start = start; + max_offset = 1; // for the rparen + } + } + n = max_node; + end_offset += max_offset; + }, + .while_cont => { + const extra = tree.extraData(datas[n].rhs, Node.WhileCont); + assert(extra.then_expr != 0); + n = extra.then_expr; + }, + .@"while" => { + const extra = tree.extraData(datas[n].rhs, Node.While); + assert(extra.else_expr != 0); + n = extra.else_expr; + }, + .@"if" => { + const extra = tree.extraData(datas[n].rhs, Node.If); + assert(extra.else_expr != 0); + n = extra.else_expr; + }, + .@"for" => { + const extra = @as(Node.For, @bitCast(datas[n].rhs)); + n = tree.extra_data[datas[n].lhs + extra.inputs + @intFromBool(extra.has_else)]; + }, + .@"suspend" => { + if (datas[n].lhs != 0) { + n = datas[n].lhs; + } else { + return main_tokens[n] + end_offset; + } + }, + .array_type_sentinel => { + const extra = tree.extraData(datas[n].rhs, Node.ArrayTypeSentinel); + n = extra.elem_type; + }, + }; +} + +pub fn tokensOnSameLine(tree: Ast, token1: TokenIndex, token2: TokenIndex) bool { + const token_starts = tree.tokens.items(.start); + const source = tree.source[token_starts[token1]..token_starts[token2]]; + return mem.indexOfScalar(u8, source, '\n') == null; +} + +pub fn getNodeSource(tree: Ast, node: Node.Index) []const u8 { + const token_starts = tree.tokens.items(.start); + const first_token = tree.firstToken(node); + const last_token = tree.lastToken(node); + const start = token_starts[first_token]; + const end = token_starts[last_token] + tree.tokenSlice(last_token).len; + return tree.source[start..end]; +} + +pub fn globalVarDecl(tree: Ast, node: Node.Index) full.VarDecl { + assert(tree.nodes.items(.tag)[node] == .global_var_decl); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.GlobalVarDecl); + return tree.fullVarDeclComponents(.{ + .type_node = extra.type_node, + .align_node = extra.align_node, + .addrspace_node = extra.addrspace_node, + .section_node = extra.section_node, + .init_node = data.rhs, + .mut_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn localVarDecl(tree: Ast, node: Node.Index) full.VarDecl { + assert(tree.nodes.items(.tag)[node] == .local_var_decl); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.LocalVarDecl); + return tree.fullVarDeclComponents(.{ + .type_node = extra.type_node, + .align_node = extra.align_node, + .addrspace_node = 0, + .section_node = 0, + .init_node = data.rhs, + .mut_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn simpleVarDecl(tree: Ast, node: Node.Index) full.VarDecl { + assert(tree.nodes.items(.tag)[node] == .simple_var_decl); + const data = tree.nodes.items(.data)[node]; + return tree.fullVarDeclComponents(.{ + .type_node = data.lhs, + .align_node = 0, + .addrspace_node = 0, + .section_node = 0, + .init_node = data.rhs, + .mut_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn alignedVarDecl(tree: Ast, node: Node.Index) full.VarDecl { + assert(tree.nodes.items(.tag)[node] == .aligned_var_decl); + const data = tree.nodes.items(.data)[node]; + return tree.fullVarDeclComponents(.{ + .type_node = 0, + .align_node = data.lhs, + .addrspace_node = 0, + .section_node = 0, + .init_node = data.rhs, + .mut_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn ifSimple(tree: Ast, node: Node.Index) full.If { + assert(tree.nodes.items(.tag)[node] == .if_simple); + const data = tree.nodes.items(.data)[node]; + return tree.fullIfComponents(.{ + .cond_expr = data.lhs, + .then_expr = data.rhs, + .else_expr = 0, + .if_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn ifFull(tree: Ast, node: Node.Index) full.If { + assert(tree.nodes.items(.tag)[node] == .@"if"); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.If); + return tree.fullIfComponents(.{ + .cond_expr = data.lhs, + .then_expr = extra.then_expr, + .else_expr = extra.else_expr, + .if_token = tree.nodes.items(.main_token)[node], + }); +} + +pub fn containerField(tree: Ast, node: Node.Index) full.ContainerField { + assert(tree.nodes.items(.tag)[node] == .container_field); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.ContainerField); + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerFieldComponents(.{ + .main_token = main_token, + .type_expr = data.lhs, + .value_expr = extra.value_expr, + .align_expr = extra.align_expr, + .tuple_like = tree.tokens.items(.tag)[main_token] != .identifier or + tree.tokens.items(.tag)[main_token + 1] != .colon, + }); +} + +pub fn containerFieldInit(tree: Ast, node: Node.Index) full.ContainerField { + assert(tree.nodes.items(.tag)[node] == .container_field_init); + const data = tree.nodes.items(.data)[node]; + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerFieldComponents(.{ + .main_token = main_token, + .type_expr = data.lhs, + .value_expr = data.rhs, + .align_expr = 0, + .tuple_like = tree.tokens.items(.tag)[main_token] != .identifier or + tree.tokens.items(.tag)[main_token + 1] != .colon, + }); +} + +pub fn containerFieldAlign(tree: Ast, node: Node.Index) full.ContainerField { + assert(tree.nodes.items(.tag)[node] == .container_field_align); + const data = tree.nodes.items(.data)[node]; + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerFieldComponents(.{ + .main_token = main_token, + .type_expr = data.lhs, + .value_expr = 0, + .align_expr = data.rhs, + .tuple_like = tree.tokens.items(.tag)[main_token] != .identifier or + tree.tokens.items(.tag)[main_token + 1] != .colon, + }); +} + +pub fn fnProtoSimple(tree: Ast, buffer: *[1]Node.Index, node: Node.Index) full.FnProto { + assert(tree.nodes.items(.tag)[node] == .fn_proto_simple); + const data = tree.nodes.items(.data)[node]; + buffer[0] = data.lhs; + const params = if (data.lhs == 0) buffer[0..0] else buffer[0..1]; + return tree.fullFnProtoComponents(.{ + .proto_node = node, + .fn_token = tree.nodes.items(.main_token)[node], + .return_type = data.rhs, + .params = params, + .align_expr = 0, + .addrspace_expr = 0, + .section_expr = 0, + .callconv_expr = 0, + }); +} + +pub fn fnProtoMulti(tree: Ast, node: Node.Index) full.FnProto { + assert(tree.nodes.items(.tag)[node] == .fn_proto_multi); + const data = tree.nodes.items(.data)[node]; + const params_range = tree.extraData(data.lhs, Node.SubRange); + const params = tree.extra_data[params_range.start..params_range.end]; + return tree.fullFnProtoComponents(.{ + .proto_node = node, + .fn_token = tree.nodes.items(.main_token)[node], + .return_type = data.rhs, + .params = params, + .align_expr = 0, + .addrspace_expr = 0, + .section_expr = 0, + .callconv_expr = 0, + }); +} + +pub fn fnProtoOne(tree: Ast, buffer: *[1]Node.Index, node: Node.Index) full.FnProto { + assert(tree.nodes.items(.tag)[node] == .fn_proto_one); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.FnProtoOne); + buffer[0] = extra.param; + const params = if (extra.param == 0) buffer[0..0] else buffer[0..1]; + return tree.fullFnProtoComponents(.{ + .proto_node = node, + .fn_token = tree.nodes.items(.main_token)[node], + .return_type = data.rhs, + .params = params, + .align_expr = extra.align_expr, + .addrspace_expr = extra.addrspace_expr, + .section_expr = extra.section_expr, + .callconv_expr = extra.callconv_expr, + }); +} + +pub fn fnProto(tree: Ast, node: Node.Index) full.FnProto { + assert(tree.nodes.items(.tag)[node] == .fn_proto); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.FnProto); + const params = tree.extra_data[extra.params_start..extra.params_end]; + return tree.fullFnProtoComponents(.{ + .proto_node = node, + .fn_token = tree.nodes.items(.main_token)[node], + .return_type = data.rhs, + .params = params, + .align_expr = extra.align_expr, + .addrspace_expr = extra.addrspace_expr, + .section_expr = extra.section_expr, + .callconv_expr = extra.callconv_expr, + }); +} + +pub fn structInitOne(tree: Ast, buffer: *[1]Node.Index, node: Node.Index) full.StructInit { + assert(tree.nodes.items(.tag)[node] == .struct_init_one or + tree.nodes.items(.tag)[node] == .struct_init_one_comma); + const data = tree.nodes.items(.data)[node]; + buffer[0] = data.rhs; + const fields = if (data.rhs == 0) buffer[0..0] else buffer[0..1]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .fields = fields, + .type_expr = data.lhs, + }, + }; +} + +pub fn structInitDotTwo(tree: Ast, buffer: *[2]Node.Index, node: Node.Index) full.StructInit { + assert(tree.nodes.items(.tag)[node] == .struct_init_dot_two or + tree.nodes.items(.tag)[node] == .struct_init_dot_two_comma); + const data = tree.nodes.items(.data)[node]; + buffer.* = .{ data.lhs, data.rhs }; + const fields = if (data.rhs != 0) + buffer[0..2] + else if (data.lhs != 0) + buffer[0..1] + else + buffer[0..0]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .fields = fields, + .type_expr = 0, + }, + }; +} + +pub fn structInitDot(tree: Ast, node: Node.Index) full.StructInit { + assert(tree.nodes.items(.tag)[node] == .struct_init_dot or + tree.nodes.items(.tag)[node] == .struct_init_dot_comma); + const data = tree.nodes.items(.data)[node]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .fields = tree.extra_data[data.lhs..data.rhs], + .type_expr = 0, + }, + }; +} + +pub fn structInit(tree: Ast, node: Node.Index) full.StructInit { + assert(tree.nodes.items(.tag)[node] == .struct_init or + tree.nodes.items(.tag)[node] == .struct_init_comma); + const data = tree.nodes.items(.data)[node]; + const fields_range = tree.extraData(data.rhs, Node.SubRange); + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .fields = tree.extra_data[fields_range.start..fields_range.end], + .type_expr = data.lhs, + }, + }; +} + +pub fn arrayInitOne(tree: Ast, buffer: *[1]Node.Index, node: Node.Index) full.ArrayInit { + assert(tree.nodes.items(.tag)[node] == .array_init_one or + tree.nodes.items(.tag)[node] == .array_init_one_comma); + const data = tree.nodes.items(.data)[node]; + buffer[0] = data.rhs; + const elements = if (data.rhs == 0) buffer[0..0] else buffer[0..1]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .elements = elements, + .type_expr = data.lhs, + }, + }; +} + +pub fn arrayInitDotTwo(tree: Ast, buffer: *[2]Node.Index, node: Node.Index) full.ArrayInit { + assert(tree.nodes.items(.tag)[node] == .array_init_dot_two or + tree.nodes.items(.tag)[node] == .array_init_dot_two_comma); + const data = tree.nodes.items(.data)[node]; + buffer.* = .{ data.lhs, data.rhs }; + const elements = if (data.rhs != 0) + buffer[0..2] + else if (data.lhs != 0) + buffer[0..1] + else + buffer[0..0]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .elements = elements, + .type_expr = 0, + }, + }; +} + +pub fn arrayInitDot(tree: Ast, node: Node.Index) full.ArrayInit { + assert(tree.nodes.items(.tag)[node] == .array_init_dot or + tree.nodes.items(.tag)[node] == .array_init_dot_comma); + const data = tree.nodes.items(.data)[node]; + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .elements = tree.extra_data[data.lhs..data.rhs], + .type_expr = 0, + }, + }; +} + +pub fn arrayInit(tree: Ast, node: Node.Index) full.ArrayInit { + assert(tree.nodes.items(.tag)[node] == .array_init or + tree.nodes.items(.tag)[node] == .array_init_comma); + const data = tree.nodes.items(.data)[node]; + const elem_range = tree.extraData(data.rhs, Node.SubRange); + return .{ + .ast = .{ + .lbrace = tree.nodes.items(.main_token)[node], + .elements = tree.extra_data[elem_range.start..elem_range.end], + .type_expr = data.lhs, + }, + }; +} + +pub fn arrayType(tree: Ast, node: Node.Index) full.ArrayType { + assert(tree.nodes.items(.tag)[node] == .array_type); + const data = tree.nodes.items(.data)[node]; + return .{ + .ast = .{ + .lbracket = tree.nodes.items(.main_token)[node], + .elem_count = data.lhs, + .sentinel = 0, + .elem_type = data.rhs, + }, + }; +} + +pub fn arrayTypeSentinel(tree: Ast, node: Node.Index) full.ArrayType { + assert(tree.nodes.items(.tag)[node] == .array_type_sentinel); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.ArrayTypeSentinel); + assert(extra.sentinel != 0); + return .{ + .ast = .{ + .lbracket = tree.nodes.items(.main_token)[node], + .elem_count = data.lhs, + .sentinel = extra.sentinel, + .elem_type = extra.elem_type, + }, + }; +} + +pub fn ptrTypeAligned(tree: Ast, node: Node.Index) full.PtrType { + assert(tree.nodes.items(.tag)[node] == .ptr_type_aligned); + const data = tree.nodes.items(.data)[node]; + return tree.fullPtrTypeComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .align_node = data.lhs, + .addrspace_node = 0, + .sentinel = 0, + .bit_range_start = 0, + .bit_range_end = 0, + .child_type = data.rhs, + }); +} + +pub fn ptrTypeSentinel(tree: Ast, node: Node.Index) full.PtrType { + assert(tree.nodes.items(.tag)[node] == .ptr_type_sentinel); + const data = tree.nodes.items(.data)[node]; + return tree.fullPtrTypeComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .align_node = 0, + .addrspace_node = 0, + .sentinel = data.lhs, + .bit_range_start = 0, + .bit_range_end = 0, + .child_type = data.rhs, + }); +} + +pub fn ptrType(tree: Ast, node: Node.Index) full.PtrType { + assert(tree.nodes.items(.tag)[node] == .ptr_type); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.PtrType); + return tree.fullPtrTypeComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .align_node = extra.align_node, + .addrspace_node = extra.addrspace_node, + .sentinel = extra.sentinel, + .bit_range_start = 0, + .bit_range_end = 0, + .child_type = data.rhs, + }); +} + +pub fn ptrTypeBitRange(tree: Ast, node: Node.Index) full.PtrType { + assert(tree.nodes.items(.tag)[node] == .ptr_type_bit_range); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.PtrTypeBitRange); + return tree.fullPtrTypeComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .align_node = extra.align_node, + .addrspace_node = extra.addrspace_node, + .sentinel = extra.sentinel, + .bit_range_start = extra.bit_range_start, + .bit_range_end = extra.bit_range_end, + .child_type = data.rhs, + }); +} + +pub fn sliceOpen(tree: Ast, node: Node.Index) full.Slice { + assert(tree.nodes.items(.tag)[node] == .slice_open); + const data = tree.nodes.items(.data)[node]; + return .{ + .ast = .{ + .sliced = data.lhs, + .lbracket = tree.nodes.items(.main_token)[node], + .start = data.rhs, + .end = 0, + .sentinel = 0, + }, + }; +} + +pub fn slice(tree: Ast, node: Node.Index) full.Slice { + assert(tree.nodes.items(.tag)[node] == .slice); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.Slice); + return .{ + .ast = .{ + .sliced = data.lhs, + .lbracket = tree.nodes.items(.main_token)[node], + .start = extra.start, + .end = extra.end, + .sentinel = 0, + }, + }; +} + +pub fn sliceSentinel(tree: Ast, node: Node.Index) full.Slice { + assert(tree.nodes.items(.tag)[node] == .slice_sentinel); + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.SliceSentinel); + return .{ + .ast = .{ + .sliced = data.lhs, + .lbracket = tree.nodes.items(.main_token)[node], + .start = extra.start, + .end = extra.end, + .sentinel = extra.sentinel, + }, + }; +} + +pub fn containerDeclTwo(tree: Ast, buffer: *[2]Node.Index, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .container_decl_two or + tree.nodes.items(.tag)[node] == .container_decl_two_trailing); + const data = tree.nodes.items(.data)[node]; + buffer.* = .{ data.lhs, data.rhs }; + const members = if (data.rhs != 0) + buffer[0..2] + else if (data.lhs != 0) + buffer[0..1] + else + buffer[0..0]; + return tree.fullContainerDeclComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .enum_token = null, + .members = members, + .arg = 0, + }); +} + +pub fn containerDecl(tree: Ast, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .container_decl or + tree.nodes.items(.tag)[node] == .container_decl_trailing); + const data = tree.nodes.items(.data)[node]; + return tree.fullContainerDeclComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .enum_token = null, + .members = tree.extra_data[data.lhs..data.rhs], + .arg = 0, + }); +} + +pub fn containerDeclArg(tree: Ast, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .container_decl_arg or + tree.nodes.items(.tag)[node] == .container_decl_arg_trailing); + const data = tree.nodes.items(.data)[node]; + const members_range = tree.extraData(data.rhs, Node.SubRange); + return tree.fullContainerDeclComponents(.{ + .main_token = tree.nodes.items(.main_token)[node], + .enum_token = null, + .members = tree.extra_data[members_range.start..members_range.end], + .arg = data.lhs, + }); +} + +pub fn containerDeclRoot(tree: Ast) full.ContainerDecl { + return .{ + .layout_token = null, + .ast = .{ + .main_token = undefined, + .enum_token = null, + .members = tree.rootDecls(), + .arg = 0, + }, + }; +} + +pub fn taggedUnionTwo(tree: Ast, buffer: *[2]Node.Index, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .tagged_union_two or + tree.nodes.items(.tag)[node] == .tagged_union_two_trailing); + const data = tree.nodes.items(.data)[node]; + buffer.* = .{ data.lhs, data.rhs }; + const members = if (data.rhs != 0) + buffer[0..2] + else if (data.lhs != 0) + buffer[0..1] + else + buffer[0..0]; + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerDeclComponents(.{ + .main_token = main_token, + .enum_token = main_token + 2, // union lparen enum + .members = members, + .arg = 0, + }); +} + +pub fn taggedUnion(tree: Ast, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .tagged_union or + tree.nodes.items(.tag)[node] == .tagged_union_trailing); + const data = tree.nodes.items(.data)[node]; + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerDeclComponents(.{ + .main_token = main_token, + .enum_token = main_token + 2, // union lparen enum + .members = tree.extra_data[data.lhs..data.rhs], + .arg = 0, + }); +} + +pub fn taggedUnionEnumTag(tree: Ast, node: Node.Index) full.ContainerDecl { + assert(tree.nodes.items(.tag)[node] == .tagged_union_enum_tag or + tree.nodes.items(.tag)[node] == .tagged_union_enum_tag_trailing); + const data = tree.nodes.items(.data)[node]; + const members_range = tree.extraData(data.rhs, Node.SubRange); + const main_token = tree.nodes.items(.main_token)[node]; + return tree.fullContainerDeclComponents(.{ + .main_token = main_token, + .enum_token = main_token + 2, // union lparen enum + .members = tree.extra_data[members_range.start..members_range.end], + .arg = data.lhs, + }); +} + +pub fn switchCaseOne(tree: Ast, node: Node.Index) full.SwitchCase { + const data = &tree.nodes.items(.data)[node]; + const values: *[1]Node.Index = &data.lhs; + return tree.fullSwitchCaseComponents(.{ + .values = if (data.lhs == 0) values[0..0] else values[0..1], + .arrow_token = tree.nodes.items(.main_token)[node], + .target_expr = data.rhs, + }, node); +} + +pub fn switchCase(tree: Ast, node: Node.Index) full.SwitchCase { + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.lhs, Node.SubRange); + return tree.fullSwitchCaseComponents(.{ + .values = tree.extra_data[extra.start..extra.end], + .arrow_token = tree.nodes.items(.main_token)[node], + .target_expr = data.rhs, + }, node); +} + +pub fn asmSimple(tree: Ast, node: Node.Index) full.Asm { + const data = tree.nodes.items(.data)[node]; + return tree.fullAsmComponents(.{ + .asm_token = tree.nodes.items(.main_token)[node], + .template = data.lhs, + .items = &.{}, + .rparen = data.rhs, + }); +} + +pub fn asmFull(tree: Ast, node: Node.Index) full.Asm { + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.Asm); + return tree.fullAsmComponents(.{ + .asm_token = tree.nodes.items(.main_token)[node], + .template = data.lhs, + .items = tree.extra_data[extra.items_start..extra.items_end], + .rparen = extra.rparen, + }); +} + +pub fn whileSimple(tree: Ast, node: Node.Index) full.While { + const data = tree.nodes.items(.data)[node]; + return tree.fullWhileComponents(.{ + .while_token = tree.nodes.items(.main_token)[node], + .cond_expr = data.lhs, + .cont_expr = 0, + .then_expr = data.rhs, + .else_expr = 0, + }); +} + +pub fn whileCont(tree: Ast, node: Node.Index) full.While { + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.WhileCont); + return tree.fullWhileComponents(.{ + .while_token = tree.nodes.items(.main_token)[node], + .cond_expr = data.lhs, + .cont_expr = extra.cont_expr, + .then_expr = extra.then_expr, + .else_expr = 0, + }); +} + +pub fn whileFull(tree: Ast, node: Node.Index) full.While { + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.While); + return tree.fullWhileComponents(.{ + .while_token = tree.nodes.items(.main_token)[node], + .cond_expr = data.lhs, + .cont_expr = extra.cont_expr, + .then_expr = extra.then_expr, + .else_expr = extra.else_expr, + }); +} + +pub fn forSimple(tree: Ast, node: Node.Index) full.For { + const data = &tree.nodes.items(.data)[node]; + const inputs: *[1]Node.Index = &data.lhs; + return tree.fullForComponents(.{ + .for_token = tree.nodes.items(.main_token)[node], + .inputs = inputs[0..1], + .then_expr = data.rhs, + .else_expr = 0, + }); +} + +pub fn forFull(tree: Ast, node: Node.Index) full.For { + const data = tree.nodes.items(.data)[node]; + const extra = @as(Node.For, @bitCast(data.rhs)); + const inputs = tree.extra_data[data.lhs..][0..extra.inputs]; + const then_expr = tree.extra_data[data.lhs + extra.inputs]; + const else_expr = if (extra.has_else) tree.extra_data[data.lhs + extra.inputs + 1] else 0; + return tree.fullForComponents(.{ + .for_token = tree.nodes.items(.main_token)[node], + .inputs = inputs, + .then_expr = then_expr, + .else_expr = else_expr, + }); +} + +pub fn callOne(tree: Ast, buffer: *[1]Node.Index, node: Node.Index) full.Call { + const data = tree.nodes.items(.data)[node]; + buffer.* = .{data.rhs}; + const params = if (data.rhs != 0) buffer[0..1] else buffer[0..0]; + return tree.fullCallComponents(.{ + .lparen = tree.nodes.items(.main_token)[node], + .fn_expr = data.lhs, + .params = params, + }); +} + +pub fn callFull(tree: Ast, node: Node.Index) full.Call { + const data = tree.nodes.items(.data)[node]; + const extra = tree.extraData(data.rhs, Node.SubRange); + return tree.fullCallComponents(.{ + .lparen = tree.nodes.items(.main_token)[node], + .fn_expr = data.lhs, + .params = tree.extra_data[extra.start..extra.end], + }); +} + +fn fullVarDeclComponents(tree: Ast, info: full.VarDecl.Components) full.VarDecl { + const token_tags = tree.tokens.items(.tag); + var result: full.VarDecl = .{ + .ast = info, + .visib_token = null, + .extern_export_token = null, + .lib_name = null, + .threadlocal_token = null, + .comptime_token = null, + }; + var i = info.mut_token; + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, .keyword_export => result.extern_export_token = i, + .keyword_comptime => result.comptime_token = i, + .keyword_pub => result.visib_token = i, + .keyword_threadlocal => result.threadlocal_token = i, + .string_literal => result.lib_name = i, + else => break, + } + } + return result; +} + +fn fullIfComponents(tree: Ast, info: full.If.Components) full.If { + const token_tags = tree.tokens.items(.tag); + var result: full.If = .{ + .ast = info, + .payload_token = null, + .error_token = null, + .else_token = undefined, + }; + // if (cond_expr) |x| + // ^ ^ + const payload_pipe = tree.lastToken(info.cond_expr) + 2; + if (token_tags[payload_pipe] == .pipe) { + result.payload_token = payload_pipe + 1; + } + if (info.else_expr != 0) { + // then_expr else |x| + // ^ ^ + result.else_token = tree.lastToken(info.then_expr) + 1; + if (token_tags[result.else_token + 1] == .pipe) { + result.error_token = result.else_token + 2; + } + } + return result; +} + +fn fullContainerFieldComponents(tree: Ast, info: full.ContainerField.Components) full.ContainerField { + const token_tags = tree.tokens.items(.tag); + var result: full.ContainerField = .{ + .ast = info, + .comptime_token = null, + }; + if (token_tags[info.main_token] == .keyword_comptime) { + // comptime type = init, + // ^ + result.comptime_token = info.main_token; + } else if (info.main_token > 0 and token_tags[info.main_token - 1] == .keyword_comptime) { + // comptime name: type = init, + // ^ + result.comptime_token = info.main_token - 1; + } + return result; +} + +fn fullFnProtoComponents(tree: Ast, info: full.FnProto.Components) full.FnProto { + const token_tags = tree.tokens.items(.tag); + var result: full.FnProto = .{ + .ast = info, + .visib_token = null, + .extern_export_inline_token = null, + .lib_name = null, + .name_token = null, + .lparen = undefined, + }; + var i = info.fn_token; + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, + .keyword_export, + .keyword_inline, + .keyword_noinline, + => result.extern_export_inline_token = i, + .keyword_pub => result.visib_token = i, + .string_literal => result.lib_name = i, + else => break, + } + } + const after_fn_token = info.fn_token + 1; + if (token_tags[after_fn_token] == .identifier) { + result.name_token = after_fn_token; + result.lparen = after_fn_token + 1; + } else { + result.lparen = after_fn_token; + } + assert(token_tags[result.lparen] == .l_paren); + + return result; +} + +fn fullPtrTypeComponents(tree: Ast, info: full.PtrType.Components) full.PtrType { + const token_tags = tree.tokens.items(.tag); + const Size = std.builtin.Type.Pointer.Size; + const size: Size = switch (token_tags[info.main_token]) { + .asterisk, + .asterisk_asterisk, + => switch (token_tags[info.main_token + 1]) { + .r_bracket, .colon => .Many, + .identifier => if (token_tags[info.main_token -| 1] == .l_bracket) Size.C else .One, + else => .One, + }, + .l_bracket => Size.Slice, + else => unreachable, + }; + var result: full.PtrType = .{ + .size = size, + .allowzero_token = null, + .const_token = null, + .volatile_token = null, + .ast = info, + }; + // We need to be careful that we don't iterate over any sub-expressions + // here while looking for modifiers as that could result in false + // positives. Therefore, start after a sentinel if there is one and + // skip over any align node and bit range nodes. + var i = if (info.sentinel != 0) tree.lastToken(info.sentinel) + 1 else info.main_token; + const end = tree.firstToken(info.child_type); + while (i < end) : (i += 1) { + switch (token_tags[i]) { + .keyword_allowzero => result.allowzero_token = i, + .keyword_const => result.const_token = i, + .keyword_volatile => result.volatile_token = i, + .keyword_align => { + assert(info.align_node != 0); + if (info.bit_range_end != 0) { + assert(info.bit_range_start != 0); + i = tree.lastToken(info.bit_range_end) + 1; + } else { + i = tree.lastToken(info.align_node) + 1; + } + }, + else => {}, + } + } + return result; +} + +fn fullContainerDeclComponents(tree: Ast, info: full.ContainerDecl.Components) full.ContainerDecl { + const token_tags = tree.tokens.items(.tag); + var result: full.ContainerDecl = .{ + .ast = info, + .layout_token = null, + }; + + if (info.main_token == 0) return result; + + switch (token_tags[info.main_token - 1]) { + .keyword_extern, .keyword_packed => result.layout_token = info.main_token - 1, + else => {}, + } + return result; +} + +fn fullSwitchCaseComponents(tree: Ast, info: full.SwitchCase.Components, node: Node.Index) full.SwitchCase { + const token_tags = tree.tokens.items(.tag); + const node_tags = tree.nodes.items(.tag); + var result: full.SwitchCase = .{ + .ast = info, + .payload_token = null, + .inline_token = null, + }; + if (token_tags[info.arrow_token + 1] == .pipe) { + result.payload_token = info.arrow_token + 2; + } + switch (node_tags[node]) { + .switch_case_inline, .switch_case_inline_one => result.inline_token = firstToken(tree, node), + else => {}, + } + return result; +} + +fn fullAsmComponents(tree: Ast, info: full.Asm.Components) full.Asm { + const token_tags = tree.tokens.items(.tag); + const node_tags = tree.nodes.items(.tag); + var result: full.Asm = .{ + .ast = info, + .volatile_token = null, + .inputs = &.{}, + .outputs = &.{}, + .first_clobber = null, + }; + if (token_tags[info.asm_token + 1] == .keyword_volatile) { + result.volatile_token = info.asm_token + 1; + } + const outputs_end: usize = for (info.items, 0..) |item, i| { + switch (node_tags[item]) { + .asm_output => continue, + else => break i, + } + } else info.items.len; + + result.outputs = info.items[0..outputs_end]; + result.inputs = info.items[outputs_end..]; + + if (info.items.len == 0) { + // asm ("foo" ::: "a", "b"); + const template_token = tree.lastToken(info.template); + if (token_tags[template_token + 1] == .colon and + token_tags[template_token + 2] == .colon and + token_tags[template_token + 3] == .colon and + token_tags[template_token + 4] == .string_literal) + { + result.first_clobber = template_token + 4; + } + } else if (result.inputs.len != 0) { + // asm ("foo" :: [_] "" (y) : "a", "b"); + const last_input = result.inputs[result.inputs.len - 1]; + const rparen = tree.lastToken(last_input); + var i = rparen + 1; + // Allow a (useless) comma right after the closing parenthesis. + if (token_tags[i] == .comma) i += 1; + if (token_tags[i] == .colon and + token_tags[i + 1] == .string_literal) + { + result.first_clobber = i + 1; + } + } else { + // asm ("foo" : [_] "" (x) :: "a", "b"); + const last_output = result.outputs[result.outputs.len - 1]; + const rparen = tree.lastToken(last_output); + var i = rparen + 1; + // Allow a (useless) comma right after the closing parenthesis. + if (token_tags[i] == .comma) i += 1; + if (token_tags[i] == .colon and + token_tags[i + 1] == .colon and + token_tags[i + 2] == .string_literal) + { + result.first_clobber = i + 2; + } + } + + return result; +} + +fn fullWhileComponents(tree: Ast, info: full.While.Components) full.While { + const token_tags = tree.tokens.items(.tag); + var result: full.While = .{ + .ast = info, + .inline_token = null, + .label_token = null, + .payload_token = null, + .else_token = undefined, + .error_token = null, + }; + var tok_i = info.while_token -| 1; + if (token_tags[tok_i] == .keyword_inline) { + result.inline_token = tok_i; + tok_i -|= 1; + } + if (token_tags[tok_i] == .colon and + token_tags[tok_i -| 1] == .identifier) + { + result.label_token = tok_i - 1; + } + const last_cond_token = tree.lastToken(info.cond_expr); + if (token_tags[last_cond_token + 2] == .pipe) { + result.payload_token = last_cond_token + 3; + } + if (info.else_expr != 0) { + // then_expr else |x| + // ^ ^ + result.else_token = tree.lastToken(info.then_expr) + 1; + if (token_tags[result.else_token + 1] == .pipe) { + result.error_token = result.else_token + 2; + } + } + return result; +} + +fn fullForComponents(tree: Ast, info: full.For.Components) full.For { + const token_tags = tree.tokens.items(.tag); + var result: full.For = .{ + .ast = info, + .inline_token = null, + .label_token = null, + .payload_token = undefined, + .else_token = undefined, + }; + var tok_i = info.for_token -| 1; + if (token_tags[tok_i] == .keyword_inline) { + result.inline_token = tok_i; + tok_i -|= 1; + } + if (token_tags[tok_i] == .colon and + token_tags[tok_i -| 1] == .identifier) + { + result.label_token = tok_i - 1; + } + const last_cond_token = tree.lastToken(info.inputs[info.inputs.len - 1]); + result.payload_token = last_cond_token + 3 + @intFromBool(token_tags[last_cond_token + 1] == .comma); + if (info.else_expr != 0) { + result.else_token = tree.lastToken(info.then_expr) + 1; + } + return result; +} + +fn fullCallComponents(tree: Ast, info: full.Call.Components) full.Call { + const token_tags = tree.tokens.items(.tag); + var result: full.Call = .{ + .ast = info, + .async_token = null, + }; + const first_token = tree.firstToken(info.fn_expr); + if (first_token != 0 and token_tags[first_token - 1] == .keyword_async) { + result.async_token = first_token - 1; + } + return result; +} + +pub fn fullVarDecl(tree: Ast, node: Node.Index) ?full.VarDecl { + return switch (tree.nodes.items(.tag)[node]) { + .global_var_decl => tree.globalVarDecl(node), + .local_var_decl => tree.localVarDecl(node), + .aligned_var_decl => tree.alignedVarDecl(node), + .simple_var_decl => tree.simpleVarDecl(node), + else => null, + }; +} + +pub fn fullIf(tree: Ast, node: Node.Index) ?full.If { + return switch (tree.nodes.items(.tag)[node]) { + .if_simple => tree.ifSimple(node), + .@"if" => tree.ifFull(node), + else => null, + }; +} + +pub fn fullWhile(tree: Ast, node: Node.Index) ?full.While { + return switch (tree.nodes.items(.tag)[node]) { + .while_simple => tree.whileSimple(node), + .while_cont => tree.whileCont(node), + .@"while" => tree.whileFull(node), + else => null, + }; +} + +pub fn fullFor(tree: Ast, node: Node.Index) ?full.For { + return switch (tree.nodes.items(.tag)[node]) { + .for_simple => tree.forSimple(node), + .@"for" => tree.forFull(node), + else => null, + }; +} + +pub fn fullContainerField(tree: Ast, node: Node.Index) ?full.ContainerField { + return switch (tree.nodes.items(.tag)[node]) { + .container_field_init => tree.containerFieldInit(node), + .container_field_align => tree.containerFieldAlign(node), + .container_field => tree.containerField(node), + else => null, + }; +} + +pub fn fullFnProto(tree: Ast, buffer: *[1]Ast.Node.Index, node: Node.Index) ?full.FnProto { + return switch (tree.nodes.items(.tag)[node]) { + .fn_proto => tree.fnProto(node), + .fn_proto_multi => tree.fnProtoMulti(node), + .fn_proto_one => tree.fnProtoOne(buffer, node), + .fn_proto_simple => tree.fnProtoSimple(buffer, node), + .fn_decl => tree.fullFnProto(buffer, tree.nodes.items(.data)[node].lhs), + else => null, + }; +} + +pub fn fullStructInit(tree: Ast, buffer: *[2]Ast.Node.Index, node: Node.Index) ?full.StructInit { + return switch (tree.nodes.items(.tag)[node]) { + .struct_init_one, .struct_init_one_comma => tree.structInitOne(buffer[0..1], node), + .struct_init_dot_two, .struct_init_dot_two_comma => tree.structInitDotTwo(buffer, node), + .struct_init_dot, .struct_init_dot_comma => tree.structInitDot(node), + .struct_init, .struct_init_comma => tree.structInit(node), + else => null, + }; +} + +pub fn fullArrayInit(tree: Ast, buffer: *[2]Node.Index, node: Node.Index) ?full.ArrayInit { + return switch (tree.nodes.items(.tag)[node]) { + .array_init_one, .array_init_one_comma => tree.arrayInitOne(buffer[0..1], node), + .array_init_dot_two, .array_init_dot_two_comma => tree.arrayInitDotTwo(buffer, node), + .array_init_dot, .array_init_dot_comma => tree.arrayInitDot(node), + .array_init, .array_init_comma => tree.arrayInit(node), + else => null, + }; +} + +pub fn fullArrayType(tree: Ast, node: Node.Index) ?full.ArrayType { + return switch (tree.nodes.items(.tag)[node]) { + .array_type => tree.arrayType(node), + .array_type_sentinel => tree.arrayTypeSentinel(node), + else => null, + }; +} + +pub fn fullPtrType(tree: Ast, node: Node.Index) ?full.PtrType { + return switch (tree.nodes.items(.tag)[node]) { + .ptr_type_aligned => tree.ptrTypeAligned(node), + .ptr_type_sentinel => tree.ptrTypeSentinel(node), + .ptr_type => tree.ptrType(node), + .ptr_type_bit_range => tree.ptrTypeBitRange(node), + else => null, + }; +} + +pub fn fullSlice(tree: Ast, node: Node.Index) ?full.Slice { + return switch (tree.nodes.items(.tag)[node]) { + .slice_open => tree.sliceOpen(node), + .slice => tree.slice(node), + .slice_sentinel => tree.sliceSentinel(node), + else => null, + }; +} + +pub fn fullContainerDecl(tree: Ast, buffer: *[2]Ast.Node.Index, node: Node.Index) ?full.ContainerDecl { + return switch (tree.nodes.items(.tag)[node]) { + .root => tree.containerDeclRoot(), + .container_decl, .container_decl_trailing => tree.containerDecl(node), + .container_decl_arg, .container_decl_arg_trailing => tree.containerDeclArg(node), + .container_decl_two, .container_decl_two_trailing => tree.containerDeclTwo(buffer, node), + .tagged_union, .tagged_union_trailing => tree.taggedUnion(node), + .tagged_union_enum_tag, .tagged_union_enum_tag_trailing => tree.taggedUnionEnumTag(node), + .tagged_union_two, .tagged_union_two_trailing => tree.taggedUnionTwo(buffer, node), + else => null, + }; +} + +pub fn fullSwitchCase(tree: Ast, node: Node.Index) ?full.SwitchCase { + return switch (tree.nodes.items(.tag)[node]) { + .switch_case_one, .switch_case_inline_one => tree.switchCaseOne(node), + .switch_case, .switch_case_inline => tree.switchCase(node), + else => null, + }; +} + +pub fn fullAsm(tree: Ast, node: Node.Index) ?full.Asm { + return switch (tree.nodes.items(.tag)[node]) { + .asm_simple => tree.asmSimple(node), + .@"asm" => tree.asmFull(node), + else => null, + }; +} + +pub fn fullCall(tree: Ast, buffer: *[1]Ast.Node.Index, node: Node.Index) ?full.Call { + return switch (tree.nodes.items(.tag)[node]) { + .call, .call_comma, .async_call, .async_call_comma => tree.callFull(node), + .call_one, .call_one_comma, .async_call_one, .async_call_one_comma => tree.callOne(buffer, node), + else => null, + }; +} + +/// Fully assembled AST node information. +pub const full = struct { + pub const VarDecl = struct { + visib_token: ?TokenIndex, + extern_export_token: ?TokenIndex, + lib_name: ?TokenIndex, + threadlocal_token: ?TokenIndex, + comptime_token: ?TokenIndex, + ast: Components, + + pub const Components = struct { + mut_token: TokenIndex, + type_node: Node.Index, + align_node: Node.Index, + addrspace_node: Node.Index, + section_node: Node.Index, + init_node: Node.Index, + }; + + pub fn firstToken(var_decl: VarDecl) TokenIndex { + return var_decl.visib_token orelse + var_decl.extern_export_token orelse + var_decl.threadlocal_token orelse + var_decl.comptime_token orelse + var_decl.ast.mut_token; + } + }; + + pub const If = struct { + /// Points to the first token after the `|`. Will either be an identifier or + /// a `*` (with an identifier immediately after it). + payload_token: ?TokenIndex, + /// Points to the identifier after the `|`. + error_token: ?TokenIndex, + /// Populated only if else_expr != 0. + else_token: TokenIndex, + ast: Components, + + pub const Components = struct { + if_token: TokenIndex, + cond_expr: Node.Index, + then_expr: Node.Index, + else_expr: Node.Index, + }; + }; + + pub const While = struct { + ast: Components, + inline_token: ?TokenIndex, + label_token: ?TokenIndex, + payload_token: ?TokenIndex, + error_token: ?TokenIndex, + /// Populated only if else_expr != 0. + else_token: TokenIndex, + + pub const Components = struct { + while_token: TokenIndex, + cond_expr: Node.Index, + cont_expr: Node.Index, + then_expr: Node.Index, + else_expr: Node.Index, + }; + }; + + pub const For = struct { + ast: Components, + inline_token: ?TokenIndex, + label_token: ?TokenIndex, + payload_token: TokenIndex, + /// Populated only if else_expr != 0. + else_token: TokenIndex, + + pub const Components = struct { + for_token: TokenIndex, + inputs: []const Node.Index, + then_expr: Node.Index, + else_expr: Node.Index, + }; + }; + + pub const ContainerField = struct { + comptime_token: ?TokenIndex, + ast: Components, + + pub const Components = struct { + main_token: TokenIndex, + type_expr: Node.Index, + value_expr: Node.Index, + align_expr: Node.Index, + tuple_like: bool, + }; + + pub fn firstToken(cf: ContainerField) TokenIndex { + return cf.comptime_token orelse cf.ast.main_token; + } + + pub fn convertToNonTupleLike(cf: *ContainerField, nodes: std.zig.Ast.NodeList.Slice) void { + if (!cf.ast.tuple_like) return; + if (cf.ast.type_expr == 0) return; + if (nodes.items(.tag)[cf.ast.type_expr] != .identifier) return; + + const ident = nodes.items(.main_token)[cf.ast.type_expr]; + cf.ast.tuple_like = false; + cf.ast.main_token = ident; + cf.ast.type_expr = 0; + } + }; + + pub const FnProto = struct { + visib_token: ?TokenIndex, + extern_export_inline_token: ?TokenIndex, + lib_name: ?TokenIndex, + name_token: ?TokenIndex, + lparen: TokenIndex, + ast: Components, + + pub const Components = struct { + proto_node: Node.Index, + fn_token: TokenIndex, + return_type: Node.Index, + params: []const Node.Index, + align_expr: Node.Index, + addrspace_expr: Node.Index, + section_expr: Node.Index, + callconv_expr: Node.Index, + }; + + pub const Param = struct { + first_doc_comment: ?TokenIndex, + name_token: ?TokenIndex, + comptime_noalias: ?TokenIndex, + anytype_ellipsis3: ?TokenIndex, + type_expr: Node.Index, + }; + + pub fn firstToken(fn_proto: FnProto) TokenIndex { + return fn_proto.visib_token orelse + fn_proto.extern_export_inline_token orelse + fn_proto.ast.fn_token; + } + + /// Abstracts over the fact that anytype and ... are not included + /// in the params slice, since they are simple identifiers and + /// not sub-expressions. + pub const Iterator = struct { + tree: *const Ast, + fn_proto: *const FnProto, + param_i: usize, + tok_i: TokenIndex, + tok_flag: bool, + + pub fn next(it: *Iterator) ?Param { + const token_tags = it.tree.tokens.items(.tag); + while (true) { + var first_doc_comment: ?TokenIndex = null; + var comptime_noalias: ?TokenIndex = null; + var name_token: ?TokenIndex = null; + if (!it.tok_flag) { + if (it.param_i >= it.fn_proto.ast.params.len) { + return null; + } + const param_type = it.fn_proto.ast.params[it.param_i]; + var tok_i = it.tree.firstToken(param_type) - 1; + while (true) : (tok_i -= 1) switch (token_tags[tok_i]) { + .colon => continue, + .identifier => name_token = tok_i, + .doc_comment => first_doc_comment = tok_i, + .keyword_comptime, .keyword_noalias => comptime_noalias = tok_i, + else => break, + }; + it.param_i += 1; + it.tok_i = it.tree.lastToken(param_type) + 1; + // Look for anytype and ... params afterwards. + if (token_tags[it.tok_i] == .comma) { + it.tok_i += 1; + } + it.tok_flag = true; + return Param{ + .first_doc_comment = first_doc_comment, + .comptime_noalias = comptime_noalias, + .name_token = name_token, + .anytype_ellipsis3 = null, + .type_expr = param_type, + }; + } + if (token_tags[it.tok_i] == .comma) { + it.tok_i += 1; + } + if (token_tags[it.tok_i] == .r_paren) { + return null; + } + if (token_tags[it.tok_i] == .doc_comment) { + first_doc_comment = it.tok_i; + while (token_tags[it.tok_i] == .doc_comment) { + it.tok_i += 1; + } + } + switch (token_tags[it.tok_i]) { + .ellipsis3 => { + it.tok_flag = false; // Next iteration should return null. + return Param{ + .first_doc_comment = first_doc_comment, + .comptime_noalias = null, + .name_token = null, + .anytype_ellipsis3 = it.tok_i, + .type_expr = 0, + }; + }, + .keyword_noalias, .keyword_comptime => { + comptime_noalias = it.tok_i; + it.tok_i += 1; + }, + else => {}, + } + if (token_tags[it.tok_i] == .identifier and + token_tags[it.tok_i + 1] == .colon) + { + name_token = it.tok_i; + it.tok_i += 2; + } + if (token_tags[it.tok_i] == .keyword_anytype) { + it.tok_i += 1; + return Param{ + .first_doc_comment = first_doc_comment, + .comptime_noalias = comptime_noalias, + .name_token = name_token, + .anytype_ellipsis3 = it.tok_i - 1, + .type_expr = 0, + }; + } + it.tok_flag = false; + } + } + }; + + pub fn iterate(fn_proto: *const FnProto, tree: *const Ast) Iterator { + return .{ + .tree = tree, + .fn_proto = fn_proto, + .param_i = 0, + .tok_i = fn_proto.lparen + 1, + .tok_flag = true, + }; + } + }; + + pub const StructInit = struct { + ast: Components, + + pub const Components = struct { + lbrace: TokenIndex, + fields: []const Node.Index, + type_expr: Node.Index, + }; + }; + + pub const ArrayInit = struct { + ast: Components, + + pub const Components = struct { + lbrace: TokenIndex, + elements: []const Node.Index, + type_expr: Node.Index, + }; + }; + + pub const ArrayType = struct { + ast: Components, + + pub const Components = struct { + lbracket: TokenIndex, + elem_count: Node.Index, + sentinel: Node.Index, + elem_type: Node.Index, + }; + }; + + pub const PtrType = struct { + size: std.builtin.Type.Pointer.Size, + allowzero_token: ?TokenIndex, + const_token: ?TokenIndex, + volatile_token: ?TokenIndex, + ast: Components, + + pub const Components = struct { + main_token: TokenIndex, + align_node: Node.Index, + addrspace_node: Node.Index, + sentinel: Node.Index, + bit_range_start: Node.Index, + bit_range_end: Node.Index, + child_type: Node.Index, + }; + }; + + pub const Slice = struct { + ast: Components, + + pub const Components = struct { + sliced: Node.Index, + lbracket: TokenIndex, + start: Node.Index, + end: Node.Index, + sentinel: Node.Index, + }; + }; + + pub const ContainerDecl = struct { + layout_token: ?TokenIndex, + ast: Components, + + pub const Components = struct { + main_token: TokenIndex, + /// Populated when main_token is Keyword_union. + enum_token: ?TokenIndex, + members: []const Node.Index, + arg: Node.Index, + }; + }; + + pub const SwitchCase = struct { + inline_token: ?TokenIndex, + /// Points to the first token after the `|`. Will either be an identifier or + /// a `*` (with an identifier immediately after it). + payload_token: ?TokenIndex, + ast: Components, + + pub const Components = struct { + /// If empty, this is an else case + values: []const Node.Index, + arrow_token: TokenIndex, + target_expr: Node.Index, + }; + }; + + pub const Asm = struct { + ast: Components, + volatile_token: ?TokenIndex, + first_clobber: ?TokenIndex, + outputs: []const Node.Index, + inputs: []const Node.Index, + + pub const Components = struct { + asm_token: TokenIndex, + template: Node.Index, + items: []const Node.Index, + rparen: TokenIndex, + }; + }; + + pub const Call = struct { + ast: Components, + async_token: ?TokenIndex, + + pub const Components = struct { + lparen: TokenIndex, + fn_expr: Node.Index, + params: []const Node.Index, + }; + }; +}; + +pub const Error = struct { + tag: Tag, + is_note: bool = false, + /// True if `token` points to the token before the token causing an issue. + token_is_prev: bool = false, + token: TokenIndex, + extra: union { + none: void, + expected_tag: Token.Tag, + } = .{ .none = {} }, + + pub const Tag = enum { + asterisk_after_ptr_deref, + chained_comparison_operators, + decl_between_fields, + expected_block, + expected_block_or_assignment, + expected_block_or_expr, + expected_block_or_field, + expected_container_members, + expected_expr, + expected_expr_or_assignment, + expected_expr_or_var_decl, + expected_fn, + expected_inlinable, + expected_labelable, + expected_param_list, + expected_prefix_expr, + expected_primary_type_expr, + expected_pub_item, + expected_return_type, + expected_semi_or_else, + expected_semi_or_lbrace, + expected_statement, + expected_suffix_op, + expected_type_expr, + expected_var_decl, + expected_var_decl_or_fn, + expected_loop_payload, + expected_container, + extern_fn_body, + extra_addrspace_qualifier, + extra_align_qualifier, + extra_allowzero_qualifier, + extra_const_qualifier, + extra_volatile_qualifier, + ptr_mod_on_array_child_type, + invalid_bit_range, + same_line_doc_comment, + unattached_doc_comment, + test_doc_comment, + comptime_doc_comment, + varargs_nonfinal, + expected_continue_expr, + expected_semi_after_decl, + expected_semi_after_stmt, + expected_comma_after_field, + expected_comma_after_arg, + expected_comma_after_param, + expected_comma_after_initializer, + expected_comma_after_switch_prong, + expected_comma_after_for_operand, + expected_comma_after_capture, + expected_initializer, + mismatched_binary_op_whitespace, + invalid_ampersand_ampersand, + c_style_container, + expected_var_const, + wrong_equal_var_decl, + var_const_decl, + extra_for_capture, + for_input_not_captured, + + zig_style_container, + previous_field, + next_field, + + /// `expected_tag` is populated. + expected_token, + }; +}; + +pub const Node = struct { + tag: Tag, + main_token: TokenIndex, + data: Data, + + pub const Index = u32; + + comptime { + // Goal is to keep this under one byte for efficiency. + assert(@sizeOf(Tag) == 1); + } + + /// Note: The FooComma/FooSemicolon variants exist to ease the implementation of + /// Ast.lastToken() + pub const Tag = enum { + /// sub_list[lhs...rhs] + root, + /// `usingnamespace lhs;`. rhs unused. main_token is `usingnamespace`. + @"usingnamespace", + /// lhs is test name token (must be string literal or identifier), if any. + /// rhs is the body node. + test_decl, + /// lhs is the index into extra_data. + /// rhs is the initialization expression, if any. + /// main_token is `var` or `const`. + global_var_decl, + /// `var a: x align(y) = rhs` + /// lhs is the index into extra_data. + /// main_token is `var` or `const`. + local_var_decl, + /// `var a: lhs = rhs`. lhs and rhs may be unused. + /// Can be local or global. + /// main_token is `var` or `const`. + simple_var_decl, + /// `var a align(lhs) = rhs`. lhs and rhs may be unused. + /// Can be local or global. + /// main_token is `var` or `const`. + aligned_var_decl, + /// lhs is the identifier token payload if any, + /// rhs is the deferred expression. + @"errdefer", + /// lhs is unused. + /// rhs is the deferred expression. + @"defer", + /// lhs catch rhs + /// lhs catch |err| rhs + /// main_token is the `catch` keyword. + /// payload is determined by looking at the next token after the `catch` keyword. + @"catch", + /// `lhs.a`. main_token is the dot. rhs is the identifier token index. + field_access, + /// `lhs.?`. main_token is the dot. rhs is the `?` token index. + unwrap_optional, + /// `lhs == rhs`. main_token is op. + equal_equal, + /// `lhs != rhs`. main_token is op. + bang_equal, + /// `lhs < rhs`. main_token is op. + less_than, + /// `lhs > rhs`. main_token is op. + greater_than, + /// `lhs <= rhs`. main_token is op. + less_or_equal, + /// `lhs >= rhs`. main_token is op. + greater_or_equal, + /// `lhs *= rhs`. main_token is op. + assign_mul, + /// `lhs /= rhs`. main_token is op. + assign_div, + /// `lhs *= rhs`. main_token is op. + assign_mod, + /// `lhs += rhs`. main_token is op. + assign_add, + /// `lhs -= rhs`. main_token is op. + assign_sub, + /// `lhs <<= rhs`. main_token is op. + assign_shl, + /// `lhs <<|= rhs`. main_token is op. + assign_shl_sat, + /// `lhs >>= rhs`. main_token is op. + assign_shr, + /// `lhs &= rhs`. main_token is op. + assign_bit_and, + /// `lhs ^= rhs`. main_token is op. + assign_bit_xor, + /// `lhs |= rhs`. main_token is op. + assign_bit_or, + /// `lhs *%= rhs`. main_token is op. + assign_mul_wrap, + /// `lhs +%= rhs`. main_token is op. + assign_add_wrap, + /// `lhs -%= rhs`. main_token is op. + assign_sub_wrap, + /// `lhs *|= rhs`. main_token is op. + assign_mul_sat, + /// `lhs +|= rhs`. main_token is op. + assign_add_sat, + /// `lhs -|= rhs`. main_token is op. + assign_sub_sat, + /// `lhs = rhs`. main_token is op. + assign, + /// `a, b, ... = rhs`. main_token is op. lhs is index into `extra_data` + /// of an lhs elem count followed by an array of that many `Node.Index`, + /// with each node having one of the following types: + /// * `global_var_decl` + /// * `local_var_decl` + /// * `simple_var_decl` + /// * `aligned_var_decl` + /// * Any expression node + /// The first 3 types correspond to a `var` or `const` lhs node (note + /// that their `rhs` is always 0). An expression node corresponds to a + /// standard assignment LHS (which must be evaluated as an lvalue). + /// There may be a preceding `comptime` token, which does not create a + /// corresponding `comptime` node so must be manually detected. + assign_destructure, + /// `lhs || rhs`. main_token is the `||`. + merge_error_sets, + /// `lhs * rhs`. main_token is the `*`. + mul, + /// `lhs / rhs`. main_token is the `/`. + div, + /// `lhs % rhs`. main_token is the `%`. + mod, + /// `lhs ** rhs`. main_token is the `**`. + array_mult, + /// `lhs *% rhs`. main_token is the `*%`. + mul_wrap, + /// `lhs *| rhs`. main_token is the `*|`. + mul_sat, + /// `lhs + rhs`. main_token is the `+`. + add, + /// `lhs - rhs`. main_token is the `-`. + sub, + /// `lhs ++ rhs`. main_token is the `++`. + array_cat, + /// `lhs +% rhs`. main_token is the `+%`. + add_wrap, + /// `lhs -% rhs`. main_token is the `-%`. + sub_wrap, + /// `lhs +| rhs`. main_token is the `+|`. + add_sat, + /// `lhs -| rhs`. main_token is the `-|`. + sub_sat, + /// `lhs << rhs`. main_token is the `<<`. + shl, + /// `lhs <<| rhs`. main_token is the `<<|`. + shl_sat, + /// `lhs >> rhs`. main_token is the `>>`. + shr, + /// `lhs & rhs`. main_token is the `&`. + bit_and, + /// `lhs ^ rhs`. main_token is the `^`. + bit_xor, + /// `lhs | rhs`. main_token is the `|`. + bit_or, + /// `lhs orelse rhs`. main_token is the `orelse`. + @"orelse", + /// `lhs and rhs`. main_token is the `and`. + bool_and, + /// `lhs or rhs`. main_token is the `or`. + bool_or, + /// `op lhs`. rhs unused. main_token is op. + bool_not, + /// `op lhs`. rhs unused. main_token is op. + negation, + /// `op lhs`. rhs unused. main_token is op. + bit_not, + /// `op lhs`. rhs unused. main_token is op. + negation_wrap, + /// `op lhs`. rhs unused. main_token is op. + address_of, + /// `op lhs`. rhs unused. main_token is op. + @"try", + /// `op lhs`. rhs unused. main_token is op. + @"await", + /// `?lhs`. rhs unused. main_token is the `?`. + optional_type, + /// `[lhs]rhs`. + array_type, + /// `[lhs:a]b`. `ArrayTypeSentinel[rhs]`. + array_type_sentinel, + /// `[*]align(lhs) rhs`. lhs can be omitted. + /// `*align(lhs) rhs`. lhs can be omitted. + /// `[]rhs`. + /// main_token is the asterisk if a pointer or the lbracket if a slice + /// main_token might be a ** token, which is shared with a parent/child + /// pointer type and may require special handling. + ptr_type_aligned, + /// `[*:lhs]rhs`. lhs can be omitted. + /// `*rhs`. + /// `[:lhs]rhs`. + /// main_token is the asterisk if a pointer or the lbracket if a slice + /// main_token might be a ** token, which is shared with a parent/child + /// pointer type and may require special handling. + ptr_type_sentinel, + /// lhs is index into ptr_type. rhs is the element type expression. + /// main_token is the asterisk if a pointer or the lbracket if a slice + /// main_token might be a ** token, which is shared with a parent/child + /// pointer type and may require special handling. + ptr_type, + /// lhs is index into ptr_type_bit_range. rhs is the element type expression. + /// main_token is the asterisk if a pointer or the lbracket if a slice + /// main_token might be a ** token, which is shared with a parent/child + /// pointer type and may require special handling. + ptr_type_bit_range, + /// `lhs[rhs..]` + /// main_token is the lbracket. + slice_open, + /// `lhs[b..c]`. rhs is index into Slice + /// main_token is the lbracket. + slice, + /// `lhs[b..c :d]`. rhs is index into SliceSentinel + /// main_token is the lbracket. + slice_sentinel, + /// `lhs.*`. rhs is unused. + deref, + /// `lhs[rhs]`. + array_access, + /// `lhs{rhs}`. rhs can be omitted. + array_init_one, + /// `lhs{rhs,}`. rhs can *not* be omitted + array_init_one_comma, + /// `.{lhs, rhs}`. lhs and rhs can be omitted. + array_init_dot_two, + /// Same as `array_init_dot_two` except there is known to be a trailing comma + /// before the final rbrace. + array_init_dot_two_comma, + /// `.{a, b}`. `sub_list[lhs..rhs]`. + array_init_dot, + /// Same as `array_init_dot` except there is known to be a trailing comma + /// before the final rbrace. + array_init_dot_comma, + /// `lhs{a, b}`. `sub_range_list[rhs]`. lhs can be omitted which means `.{a, b}`. + array_init, + /// Same as `array_init` except there is known to be a trailing comma + /// before the final rbrace. + array_init_comma, + /// `lhs{.a = rhs}`. rhs can be omitted making it empty. + /// main_token is the lbrace. + struct_init_one, + /// `lhs{.a = rhs,}`. rhs can *not* be omitted. + /// main_token is the lbrace. + struct_init_one_comma, + /// `.{.a = lhs, .b = rhs}`. lhs and rhs can be omitted. + /// main_token is the lbrace. + /// No trailing comma before the rbrace. + struct_init_dot_two, + /// Same as `struct_init_dot_two` except there is known to be a trailing comma + /// before the final rbrace. + struct_init_dot_two_comma, + /// `.{.a = b, .c = d}`. `sub_list[lhs..rhs]`. + /// main_token is the lbrace. + struct_init_dot, + /// Same as `struct_init_dot` except there is known to be a trailing comma + /// before the final rbrace. + struct_init_dot_comma, + /// `lhs{.a = b, .c = d}`. `sub_range_list[rhs]`. + /// lhs can be omitted which means `.{.a = b, .c = d}`. + /// main_token is the lbrace. + struct_init, + /// Same as `struct_init` except there is known to be a trailing comma + /// before the final rbrace. + struct_init_comma, + /// `lhs(rhs)`. rhs can be omitted. + /// main_token is the lparen. + call_one, + /// `lhs(rhs,)`. rhs can be omitted. + /// main_token is the lparen. + call_one_comma, + /// `async lhs(rhs)`. rhs can be omitted. + async_call_one, + /// `async lhs(rhs,)`. + async_call_one_comma, + /// `lhs(a, b, c)`. `SubRange[rhs]`. + /// main_token is the `(`. + call, + /// `lhs(a, b, c,)`. `SubRange[rhs]`. + /// main_token is the `(`. + call_comma, + /// `async lhs(a, b, c)`. `SubRange[rhs]`. + /// main_token is the `(`. + async_call, + /// `async lhs(a, b, c,)`. `SubRange[rhs]`. + /// main_token is the `(`. + async_call_comma, + /// `switch(lhs) {}`. `SubRange[rhs]`. + @"switch", + /// Same as switch except there is known to be a trailing comma + /// before the final rbrace + switch_comma, + /// `lhs => rhs`. If lhs is omitted it means `else`. + /// main_token is the `=>` + switch_case_one, + /// Same ast `switch_case_one` but the case is inline + switch_case_inline_one, + /// `a, b, c => rhs`. `SubRange[lhs]`. + /// main_token is the `=>` + switch_case, + /// Same ast `switch_case` but the case is inline + switch_case_inline, + /// `lhs...rhs`. + switch_range, + /// `while (lhs) rhs`. + /// `while (lhs) |x| rhs`. + while_simple, + /// `while (lhs) : (a) b`. `WhileCont[rhs]`. + /// `while (lhs) : (a) b`. `WhileCont[rhs]`. + while_cont, + /// `while (lhs) : (a) b else c`. `While[rhs]`. + /// `while (lhs) |x| : (a) b else c`. `While[rhs]`. + /// `while (lhs) |x| : (a) b else |y| c`. `While[rhs]`. + @"while", + /// `for (lhs) rhs`. + for_simple, + /// `for (lhs[0..inputs]) lhs[inputs + 1] else lhs[inputs + 2]`. `For[rhs]`. + @"for", + /// `lhs..rhs`. + for_range, + /// `if (lhs) rhs`. + /// `if (lhs) |a| rhs`. + if_simple, + /// `if (lhs) a else b`. `If[rhs]`. + /// `if (lhs) |x| a else b`. `If[rhs]`. + /// `if (lhs) |x| a else |y| b`. `If[rhs]`. + @"if", + /// `suspend lhs`. lhs can be omitted. rhs is unused. + @"suspend", + /// `resume lhs`. rhs is unused. + @"resume", + /// `continue`. lhs is token index of label if any. rhs is unused. + @"continue", + /// `break :lhs rhs` + /// both lhs and rhs may be omitted. + @"break", + /// `return lhs`. lhs can be omitted. rhs is unused. + @"return", + /// `fn (a: lhs) rhs`. lhs can be omitted. + /// anytype and ... parameters are omitted from the AST tree. + /// main_token is the `fn` keyword. + /// extern function declarations use this tag. + fn_proto_simple, + /// `fn (a: b, c: d) rhs`. `sub_range_list[lhs]`. + /// anytype and ... parameters are omitted from the AST tree. + /// main_token is the `fn` keyword. + /// extern function declarations use this tag. + fn_proto_multi, + /// `fn (a: b) rhs addrspace(e) linksection(f) callconv(g)`. `FnProtoOne[lhs]`. + /// zero or one parameters. + /// anytype and ... parameters are omitted from the AST tree. + /// main_token is the `fn` keyword. + /// extern function declarations use this tag. + fn_proto_one, + /// `fn (a: b, c: d) rhs addrspace(e) linksection(f) callconv(g)`. `FnProto[lhs]`. + /// anytype and ... parameters are omitted from the AST tree. + /// main_token is the `fn` keyword. + /// extern function declarations use this tag. + fn_proto, + /// lhs is the fn_proto. + /// rhs is the function body block. + /// Note that extern function declarations use the fn_proto tags rather + /// than this one. + fn_decl, + /// `anyframe->rhs`. main_token is `anyframe`. `lhs` is arrow token index. + anyframe_type, + /// Both lhs and rhs unused. + anyframe_literal, + /// Both lhs and rhs unused. + char_literal, + /// Both lhs and rhs unused. + number_literal, + /// Both lhs and rhs unused. + unreachable_literal, + /// Both lhs and rhs unused. + /// Most identifiers will not have explicit AST nodes, however for expressions + /// which could be one of many different kinds of AST nodes, there will be an + /// identifier AST node for it. + identifier, + /// lhs is the dot token index, rhs unused, main_token is the identifier. + enum_literal, + /// main_token is the string literal token + /// Both lhs and rhs unused. + string_literal, + /// main_token is the first token index (redundant with lhs) + /// lhs is the first token index; rhs is the last token index. + /// Could be a series of multiline_string_literal_line tokens, or a single + /// string_literal token. + multiline_string_literal, + /// `(lhs)`. main_token is the `(`; rhs is the token index of the `)`. + grouped_expression, + /// `@a(lhs, rhs)`. lhs and rhs may be omitted. + /// main_token is the builtin token. + builtin_call_two, + /// Same as builtin_call_two but there is known to be a trailing comma before the rparen. + builtin_call_two_comma, + /// `@a(b, c)`. `sub_list[lhs..rhs]`. + /// main_token is the builtin token. + builtin_call, + /// Same as builtin_call but there is known to be a trailing comma before the rparen. + builtin_call_comma, + /// `error{a, b}`. + /// rhs is the rbrace, lhs is unused. + error_set_decl, + /// `struct {}`, `union {}`, `opaque {}`, `enum {}`. `extra_data[lhs..rhs]`. + /// main_token is `struct`, `union`, `opaque`, `enum` keyword. + container_decl, + /// Same as ContainerDecl but there is known to be a trailing comma + /// or semicolon before the rbrace. + container_decl_trailing, + /// `struct {lhs, rhs}`, `union {lhs, rhs}`, `opaque {lhs, rhs}`, `enum {lhs, rhs}`. + /// lhs or rhs can be omitted. + /// main_token is `struct`, `union`, `opaque`, `enum` keyword. + container_decl_two, + /// Same as ContainerDeclTwo except there is known to be a trailing comma + /// or semicolon before the rbrace. + container_decl_two_trailing, + /// `struct(lhs)` / `union(lhs)` / `enum(lhs)`. `SubRange[rhs]`. + container_decl_arg, + /// Same as container_decl_arg but there is known to be a trailing + /// comma or semicolon before the rbrace. + container_decl_arg_trailing, + /// `union(enum) {}`. `sub_list[lhs..rhs]`. + /// Note that tagged unions with explicitly provided enums are represented + /// by `container_decl_arg`. + tagged_union, + /// Same as tagged_union but there is known to be a trailing comma + /// or semicolon before the rbrace. + tagged_union_trailing, + /// `union(enum) {lhs, rhs}`. lhs or rhs may be omitted. + /// Note that tagged unions with explicitly provided enums are represented + /// by `container_decl_arg`. + tagged_union_two, + /// Same as tagged_union_two but there is known to be a trailing comma + /// or semicolon before the rbrace. + tagged_union_two_trailing, + /// `union(enum(lhs)) {}`. `SubRange[rhs]`. + tagged_union_enum_tag, + /// Same as tagged_union_enum_tag but there is known to be a trailing comma + /// or semicolon before the rbrace. + tagged_union_enum_tag_trailing, + /// `a: lhs = rhs,`. lhs and rhs can be omitted. + /// main_token is the field name identifier. + /// lastToken() does not include the possible trailing comma. + container_field_init, + /// `a: lhs align(rhs),`. rhs can be omitted. + /// main_token is the field name identifier. + /// lastToken() does not include the possible trailing comma. + container_field_align, + /// `a: lhs align(c) = d,`. `container_field_list[rhs]`. + /// main_token is the field name identifier. + /// lastToken() does not include the possible trailing comma. + container_field, + /// `comptime lhs`. rhs unused. + @"comptime", + /// `nosuspend lhs`. rhs unused. + @"nosuspend", + /// `{lhs rhs}`. rhs or lhs can be omitted. + /// main_token points at the lbrace. + block_two, + /// Same as block_two but there is known to be a semicolon before the rbrace. + block_two_semicolon, + /// `{}`. `sub_list[lhs..rhs]`. + /// main_token points at the lbrace. + block, + /// Same as block but there is known to be a semicolon before the rbrace. + block_semicolon, + /// `asm(lhs)`. rhs is the token index of the rparen. + asm_simple, + /// `asm(lhs, a)`. `Asm[rhs]`. + @"asm", + /// `[a] "b" (c)`. lhs is 0, rhs is token index of the rparen. + /// `[a] "b" (-> lhs)`. rhs is token index of the rparen. + /// main_token is `a`. + asm_output, + /// `[a] "b" (lhs)`. rhs is token index of the rparen. + /// main_token is `a`. + asm_input, + /// `error.a`. lhs is token index of `.`. rhs is token index of `a`. + error_value, + /// `lhs!rhs`. main_token is the `!`. + error_union, + + pub fn isContainerField(tag: Tag) bool { + return switch (tag) { + .container_field_init, + .container_field_align, + .container_field, + => true, + + else => false, + }; + } + }; + + pub const Data = struct { + lhs: Index, + rhs: Index, + }; + + pub const LocalVarDecl = struct { + type_node: Index, + align_node: Index, + }; + + pub const ArrayTypeSentinel = struct { + elem_type: Index, + sentinel: Index, + }; + + pub const PtrType = struct { + sentinel: Index, + align_node: Index, + addrspace_node: Index, + }; + + pub const PtrTypeBitRange = struct { + sentinel: Index, + align_node: Index, + addrspace_node: Index, + bit_range_start: Index, + bit_range_end: Index, + }; + + pub const SubRange = struct { + /// Index into sub_list. + start: Index, + /// Index into sub_list. + end: Index, + }; + + pub const If = struct { + then_expr: Index, + else_expr: Index, + }; + + pub const ContainerField = struct { + value_expr: Index, + align_expr: Index, + }; + + pub const GlobalVarDecl = struct { + /// Populated if there is an explicit type ascription. + type_node: Index, + /// Populated if align(A) is present. + align_node: Index, + /// Populated if addrspace(A) is present. + addrspace_node: Index, + /// Populated if linksection(A) is present. + section_node: Index, + }; + + pub const Slice = struct { + start: Index, + end: Index, + }; + + pub const SliceSentinel = struct { + start: Index, + /// May be 0 if the slice is "open" + end: Index, + sentinel: Index, + }; + + pub const While = struct { + cont_expr: Index, + then_expr: Index, + else_expr: Index, + }; + + pub const WhileCont = struct { + cont_expr: Index, + then_expr: Index, + }; + + pub const For = packed struct(u32) { + inputs: u31, + has_else: bool, + }; + + pub const FnProtoOne = struct { + /// Populated if there is exactly 1 parameter. Otherwise there are 0 parameters. + param: Index, + /// Populated if align(A) is present. + align_expr: Index, + /// Populated if addrspace(A) is present. + addrspace_expr: Index, + /// Populated if linksection(A) is present. + section_expr: Index, + /// Populated if callconv(A) is present. + callconv_expr: Index, + }; + + pub const FnProto = struct { + params_start: Index, + params_end: Index, + /// Populated if align(A) is present. + align_expr: Index, + /// Populated if addrspace(A) is present. + addrspace_expr: Index, + /// Populated if linksection(A) is present. + section_expr: Index, + /// Populated if callconv(A) is present. + callconv_expr: Index, + }; + + pub const Asm = struct { + items_start: Index, + items_end: Index, + /// Needed to make lastToken() work. + rparen: TokenIndex, + }; +}; + +const std = @import("std"); +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const Token = std.zig.Token; +const Ast = @This(); +const Allocator = std.mem.Allocator; +const Parse = @import("Parse.zig"); + +test { + testing.refAllDecls(@This()); +} diff --git a/src/stage2/Parse.zig b/src/stage2/Parse.zig new file mode 100644 index 0000000000..879978f3fa --- /dev/null +++ b/src/stage2/Parse.zig @@ -0,0 +1,4106 @@ +//! Represents in-progress parsing, will be converted to an Ast after completion. + +pub const Error = error{ParseError} || Allocator.Error; + +gpa: Allocator, +source: []const u8, +token_tags: []const Token.Tag, +token_starts: []const Ast.ByteOffset, +tok_i: TokenIndex, +errors: std.ArrayListUnmanaged(AstError), +nodes: Ast.NodeList, +extra_data: std.ArrayListUnmanaged(Node.Index), +scratch: std.ArrayListUnmanaged(Node.Index), + +const SmallSpan = union(enum) { + zero_or_one: Node.Index, + multi: Node.SubRange, +}; + +const Members = struct { + len: usize, + lhs: Node.Index, + rhs: Node.Index, + trailing: bool, + + fn toSpan(self: Members, p: *Parse) !Node.SubRange { + if (self.len <= 2) { + const nodes = [2]Node.Index{ self.lhs, self.rhs }; + return p.listToSpan(nodes[0..self.len]); + } else { + return Node.SubRange{ .start = self.lhs, .end = self.rhs }; + } + } +}; + +fn listToSpan(p: *Parse, list: []const Node.Index) !Node.SubRange { + try p.extra_data.appendSlice(p.gpa, list); + return Node.SubRange{ + .start = @as(Node.Index, @intCast(p.extra_data.items.len - list.len)), + .end = @as(Node.Index, @intCast(p.extra_data.items.len)), + }; +} + +fn addNode(p: *Parse, elem: Ast.Node) Allocator.Error!Node.Index { + const result = @as(Node.Index, @intCast(p.nodes.len)); + try p.nodes.append(p.gpa, elem); + return result; +} + +fn setNode(p: *Parse, i: usize, elem: Ast.Node) Node.Index { + p.nodes.set(i, elem); + return @as(Node.Index, @intCast(i)); +} + +fn reserveNode(p: *Parse, tag: Ast.Node.Tag) !usize { + try p.nodes.resize(p.gpa, p.nodes.len + 1); + p.nodes.items(.tag)[p.nodes.len - 1] = tag; + return p.nodes.len - 1; +} + +fn unreserveNode(p: *Parse, node_index: usize) void { + if (p.nodes.len == node_index) { + p.nodes.resize(p.gpa, p.nodes.len - 1) catch unreachable; + } else { + // There is zombie node left in the tree, let's make it as inoffensive as possible + // (sadly there's no no-op node) + p.nodes.items(.tag)[node_index] = .unreachable_literal; + p.nodes.items(.main_token)[node_index] = p.tok_i; + } +} + +fn addExtra(p: *Parse, extra: anytype) Allocator.Error!Node.Index { + const fields = std.meta.fields(@TypeOf(extra)); + try p.extra_data.ensureUnusedCapacity(p.gpa, fields.len); + const result = @as(u32, @intCast(p.extra_data.items.len)); + inline for (fields) |field| { + comptime assert(field.type == Node.Index); + p.extra_data.appendAssumeCapacity(@field(extra, field.name)); + } + return result; +} + +fn warnExpected(p: *Parse, expected_token: Token.Tag) error{OutOfMemory}!void { + @setCold(true); + try p.warnMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = expected_token }, + }); +} + +fn warn(p: *Parse, error_tag: AstError.Tag) error{OutOfMemory}!void { + @setCold(true); + try p.warnMsg(.{ .tag = error_tag, .token = p.tok_i }); +} + +fn warnMsg(p: *Parse, msg: Ast.Error) error{OutOfMemory}!void { + @setCold(true); + switch (msg.tag) { + .expected_semi_after_decl, + .expected_semi_after_stmt, + .expected_comma_after_field, + .expected_comma_after_arg, + .expected_comma_after_param, + .expected_comma_after_initializer, + .expected_comma_after_switch_prong, + .expected_comma_after_for_operand, + .expected_comma_after_capture, + .expected_semi_or_else, + .expected_semi_or_lbrace, + .expected_token, + .expected_block, + .expected_block_or_assignment, + .expected_block_or_expr, + .expected_block_or_field, + .expected_expr, + .expected_expr_or_assignment, + .expected_fn, + .expected_inlinable, + .expected_labelable, + .expected_param_list, + .expected_prefix_expr, + .expected_primary_type_expr, + .expected_pub_item, + .expected_return_type, + .expected_suffix_op, + .expected_type_expr, + .expected_var_decl, + .expected_var_decl_or_fn, + .expected_loop_payload, + .expected_container, + => if (msg.token != 0 and !p.tokensOnSameLine(msg.token - 1, msg.token)) { + var copy = msg; + copy.token_is_prev = true; + copy.token -= 1; + return p.errors.append(p.gpa, copy); + }, + else => {}, + } + try p.errors.append(p.gpa, msg); +} + +fn fail(p: *Parse, tag: Ast.Error.Tag) error{ ParseError, OutOfMemory } { + @setCold(true); + return p.failMsg(.{ .tag = tag, .token = p.tok_i }); +} + +fn failExpected(p: *Parse, expected_token: Token.Tag) error{ ParseError, OutOfMemory } { + @setCold(true); + return p.failMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = expected_token }, + }); +} + +fn failMsg(p: *Parse, msg: Ast.Error) error{ ParseError, OutOfMemory } { + @setCold(true); + try p.warnMsg(msg); + return error.ParseError; +} + +/// Root <- skip container_doc_comment? ContainerMembers eof +pub fn parseRoot(p: *Parse) !void { + // Root node must be index 0. + p.nodes.appendAssumeCapacity(.{ + .tag = .root, + .main_token = 0, + .data = undefined, + }); + const root_members = try p.parseContainerMembers(); + const root_decls = try root_members.toSpan(p); + if (p.token_tags[p.tok_i] != .eof) { + try p.warnExpected(.eof); + } + p.nodes.items(.data)[0] = .{ + .lhs = root_decls.start, + .rhs = root_decls.end, + }; +} + +/// Parse in ZON mode. Subset of the language. +/// TODO: set a flag in Parse struct, and honor that flag +/// by emitting compilation errors when non-zon nodes are encountered. +pub fn parseZon(p: *Parse) !void { + // We must use index 0 so that 0 can be used as null elsewhere. + p.nodes.appendAssumeCapacity(.{ + .tag = .root, + .main_token = 0, + .data = undefined, + }); + const node_index = p.expectExpr() catch |err| switch (err) { + error.ParseError => { + assert(p.errors.items.len > 0); + return; + }, + else => |e| return e, + }; + if (p.token_tags[p.tok_i] != .eof) { + try p.warnExpected(.eof); + } + p.nodes.items(.data)[0] = .{ + .lhs = node_index, + .rhs = undefined, + }; +} + +/// ContainerMembers <- ContainerDeclaration* (ContainerField COMMA)* (ContainerField / ContainerDeclaration*) +/// +/// ContainerDeclaration <- TestDecl / ComptimeDecl / doc_comment? KEYWORD_pub? Decl +/// +/// ComptimeDecl <- KEYWORD_comptime Block +fn parseContainerMembers(p: *Parse) !Members { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + var field_state: union(enum) { + /// No fields have been seen. + none, + /// Currently parsing fields. + seen, + /// Saw fields and then a declaration after them. + /// Payload is first token of previous declaration. + end: Node.Index, + /// There was a declaration between fields, don't report more errors. + err, + } = .none; + + var last_field: TokenIndex = undefined; + + // Skip container doc comments. + while (p.eatToken(.container_doc_comment)) |_| {} + + var trailing = false; + while (true) { + const doc_comment = try p.eatDocComments(); + + switch (p.token_tags[p.tok_i]) { + .keyword_test => { + if (doc_comment) |some| { + try p.warnMsg(.{ .tag = .test_doc_comment, .token = some }); + } + const test_decl_node = try p.expectTestDeclRecoverable(); + if (test_decl_node != 0) { + if (field_state == .seen) { + field_state = .{ .end = test_decl_node }; + } + try p.scratch.append(p.gpa, test_decl_node); + } + trailing = false; + }, + .keyword_comptime => switch (p.token_tags[p.tok_i + 1]) { + .l_brace => { + if (doc_comment) |some| { + try p.warnMsg(.{ .tag = .comptime_doc_comment, .token = some }); + } + const comptime_token = p.nextToken(); + const block = p.parseBlock() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => blk: { + p.findNextContainerMember(); + break :blk null_node; + }, + }; + if (block != 0) { + const comptime_node = try p.addNode(.{ + .tag = .@"comptime", + .main_token = comptime_token, + .data = .{ + .lhs = block, + .rhs = undefined, + }, + }); + if (field_state == .seen) { + field_state = .{ .end = comptime_node }; + } + try p.scratch.append(p.gpa, comptime_node); + } + trailing = false; + }, + else => { + const identifier = p.tok_i; + defer last_field = identifier; + const container_field = p.expectContainerField() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }; + switch (field_state) { + .none => field_state = .seen, + .err, .seen => {}, + .end => |node| { + try p.warnMsg(.{ + .tag = .decl_between_fields, + .token = p.nodes.items(.main_token)[node], + }); + try p.warnMsg(.{ + .tag = .previous_field, + .is_note = true, + .token = last_field, + }); + try p.warnMsg(.{ + .tag = .next_field, + .is_note = true, + .token = identifier, + }); + // Continue parsing; error will be reported later. + field_state = .err; + }, + } + try p.scratch.append(p.gpa, container_field); + switch (p.token_tags[p.tok_i]) { + .comma => { + p.tok_i += 1; + trailing = true; + continue; + }, + .r_brace, .eof => { + trailing = false; + break; + }, + else => {}, + } + // There is not allowed to be a decl after a field with no comma. + // Report error but recover parser. + try p.warn(.expected_comma_after_field); + p.findNextContainerMember(); + }, + }, + .keyword_pub => { + p.tok_i += 1; + const top_level_decl = try p.expectTopLevelDeclRecoverable(); + if (top_level_decl != 0) { + if (field_state == .seen) { + field_state = .{ .end = top_level_decl }; + } + try p.scratch.append(p.gpa, top_level_decl); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .keyword_usingnamespace => { + const node = try p.expectUsingNamespaceRecoverable(); + if (node != 0) { + if (field_state == .seen) { + field_state = .{ .end = node }; + } + try p.scratch.append(p.gpa, node); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .keyword_const, + .keyword_var, + .keyword_threadlocal, + .keyword_export, + .keyword_extern, + .keyword_inline, + .keyword_noinline, + .keyword_fn, + => { + const top_level_decl = try p.expectTopLevelDeclRecoverable(); + if (top_level_decl != 0) { + if (field_state == .seen) { + field_state = .{ .end = top_level_decl }; + } + try p.scratch.append(p.gpa, top_level_decl); + } + trailing = p.token_tags[p.tok_i - 1] == .semicolon; + }, + .eof, .r_brace => { + if (doc_comment) |tok| { + try p.warnMsg(.{ + .tag = .unattached_doc_comment, + .token = tok, + }); + } + break; + }, + else => { + const c_container = p.parseCStyleContainer() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => false, + }; + if (c_container) continue; + + const identifier = p.tok_i; + defer last_field = identifier; + const container_field = p.expectContainerField() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }; + switch (field_state) { + .none => field_state = .seen, + .err, .seen => {}, + .end => |node| { + try p.warnMsg(.{ + .tag = .decl_between_fields, + .token = p.nodes.items(.main_token)[node], + }); + try p.warnMsg(.{ + .tag = .previous_field, + .is_note = true, + .token = last_field, + }); + try p.warnMsg(.{ + .tag = .next_field, + .is_note = true, + .token = identifier, + }); + // Continue parsing; error will be reported later. + field_state = .err; + }, + } + try p.scratch.append(p.gpa, container_field); + switch (p.token_tags[p.tok_i]) { + .comma => { + p.tok_i += 1; + trailing = true; + continue; + }, + .r_brace, .eof => { + trailing = false; + break; + }, + else => {}, + } + // There is not allowed to be a decl after a field with no comma. + // Report error but recover parser. + try p.warn(.expected_comma_after_field); + if (p.token_tags[p.tok_i] == .semicolon and p.token_tags[identifier] == .identifier) { + try p.warnMsg(.{ + .tag = .var_const_decl, + .is_note = true, + .token = identifier, + }); + } + p.findNextContainerMember(); + continue; + }, + } + } + + const items = p.scratch.items[scratch_top..]; + switch (items.len) { + 0 => return Members{ + .len = 0, + .lhs = 0, + .rhs = 0, + .trailing = trailing, + }, + 1 => return Members{ + .len = 1, + .lhs = items[0], + .rhs = 0, + .trailing = trailing, + }, + 2 => return Members{ + .len = 2, + .lhs = items[0], + .rhs = items[1], + .trailing = trailing, + }, + else => { + const span = try p.listToSpan(items); + return Members{ + .len = items.len, + .lhs = span.start, + .rhs = span.end, + .trailing = trailing, + }; + }, + } +} + +/// Attempts to find next container member by searching for certain tokens +fn findNextContainerMember(p: *Parse) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (p.token_tags[tok]) { + // Any of these can start a new top level declaration. + .keyword_test, + .keyword_comptime, + .keyword_pub, + .keyword_export, + .keyword_extern, + .keyword_inline, + .keyword_noinline, + .keyword_usingnamespace, + .keyword_threadlocal, + .keyword_const, + .keyword_var, + .keyword_fn, + => { + if (level == 0) { + p.tok_i -= 1; + return; + } + }, + .identifier => { + if (p.token_tags[tok + 1] == .comma and level == 0) { + p.tok_i -= 1; + return; + } + }, + .comma, .semicolon => { + // this decl was likely meant to end here + if (level == 0) { + return; + } + }, + .l_paren, .l_bracket, .l_brace => level += 1, + .r_paren, .r_bracket => { + if (level != 0) level -= 1; + }, + .r_brace => { + if (level == 0) { + // end of container, exit + p.tok_i -= 1; + return; + } + level -= 1; + }, + .eof => { + p.tok_i -= 1; + return; + }, + else => {}, + } + } +} + +/// Attempts to find the next statement by searching for a semicolon +fn findNextStmt(p: *Parse) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (p.token_tags[tok]) { + .l_brace => level += 1, + .r_brace => { + if (level == 0) { + p.tok_i -= 1; + return; + } + level -= 1; + }, + .semicolon => { + if (level == 0) { + return; + } + }, + .eof => { + p.tok_i -= 1; + return; + }, + else => {}, + } + } +} + +/// TestDecl <- KEYWORD_test (STRINGLITERALSINGLE / IDENTIFIER)? Block +fn expectTestDecl(p: *Parse) !Node.Index { + const test_token = p.assertToken(.keyword_test); + const name_token = switch (p.token_tags[p.nextToken()]) { + .string_literal, .identifier => p.tok_i - 1, + else => blk: { + p.tok_i -= 1; + break :blk null; + }, + }; + const block_node = try p.parseBlock(); + if (block_node == 0) return p.fail(.expected_block); + return p.addNode(.{ + .tag = .test_decl, + .main_token = test_token, + .data = .{ + .lhs = name_token orelse 0, + .rhs = block_node, + }, + }); +} + +fn expectTestDeclRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectTestDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +/// Decl +/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / KEYWORD_inline / KEYWORD_noinline)? FnProto (SEMICOLON / Block) +/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl +/// / KEYWORD_usingnamespace Expr SEMICOLON +fn expectTopLevelDecl(p: *Parse) !Node.Index { + const extern_export_inline_token = p.nextToken(); + var is_extern: bool = false; + var expect_fn: bool = false; + var expect_var_or_fn: bool = false; + switch (p.token_tags[extern_export_inline_token]) { + .keyword_extern => { + _ = p.eatToken(.string_literal); + is_extern = true; + expect_var_or_fn = true; + }, + .keyword_export => expect_var_or_fn = true, + .keyword_inline, .keyword_noinline => expect_fn = true, + else => p.tok_i -= 1, + } + const fn_proto = try p.parseFnProto(); + if (fn_proto != 0) { + switch (p.token_tags[p.tok_i]) { + .semicolon => { + p.tok_i += 1; + return fn_proto; + }, + .l_brace => { + if (is_extern) { + try p.warnMsg(.{ .tag = .extern_fn_body, .token = extern_export_inline_token }); + return null_node; + } + const fn_decl_index = try p.reserveNode(.fn_decl); + errdefer p.unreserveNode(fn_decl_index); + + const body_block = try p.parseBlock(); + assert(body_block != 0); + return p.setNode(fn_decl_index, .{ + .tag = .fn_decl, + .main_token = p.nodes.items(.main_token)[fn_proto], + .data = .{ + .lhs = fn_proto, + .rhs = body_block, + }, + }); + }, + else => { + // Since parseBlock only return error.ParseError on + // a missing '}' we can assume this function was + // supposed to end here. + try p.warn(.expected_semi_or_lbrace); + return null_node; + }, + } + } + if (expect_fn) { + try p.warn(.expected_fn); + return error.ParseError; + } + + const thread_local_token = p.eatToken(.keyword_threadlocal); + const var_decl = try p.parseGlobalVarDecl(); + if (var_decl != 0) { + return var_decl; + } + if (thread_local_token != null) { + return p.fail(.expected_var_decl); + } + if (expect_var_or_fn) { + return p.fail(.expected_var_decl_or_fn); + } + if (p.token_tags[p.tok_i] != .keyword_usingnamespace) { + return p.fail(.expected_pub_item); + } + return p.expectUsingNamespace(); +} + +fn expectTopLevelDeclRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectTopLevelDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +fn expectUsingNamespace(p: *Parse) !Node.Index { + const usingnamespace_token = p.assertToken(.keyword_usingnamespace); + const expr = try p.expectExpr(); + try p.expectSemicolon(.expected_semi_after_decl, false); + return p.addNode(.{ + .tag = .@"usingnamespace", + .main_token = usingnamespace_token, + .data = .{ + .lhs = expr, + .rhs = undefined, + }, + }); +} + +fn expectUsingNamespaceRecoverable(p: *Parse) error{OutOfMemory}!Node.Index { + return p.expectUsingNamespace() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + return null_node; + }, + }; +} + +/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? AddrSpace? LinkSection? CallConv? EXCLAMATIONMARK? TypeExpr +fn parseFnProto(p: *Parse) !Node.Index { + const fn_token = p.eatToken(.keyword_fn) orelse return null_node; + + // We want the fn proto node to be before its children in the array. + const fn_proto_index = try p.reserveNode(.fn_proto); + errdefer p.unreserveNode(fn_proto_index); + + _ = p.eatToken(.identifier); + const params = try p.parseParamDeclList(); + const align_expr = try p.parseByteAlign(); + const addrspace_expr = try p.parseAddrSpace(); + const section_expr = try p.parseLinkSection(); + const callconv_expr = try p.parseCallconv(); + _ = p.eatToken(.bang); + + const return_type_expr = try p.parseTypeExpr(); + if (return_type_expr == 0) { + // most likely the user forgot to specify the return type. + // Mark return type as invalid and try to continue. + try p.warn(.expected_return_type); + } + + if (align_expr == 0 and section_expr == 0 and callconv_expr == 0 and addrspace_expr == 0) { + switch (params) { + .zero_or_one => |param| return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_simple, + .main_token = fn_token, + .data = .{ + .lhs = param, + .rhs = return_type_expr, + }, + }), + .multi => |span| { + return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_multi, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.SubRange{ + .start = span.start, + .end = span.end, + }), + .rhs = return_type_expr, + }, + }); + }, + } + } + switch (params) { + .zero_or_one => |param| return p.setNode(fn_proto_index, .{ + .tag = .fn_proto_one, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.FnProtoOne{ + .param = param, + .align_expr = align_expr, + .addrspace_expr = addrspace_expr, + .section_expr = section_expr, + .callconv_expr = callconv_expr, + }), + .rhs = return_type_expr, + }, + }), + .multi => |span| { + return p.setNode(fn_proto_index, .{ + .tag = .fn_proto, + .main_token = fn_token, + .data = .{ + .lhs = try p.addExtra(Node.FnProto{ + .params_start = span.start, + .params_end = span.end, + .align_expr = align_expr, + .addrspace_expr = addrspace_expr, + .section_expr = section_expr, + .callconv_expr = callconv_expr, + }), + .rhs = return_type_expr, + }, + }); + }, + } +} + +/// VarDeclProto <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? AddrSpace? LinkSection? +/// Returns a `*_var_decl` node with its rhs (init expression) initialized to 0. +fn parseVarDeclProto(p: *Parse) !Node.Index { + const mut_token = p.eatToken(.keyword_const) orelse + p.eatToken(.keyword_var) orelse + return null_node; + + _ = try p.expectToken(.identifier); + const type_node: Node.Index = if (p.eatToken(.colon) == null) 0 else try p.expectTypeExpr(); + const align_node = try p.parseByteAlign(); + const addrspace_node = try p.parseAddrSpace(); + const section_node = try p.parseLinkSection(); + + if (section_node == 0 and addrspace_node == 0) { + if (align_node == 0) { + return p.addNode(.{ + .tag = .simple_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = type_node, + .rhs = 0, + }, + }); + } + + if (type_node == 0) { + return p.addNode(.{ + .tag = .aligned_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = align_node, + .rhs = 0, + }, + }); + } + + return p.addNode(.{ + .tag = .local_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = try p.addExtra(Node.LocalVarDecl{ + .type_node = type_node, + .align_node = align_node, + }), + .rhs = 0, + }, + }); + } else { + return p.addNode(.{ + .tag = .global_var_decl, + .main_token = mut_token, + .data = .{ + .lhs = try p.addExtra(Node.GlobalVarDecl{ + .type_node = type_node, + .align_node = align_node, + .addrspace_node = addrspace_node, + .section_node = section_node, + }), + .rhs = 0, + }, + }); + } +} + +/// GlobalVarDecl <- VarDeclProto (EQUAL Expr?) SEMICOLON +fn parseGlobalVarDecl(p: *Parse) !Node.Index { + const var_decl = try p.parseVarDeclProto(); + if (var_decl == 0) { + return null_node; + } + + const init_node: Node.Index = switch (p.token_tags[p.tok_i]) { + .equal_equal => blk: { + try p.warn(.wrong_equal_var_decl); + p.tok_i += 1; + break :blk try p.expectExpr(); + }, + .equal => blk: { + p.tok_i += 1; + break :blk try p.expectExpr(); + }, + else => 0, + }; + + p.nodes.items(.data)[var_decl].rhs = init_node; + + try p.expectSemicolon(.expected_semi_after_decl, false); + return var_decl; +} + +/// ContainerField +/// <- doc_comment? KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? ByteAlign? (EQUAL Expr)? +/// / doc_comment? KEYWORD_comptime? (IDENTIFIER COLON)? !KEYWORD_fn TypeExpr ByteAlign? (EQUAL Expr)? +fn expectContainerField(p: *Parse) !Node.Index { + var main_token = p.tok_i; + _ = p.eatToken(.keyword_comptime); + const tuple_like = p.token_tags[p.tok_i] != .identifier or p.token_tags[p.tok_i + 1] != .colon; + if (!tuple_like) { + main_token = p.assertToken(.identifier); + } + + var align_expr: Node.Index = 0; + var type_expr: Node.Index = 0; + if (p.eatToken(.colon) != null or tuple_like) { + type_expr = try p.expectTypeExpr(); + align_expr = try p.parseByteAlign(); + } + + const value_expr: Node.Index = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); + + if (align_expr == 0) { + return p.addNode(.{ + .tag = .container_field_init, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = value_expr, + }, + }); + } else if (value_expr == 0) { + return p.addNode(.{ + .tag = .container_field_align, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = align_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .container_field, + .main_token = main_token, + .data = .{ + .lhs = type_expr, + .rhs = try p.addExtra(Node.ContainerField{ + .value_expr = value_expr, + .align_expr = align_expr, + }), + }, + }); + } +} + +/// Statement +/// <- KEYWORD_comptime ComptimeStatement +/// / KEYWORD_nosuspend BlockExprStatement +/// / KEYWORD_suspend BlockExprStatement +/// / KEYWORD_defer BlockExprStatement +/// / KEYWORD_errdefer Payload? BlockExprStatement +/// / IfStatement +/// / LabeledStatement +/// / SwitchExpr +/// / VarDeclExprStatement +fn expectStatement(p: *Parse, allow_defer_var: bool) Error!Node.Index { + if (p.eatToken(.keyword_comptime)) |comptime_token| { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) { + return p.addNode(.{ + .tag = .@"comptime", + .main_token = comptime_token, + .data = .{ + .lhs = block_expr, + .rhs = undefined, + }, + }); + } + + if (allow_defer_var) { + return p.expectVarDeclExprStatement(comptime_token); + } else { + const assign = try p.expectAssignExpr(); + try p.expectSemicolon(.expected_semi_after_stmt, true); + return assign; + } + } + + switch (p.token_tags[p.tok_i]) { + .keyword_nosuspend => { + return p.addNode(.{ + .tag = .@"nosuspend", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectBlockExprStatement(), + .rhs = undefined, + }, + }); + }, + .keyword_suspend => { + const token = p.nextToken(); + const block_expr = try p.expectBlockExprStatement(); + return p.addNode(.{ + .tag = .@"suspend", + .main_token = token, + .data = .{ + .lhs = block_expr, + .rhs = undefined, + }, + }); + }, + .keyword_defer => if (allow_defer_var) return p.addNode(.{ + .tag = .@"defer", + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = try p.expectBlockExprStatement(), + }, + }), + .keyword_errdefer => if (allow_defer_var) return p.addNode(.{ + .tag = .@"errdefer", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.parsePayload(), + .rhs = try p.expectBlockExprStatement(), + }, + }), + .keyword_switch => return p.expectSwitchExpr(), + .keyword_if => return p.expectIfStatement(), + .keyword_enum, .keyword_struct, .keyword_union => { + const identifier = p.tok_i + 1; + if (try p.parseCStyleContainer()) { + // Return something so that `expectStatement` is happy. + return p.addNode(.{ + .tag = .identifier, + .main_token = identifier, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + } + }, + else => {}, + } + + const labeled_statement = try p.parseLabeledStatement(); + if (labeled_statement != 0) return labeled_statement; + + if (allow_defer_var) { + return p.expectVarDeclExprStatement(null); + } else { + const assign = try p.expectAssignExpr(); + try p.expectSemicolon(.expected_semi_after_stmt, true); + return assign; + } +} + +/// ComptimeStatement +/// <- BlockExpr +/// / VarDeclExprStatement +fn expectComptimeStatement(p: *Parse, comptime_token: TokenIndex) !Node.Index { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) { + return p.addNode(.{ + .tag = .@"comptime", + .main_token = comptime_token, + .data = .{ .lhs = block_expr, .rhs = undefined }, + }); + } + return p.expectVarDeclExprStatement(comptime_token); +} + +/// VarDeclExprStatement +/// <- VarDeclProto (COMMA (VarDeclProto / Expr))* EQUAL Expr SEMICOLON +/// / Expr (AssignOp Expr / (COMMA (VarDeclProto / Expr))+ EQUAL Expr)? SEMICOLON +fn expectVarDeclExprStatement(p: *Parse, comptime_token: ?TokenIndex) !Node.Index { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const var_decl_proto = try p.parseVarDeclProto(); + if (var_decl_proto != 0) { + try p.scratch.append(p.gpa, var_decl_proto); + } else { + const expr = try p.parseExpr(); + if (expr == 0) { + if (p.scratch.items.len == scratch_top) { + // We parsed nothing + return p.fail(.expected_statement); + } else { + // We've had at least one LHS, but had a bad comma + return p.fail(.expected_expr_or_var_decl); + } + } + try p.scratch.append(p.gpa, expr); + } + _ = p.eatToken(.comma) orelse break; + } + + const lhs_count = p.scratch.items.len - scratch_top; + assert(lhs_count > 0); + + const equal_token = p.eatToken(.equal) orelse eql: { + if (lhs_count > 1) { + // Definitely a destructure, so allow recovering from == + if (p.eatToken(.equal_equal)) |tok| { + try p.warnMsg(.{ .tag = .wrong_equal_var_decl, .token = tok }); + break :eql tok; + } + return p.failExpected(.equal); + } + const lhs = p.scratch.items[scratch_top]; + switch (p.nodes.items(.tag)[lhs]) { + .global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => { + // Definitely a var decl, so allow recovering from == + if (p.eatToken(.equal_equal)) |tok| { + try p.warnMsg(.{ .tag = .wrong_equal_var_decl, .token = tok }); + break :eql tok; + } + return p.failExpected(.equal); + }, + else => {}, + } + + const expr = try p.finishAssignExpr(lhs); + try p.expectSemicolon(.expected_semi_after_stmt, true); + if (comptime_token) |t| { + return p.addNode(.{ + .tag = .@"comptime", + .main_token = t, + .data = .{ + .lhs = expr, + .rhs = undefined, + }, + }); + } else { + return expr; + } + }; + + const rhs = try p.expectExpr(); + try p.expectSemicolon(.expected_semi_after_stmt, true); + + if (lhs_count == 1) { + const lhs = p.scratch.items[scratch_top]; + switch (p.nodes.items(.tag)[lhs]) { + .global_var_decl, .local_var_decl, .simple_var_decl, .aligned_var_decl => { + p.nodes.items(.data)[lhs].rhs = rhs; + // Don't need to wrap in comptime + return lhs; + }, + else => {}, + } + const expr = try p.addNode(.{ + .tag = .assign, + .main_token = equal_token, + .data = .{ .lhs = lhs, .rhs = rhs }, + }); + if (comptime_token) |t| { + return p.addNode(.{ + .tag = .@"comptime", + .main_token = t, + .data = .{ + .lhs = expr, + .rhs = undefined, + }, + }); + } else { + return expr; + } + } + + // An actual destructure! No need for any `comptime` wrapper here. + + const extra_start = p.extra_data.items.len; + try p.extra_data.ensureUnusedCapacity(p.gpa, lhs_count + 1); + p.extra_data.appendAssumeCapacity(@intCast(lhs_count)); + p.extra_data.appendSliceAssumeCapacity(p.scratch.items[scratch_top..]); + + return p.addNode(.{ + .tag = .assign_destructure, + .main_token = equal_token, + .data = .{ + .lhs = @intCast(extra_start), + .rhs = rhs, + }, + }); +} + +/// If a parse error occurs, reports an error, but then finds the next statement +/// and returns that one instead. If a parse error occurs but there is no following +/// statement, returns 0. +fn expectStatementRecoverable(p: *Parse) Error!Node.Index { + while (true) { + return p.expectStatement(true) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextStmt(); // Try to skip to the next statement. + switch (p.token_tags[p.tok_i]) { + .r_brace => return null_node, + .eof => return error.ParseError, + else => continue, + } + }, + }; + } +} + +/// IfStatement +/// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? +/// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) +fn expectIfStatement(p: *Parse) !Node.Index { + const if_token = p.assertToken(.keyword_if); + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + + // TODO propose to change the syntax so that semicolons are always required + // inside if statements, even if there is an `else`. + var else_required = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = assign_expr, + }, + }); + } + else_required = true; + break :blk assign_expr; + }; + _ = p.eatToken(.keyword_else) orelse { + if (else_required) { + try p.warn(.expected_semi_or_else); + } + return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + }; + _ = try p.parsePayload(); + const else_expr = try p.expectStatement(false); + return p.addNode(.{ + .tag = .@"if", + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// LabeledStatement <- BlockLabel? (Block / LoopStatement) +fn parseLabeledStatement(p: *Parse) !Node.Index { + const label_token = p.parseBlockLabel(); + const block = try p.parseBlock(); + if (block != 0) return block; + + const loop_stmt = try p.parseLoopStatement(); + if (loop_stmt != 0) return loop_stmt; + + if (label_token != 0) { + const after_colon = p.tok_i; + const node = try p.parseTypeExpr(); + if (node != 0) { + const a = try p.parseByteAlign(); + const b = try p.parseAddrSpace(); + const c = try p.parseLinkSection(); + const d = if (p.eatToken(.equal) == null) 0 else try p.expectExpr(); + if (a != 0 or b != 0 or c != 0 or d != 0) { + return p.failMsg(.{ .tag = .expected_var_const, .token = label_token }); + } + } + return p.failMsg(.{ .tag = .expected_labelable, .token = after_colon }); + } + + return null_node; +} + +/// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) +fn parseLoopStatement(p: *Parse) !Node.Index { + const inline_token = p.eatToken(.keyword_inline); + + const for_statement = try p.parseForStatement(); + if (for_statement != 0) return for_statement; + + const while_statement = try p.parseWhileStatement(); + if (while_statement != 0) return while_statement; + + if (inline_token == null) return null_node; + + // If we've seen "inline", there should have been a "for" or "while" + return p.fail(.expected_inlinable); +} + +/// ForStatement +/// <- ForPrefix BlockExpr ( KEYWORD_else Statement )? +/// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement ) +fn parseForStatement(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const inputs = try p.forPrefix(); + + var else_required = false; + var seen_semicolon = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + seen_semicolon = true; + break :blk assign_expr; + } + else_required = true; + break :blk assign_expr; + }; + var has_else = false; + if (!seen_semicolon and p.eatToken(.keyword_else) != null) { + try p.scratch.append(p.gpa, then_expr); + const else_stmt = try p.expectStatement(false); + try p.scratch.append(p.gpa, else_stmt); + has_else = true; + } else if (inputs == 1) { + if (else_required) try p.warn(.expected_semi_or_else); + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = p.scratch.items[scratch_top], + .rhs = then_expr, + }, + }); + } else { + if (else_required) try p.warn(.expected_semi_or_else); + try p.scratch.append(p.gpa, then_expr); + } + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = (try p.listToSpan(p.scratch.items[scratch_top..])).start, + .rhs = @as(u32, @bitCast(Node.For{ + .inputs = @as(u31, @intCast(inputs)), + .has_else = has_else, + })), + }, + }); +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileStatement +/// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? +/// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) +fn parseWhileStatement(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + // TODO propose to change the syntax so that semicolons are always required + // inside while statements, even if there is an `else`. + var else_required = false; + const then_expr = blk: { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) break :blk block_expr; + const assign_expr = try p.parseAssignExpr(); + if (assign_expr == 0) { + return p.fail(.expected_block_or_assignment); + } + if (p.eatToken(.semicolon)) |_| { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = assign_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = assign_expr, + }), + }, + }); + } + } + else_required = true; + break :blk assign_expr; + }; + _ = p.eatToken(.keyword_else) orelse { + if (else_required) { + try p.warn(.expected_semi_or_else); + } + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectStatement(false); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// BlockExprStatement +/// <- BlockExpr +/// / AssignExpr SEMICOLON +fn parseBlockExprStatement(p: *Parse) !Node.Index { + const block_expr = try p.parseBlockExpr(); + if (block_expr != 0) { + return block_expr; + } + const assign_expr = try p.parseAssignExpr(); + if (assign_expr != 0) { + try p.expectSemicolon(.expected_semi_after_stmt, true); + return assign_expr; + } + return null_node; +} + +fn expectBlockExprStatement(p: *Parse) !Node.Index { + const node = try p.parseBlockExprStatement(); + if (node == 0) { + return p.fail(.expected_block_or_expr); + } + return node; +} + +/// BlockExpr <- BlockLabel? Block +fn parseBlockExpr(p: *Parse) Error!Node.Index { + switch (p.token_tags[p.tok_i]) { + .identifier => { + if (p.token_tags[p.tok_i + 1] == .colon and + p.token_tags[p.tok_i + 2] == .l_brace) + { + p.tok_i += 2; + return p.parseBlock(); + } else { + return null_node; + } + }, + .l_brace => return p.parseBlock(), + else => return null_node, + } +} + +/// AssignExpr <- Expr (AssignOp Expr / (COMMA Expr)+ EQUAL Expr)? +/// +/// AssignOp +/// <- ASTERISKEQUAL +/// / ASTERISKPIPEEQUAL +/// / SLASHEQUAL +/// / PERCENTEQUAL +/// / PLUSEQUAL +/// / PLUSPIPEEQUAL +/// / MINUSEQUAL +/// / MINUSPIPEEQUAL +/// / LARROW2EQUAL +/// / LARROW2PIPEEQUAL +/// / RARROW2EQUAL +/// / AMPERSANDEQUAL +/// / CARETEQUAL +/// / PIPEEQUAL +/// / ASTERISKPERCENTEQUAL +/// / PLUSPERCENTEQUAL +/// / MINUSPERCENTEQUAL +/// / EQUAL +fn parseAssignExpr(p: *Parse) !Node.Index { + const expr = try p.parseExpr(); + if (expr == 0) return null_node; + return p.finishAssignExpr(expr); +} + +/// SingleAssignExpr <- Expr (AssignOp Expr)? +fn parseSingleAssignExpr(p: *Parse) !Node.Index { + const lhs = try p.parseExpr(); + if (lhs == 0) return null_node; + const tag = assignOpNode(p.token_tags[p.tok_i]) orelse return lhs; + return p.addNode(.{ + .tag = tag, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = try p.expectExpr(), + }, + }); +} + +fn finishAssignExpr(p: *Parse, lhs: Node.Index) !Node.Index { + const tok = p.token_tags[p.tok_i]; + if (tok == .comma) return p.finishAssignDestructureExpr(lhs); + const tag = assignOpNode(tok) orelse return lhs; + return p.addNode(.{ + .tag = tag, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = try p.expectExpr(), + }, + }); +} + +fn assignOpNode(tok: Token.Tag) ?Node.Tag { + return switch (tok) { + .asterisk_equal => .assign_mul, + .slash_equal => .assign_div, + .percent_equal => .assign_mod, + .plus_equal => .assign_add, + .minus_equal => .assign_sub, + .angle_bracket_angle_bracket_left_equal => .assign_shl, + .angle_bracket_angle_bracket_left_pipe_equal => .assign_shl_sat, + .angle_bracket_angle_bracket_right_equal => .assign_shr, + .ampersand_equal => .assign_bit_and, + .caret_equal => .assign_bit_xor, + .pipe_equal => .assign_bit_or, + .asterisk_percent_equal => .assign_mul_wrap, + .plus_percent_equal => .assign_add_wrap, + .minus_percent_equal => .assign_sub_wrap, + .asterisk_pipe_equal => .assign_mul_sat, + .plus_pipe_equal => .assign_add_sat, + .minus_pipe_equal => .assign_sub_sat, + .equal => .assign, + else => null, + }; +} + +fn finishAssignDestructureExpr(p: *Parse, first_lhs: Node.Index) !Node.Index { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + try p.scratch.append(p.gpa, first_lhs); + + while (p.eatToken(.comma)) |_| { + const expr = try p.expectExpr(); + try p.scratch.append(p.gpa, expr); + } + + const equal_token = try p.expectToken(.equal); + + const rhs = try p.expectExpr(); + + const lhs_count = p.scratch.items.len - scratch_top; + assert(lhs_count > 1); // we already had first_lhs, and must have at least one more lvalue + + const extra_start = p.extra_data.items.len; + try p.extra_data.ensureUnusedCapacity(p.gpa, lhs_count + 1); + p.extra_data.appendAssumeCapacity(@intCast(lhs_count)); + p.extra_data.appendSliceAssumeCapacity(p.scratch.items[scratch_top..]); + + return p.addNode(.{ + .tag = .assign_destructure, + .main_token = equal_token, + .data = .{ + .lhs = @intCast(extra_start), + .rhs = rhs, + }, + }); +} + +fn expectSingleAssignExpr(p: *Parse) !Node.Index { + const expr = try p.parseSingleAssignExpr(); + if (expr == 0) { + return p.fail(.expected_expr_or_assignment); + } + return expr; +} + +fn expectAssignExpr(p: *Parse) !Node.Index { + const expr = try p.parseAssignExpr(); + if (expr == 0) { + return p.fail(.expected_expr_or_assignment); + } + return expr; +} + +fn parseExpr(p: *Parse) Error!Node.Index { + return p.parseExprPrecedence(0); +} + +fn expectExpr(p: *Parse) Error!Node.Index { + const node = try p.parseExpr(); + if (node == 0) { + return p.fail(.expected_expr); + } else { + return node; + } +} + +const Assoc = enum { + left, + none, +}; + +const OperInfo = struct { + prec: i8, + tag: Node.Tag, + assoc: Assoc = Assoc.left, +}; + +// A table of binary operator information. Higher precedence numbers are +// stickier. All operators at the same precedence level should have the same +// associativity. +const operTable = std.enums.directEnumArrayDefault(Token.Tag, OperInfo, .{ .prec = -1, .tag = Node.Tag.root }, 0, .{ + .keyword_or = .{ .prec = 10, .tag = .bool_or }, + + .keyword_and = .{ .prec = 20, .tag = .bool_and }, + + .equal_equal = .{ .prec = 30, .tag = .equal_equal, .assoc = Assoc.none }, + .bang_equal = .{ .prec = 30, .tag = .bang_equal, .assoc = Assoc.none }, + .angle_bracket_left = .{ .prec = 30, .tag = .less_than, .assoc = Assoc.none }, + .angle_bracket_right = .{ .prec = 30, .tag = .greater_than, .assoc = Assoc.none }, + .angle_bracket_left_equal = .{ .prec = 30, .tag = .less_or_equal, .assoc = Assoc.none }, + .angle_bracket_right_equal = .{ .prec = 30, .tag = .greater_or_equal, .assoc = Assoc.none }, + + .ampersand = .{ .prec = 40, .tag = .bit_and }, + .caret = .{ .prec = 40, .tag = .bit_xor }, + .pipe = .{ .prec = 40, .tag = .bit_or }, + .keyword_orelse = .{ .prec = 40, .tag = .@"orelse" }, + .keyword_catch = .{ .prec = 40, .tag = .@"catch" }, + + .angle_bracket_angle_bracket_left = .{ .prec = 50, .tag = .shl }, + .angle_bracket_angle_bracket_left_pipe = .{ .prec = 50, .tag = .shl_sat }, + .angle_bracket_angle_bracket_right = .{ .prec = 50, .tag = .shr }, + + .plus = .{ .prec = 60, .tag = .add }, + .minus = .{ .prec = 60, .tag = .sub }, + .plus_plus = .{ .prec = 60, .tag = .array_cat }, + .plus_percent = .{ .prec = 60, .tag = .add_wrap }, + .minus_percent = .{ .prec = 60, .tag = .sub_wrap }, + .plus_pipe = .{ .prec = 60, .tag = .add_sat }, + .minus_pipe = .{ .prec = 60, .tag = .sub_sat }, + + .pipe_pipe = .{ .prec = 70, .tag = .merge_error_sets }, + .asterisk = .{ .prec = 70, .tag = .mul }, + .slash = .{ .prec = 70, .tag = .div }, + .percent = .{ .prec = 70, .tag = .mod }, + .asterisk_asterisk = .{ .prec = 70, .tag = .array_mult }, + .asterisk_percent = .{ .prec = 70, .tag = .mul_wrap }, + .asterisk_pipe = .{ .prec = 70, .tag = .mul_sat }, +}); + +fn parseExprPrecedence(p: *Parse, min_prec: i32) Error!Node.Index { + assert(min_prec >= 0); + var node = try p.parsePrefixExpr(); + if (node == 0) { + return null_node; + } + + var banned_prec: i8 = -1; + + while (true) { + const tok_tag = p.token_tags[p.tok_i]; + const info = operTable[@as(usize, @intCast(@intFromEnum(tok_tag)))]; + if (info.prec < min_prec) { + break; + } + if (info.prec == banned_prec) { + return p.fail(.chained_comparison_operators); + } + + const oper_token = p.nextToken(); + // Special-case handling for "catch" + if (tok_tag == .keyword_catch) { + _ = try p.parsePayload(); + } + const rhs = try p.parseExprPrecedence(info.prec + 1); + if (rhs == 0) { + try p.warn(.expected_expr); + return node; + } + + { + const tok_len = tok_tag.lexeme().?.len; + const char_before = p.source[p.token_starts[oper_token] - 1]; + const char_after = p.source[p.token_starts[oper_token] + tok_len]; + if (tok_tag == .ampersand and char_after == '&') { + // without types we don't know if '&&' was intended as 'bitwise_and address_of', or a c-style logical_and + // The best the parser can do is recommend changing it to 'and' or ' & &' + try p.warnMsg(.{ .tag = .invalid_ampersand_ampersand, .token = oper_token }); + } else if (std.ascii.isWhitespace(char_before) != std.ascii.isWhitespace(char_after)) { + try p.warnMsg(.{ .tag = .mismatched_binary_op_whitespace, .token = oper_token }); + } + } + + node = try p.addNode(.{ + .tag = info.tag, + .main_token = oper_token, + .data = .{ + .lhs = node, + .rhs = rhs, + }, + }); + + if (info.assoc == Assoc.none) { + banned_prec = info.prec; + } + } + + return node; +} + +/// PrefixExpr <- PrefixOp* PrimaryExpr +/// +/// PrefixOp +/// <- EXCLAMATIONMARK +/// / MINUS +/// / TILDE +/// / MINUSPERCENT +/// / AMPERSAND +/// / KEYWORD_try +/// / KEYWORD_await +fn parsePrefixExpr(p: *Parse) Error!Node.Index { + const tag: Node.Tag = switch (p.token_tags[p.tok_i]) { + .bang => .bool_not, + .minus => .negation, + .tilde => .bit_not, + .minus_percent => .negation_wrap, + .ampersand => .address_of, + .keyword_try => .@"try", + .keyword_await => .@"await", + else => return p.parsePrimaryExpr(), + }; + return p.addNode(.{ + .tag = tag, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectPrefixExpr(), + .rhs = undefined, + }, + }); +} + +fn expectPrefixExpr(p: *Parse) Error!Node.Index { + const node = try p.parsePrefixExpr(); + if (node == 0) { + return p.fail(.expected_prefix_expr); + } + return node; +} + +/// TypeExpr <- PrefixTypeOp* ErrorUnionExpr +/// +/// PrefixTypeOp +/// <- QUESTIONMARK +/// / KEYWORD_anyframe MINUSRARROW +/// / SliceTypeStart (ByteAlign / AddrSpace / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* +/// / PtrTypeStart (AddrSpace / KEYWORD_align LPAREN Expr (COLON Expr COLON Expr)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* +/// / ArrayTypeStart +/// +/// SliceTypeStart <- LBRACKET (COLON Expr)? RBRACKET +/// +/// PtrTypeStart +/// <- ASTERISK +/// / ASTERISK2 +/// / LBRACKET ASTERISK (LETTERC / COLON Expr)? RBRACKET +/// +/// ArrayTypeStart <- LBRACKET Expr (COLON Expr)? RBRACKET +fn parseTypeExpr(p: *Parse) Error!Node.Index { + switch (p.token_tags[p.tok_i]) { + .question_mark => return p.addNode(.{ + .tag = .optional_type, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectTypeExpr(), + .rhs = undefined, + }, + }), + .keyword_anyframe => switch (p.token_tags[p.tok_i + 1]) { + .arrow => return p.addNode(.{ + .tag = .anyframe_type, + .main_token = p.nextToken(), + .data = .{ + .lhs = p.nextToken(), + .rhs = try p.expectTypeExpr(), + }, + }), + else => return p.parseErrorUnionExpr(), + }, + .asterisk => { + const asterisk = p.nextToken(); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start != 0) { + return p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } else if (mods.addrspace_node != 0) { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } + }, + .asterisk_asterisk => { + const asterisk = p.nextToken(); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + const inner: Node.Index = inner: { + if (mods.bit_range_start != 0) { + break :inner try p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } else if (mods.addrspace_node != 0) { + break :inner try p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = 0, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } else { + break :inner try p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } + }; + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = 0, + .rhs = inner, + }, + }); + }, + .l_bracket => switch (p.token_tags[p.tok_i + 1]) { + .asterisk => { + _ = p.nextToken(); + const asterisk = p.nextToken(); + var sentinel: Node.Index = 0; + if (p.eatToken(.identifier)) |ident| { + const ident_slice = p.source[p.token_starts[ident]..p.token_starts[ident + 1]]; + if (!std.mem.eql(u8, std.mem.trimRight(u8, ident_slice, &std.ascii.whitespace), "c")) { + p.tok_i -= 1; + } + } else if (p.eatToken(.colon)) |_| { + sentinel = try p.expectExpr(); + } + _ = try p.expectToken(.r_bracket); + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start == 0) { + if (sentinel == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = asterisk, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } else if (mods.align_node == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_sentinel, + .main_token = asterisk, + .data = .{ + .lhs = sentinel, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } + } else { + return p.addNode(.{ + .tag = .ptr_type_bit_range, + .main_token = asterisk, + .data = .{ + .lhs = try p.addExtra(Node.PtrTypeBitRange{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + .bit_range_start = mods.bit_range_start, + .bit_range_end = mods.bit_range_end, + }), + .rhs = elem_type, + }, + }); + } + }, + else => { + const lbracket = p.nextToken(); + const len_expr = try p.parseExpr(); + const sentinel: Node.Index = if (p.eatToken(.colon)) |_| + try p.expectExpr() + else + 0; + _ = try p.expectToken(.r_bracket); + if (len_expr == 0) { + const mods = try p.parsePtrModifiers(); + const elem_type = try p.expectTypeExpr(); + if (mods.bit_range_start != 0) { + try p.warnMsg(.{ + .tag = .invalid_bit_range, + .token = p.nodes.items(.main_token)[mods.bit_range_start], + }); + } + if (sentinel == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_aligned, + .main_token = lbracket, + .data = .{ + .lhs = mods.align_node, + .rhs = elem_type, + }, + }); + } else if (mods.align_node == 0 and mods.addrspace_node == 0) { + return p.addNode(.{ + .tag = .ptr_type_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = sentinel, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .ptr_type, + .main_token = lbracket, + .data = .{ + .lhs = try p.addExtra(Node.PtrType{ + .sentinel = sentinel, + .align_node = mods.align_node, + .addrspace_node = mods.addrspace_node, + }), + .rhs = elem_type, + }, + }); + } + } else { + switch (p.token_tags[p.tok_i]) { + .keyword_align, + .keyword_const, + .keyword_volatile, + .keyword_allowzero, + .keyword_addrspace, + => return p.fail(.ptr_mod_on_array_child_type), + else => {}, + } + const elem_type = try p.expectTypeExpr(); + if (sentinel == 0) { + return p.addNode(.{ + .tag = .array_type, + .main_token = lbracket, + .data = .{ + .lhs = len_expr, + .rhs = elem_type, + }, + }); + } else { + return p.addNode(.{ + .tag = .array_type_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = len_expr, + .rhs = try p.addExtra(.{ + .elem_type = elem_type, + .sentinel = sentinel, + }), + }, + }); + } + } + }, + }, + else => return p.parseErrorUnionExpr(), + } +} + +fn expectTypeExpr(p: *Parse) Error!Node.Index { + const node = try p.parseTypeExpr(); + if (node == 0) { + return p.fail(.expected_type_expr); + } + return node; +} + +/// PrimaryExpr +/// <- AsmExpr +/// / IfExpr +/// / KEYWORD_break BreakLabel? Expr? +/// / KEYWORD_comptime Expr +/// / KEYWORD_nosuspend Expr +/// / KEYWORD_continue BreakLabel? +/// / KEYWORD_resume Expr +/// / KEYWORD_return Expr? +/// / BlockLabel? LoopExpr +/// / Block +/// / CurlySuffixExpr +fn parsePrimaryExpr(p: *Parse) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .keyword_asm => return p.expectAsmExpr(), + .keyword_if => return p.parseIfExpr(), + .keyword_break => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"break", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseBreakLabel(), + .rhs = try p.parseExpr(), + }, + }); + }, + .keyword_continue => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"continue", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseBreakLabel(), + .rhs = undefined, + }, + }); + }, + .keyword_comptime => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"comptime", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_nosuspend => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"nosuspend", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_resume => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"resume", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.expectExpr(), + .rhs = undefined, + }, + }); + }, + .keyword_return => { + p.tok_i += 1; + return p.addNode(.{ + .tag = .@"return", + .main_token = p.tok_i - 1, + .data = .{ + .lhs = try p.parseExpr(), + .rhs = undefined, + }, + }); + }, + .identifier => { + if (p.token_tags[p.tok_i + 1] == .colon) { + switch (p.token_tags[p.tok_i + 2]) { + .keyword_inline => { + p.tok_i += 3; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => { + p.tok_i += 2; + return p.parseForExpr(); + }, + .keyword_while => { + p.tok_i += 2; + return p.parseWhileExpr(); + }, + .l_brace => { + p.tok_i += 2; + return p.parseBlock(); + }, + else => return p.parseCurlySuffixExpr(), + } + } else { + return p.parseCurlySuffixExpr(); + } + }, + .keyword_inline => { + p.tok_i += 1; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => return p.parseForExpr(), + .keyword_while => return p.parseWhileExpr(), + .l_brace => return p.parseBlock(), + else => return p.parseCurlySuffixExpr(), + } +} + +/// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? +fn parseIfExpr(p: *Parse) !Node.Index { + return p.parseIf(expectExpr); +} + +/// Block <- LBRACE Statement* RBRACE +fn parseBlock(p: *Parse) !Node.Index { + const lbrace = p.eatToken(.l_brace) orelse return null_node; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.token_tags[p.tok_i] == .r_brace) break; + const statement = try p.expectStatementRecoverable(); + if (statement == 0) break; + try p.scratch.append(p.gpa, statement); + } + _ = try p.expectToken(.r_brace); + const semicolon = (p.token_tags[p.tok_i - 2] == .semicolon); + const statements = p.scratch.items[scratch_top..]; + switch (statements.len) { + 0 => return p.addNode(.{ + .tag = .block_two, + .main_token = lbrace, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (semicolon) .block_two_semicolon else .block_two, + .main_token = lbrace, + .data = .{ + .lhs = statements[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (semicolon) .block_two_semicolon else .block_two, + .main_token = lbrace, + .data = .{ + .lhs = statements[0], + .rhs = statements[1], + }, + }), + else => { + const span = try p.listToSpan(statements); + return p.addNode(.{ + .tag = if (semicolon) .block_semicolon else .block, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } +} + +/// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)? +fn parseForExpr(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const inputs = try p.forPrefix(); + + const then_expr = try p.expectExpr(); + var has_else = false; + if (p.eatToken(.keyword_else)) |_| { + try p.scratch.append(p.gpa, then_expr); + const else_expr = try p.expectExpr(); + try p.scratch.append(p.gpa, else_expr); + has_else = true; + } else if (inputs == 1) { + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = p.scratch.items[scratch_top], + .rhs = then_expr, + }, + }); + } else { + try p.scratch.append(p.gpa, then_expr); + } + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = (try p.listToSpan(p.scratch.items[scratch_top..])).start, + .rhs = @as(u32, @bitCast(Node.For{ + .inputs = @as(u31, @intCast(inputs)), + .has_else = has_else, + })), + }, + }); +} + +/// ForPrefix <- KEYWORD_for LPAREN ForInput (COMMA ForInput)* COMMA? RPAREN ForPayload +/// +/// ForInput <- Expr (DOT2 Expr?)? +/// +/// ForPayload <- PIPE ASTERISK? IDENTIFIER (COMMA ASTERISK? IDENTIFIER)* PIPE +fn forPrefix(p: *Parse) Error!usize { + const start = p.scratch.items.len; + _ = try p.expectToken(.l_paren); + + while (true) { + var input = try p.expectExpr(); + if (p.eatToken(.ellipsis2)) |ellipsis| { + input = try p.addNode(.{ + .tag = .for_range, + .main_token = ellipsis, + .data = .{ + .lhs = input, + .rhs = try p.parseExpr(), + }, + }); + } + + try p.scratch.append(p.gpa, input); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_for_operand), + } + if (p.eatToken(.r_paren)) |_| break; + } + const inputs = p.scratch.items.len - start; + + _ = p.eatToken(.pipe) orelse { + try p.warn(.expected_loop_payload); + return inputs; + }; + + var warned_excess = false; + var captures: u32 = 0; + while (true) { + _ = p.eatToken(.asterisk); + const identifier = try p.expectToken(.identifier); + captures += 1; + if (captures > inputs and !warned_excess) { + try p.warnMsg(.{ .tag = .extra_for_capture, .token = identifier }); + warned_excess = true; + } + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .pipe => { + p.tok_i += 1; + break; + }, + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_capture), + } + if (p.eatToken(.pipe)) |_| break; + } + + if (captures < inputs) { + const index = p.scratch.items.len - captures; + const input = p.nodes.items(.main_token)[p.scratch.items[index]]; + try p.warnMsg(.{ .tag = .for_input_not_captured, .token = input }); + } + return inputs; +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? +fn parseWhileExpr(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + const then_expr = try p.expectExpr(); + _ = p.eatToken(.keyword_else) orelse { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectExpr(); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// CurlySuffixExpr <- TypeExpr InitList? +/// +/// InitList +/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE +/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE +/// / LBRACE RBRACE +fn parseCurlySuffixExpr(p: *Parse) !Node.Index { + const lhs = try p.parseTypeExpr(); + if (lhs == 0) return null_node; + const lbrace = p.eatToken(.l_brace) orelse return lhs; + + // If there are 0 or 1 items, we can use ArrayInitOne/StructInitOne; + // otherwise we use the full ArrayInit/StructInit. + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const field_init = try p.parseFieldInit(); + if (field_init != 0) { + try p.scratch.append(p.gpa, field_init); + while (true) { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + if (p.eatToken(.r_brace)) |_| break; + const next = try p.expectFieldInit(); + try p.scratch.append(p.gpa, next); + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => unreachable, + 1 => return p.addNode(.{ + .tag = if (comma) .struct_init_one_comma else .struct_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = inits[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .struct_init_comma else .struct_init, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(try p.listToSpan(inits)), + }, + }), + } + } + + while (true) { + if (p.eatToken(.r_brace)) |_| break; + const elem_init = try p.expectExpr(); + try p.scratch.append(p.gpa, elem_init); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => return p.addNode(.{ + .tag = .struct_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .array_init_one_comma else .array_init_one, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = inits[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .array_init_comma else .array_init, + .main_token = lbrace, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(try p.listToSpan(inits)), + }, + }), + } +} + +/// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? +fn parseErrorUnionExpr(p: *Parse) !Node.Index { + const suffix_expr = try p.parseSuffixExpr(); + if (suffix_expr == 0) return null_node; + const bang = p.eatToken(.bang) orelse return suffix_expr; + return p.addNode(.{ + .tag = .error_union, + .main_token = bang, + .data = .{ + .lhs = suffix_expr, + .rhs = try p.expectTypeExpr(), + }, + }); +} + +/// SuffixExpr +/// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments +/// / PrimaryTypeExpr (SuffixOp / FnCallArguments)* +/// +/// FnCallArguments <- LPAREN ExprList RPAREN +/// +/// ExprList <- (Expr COMMA)* Expr? +fn parseSuffixExpr(p: *Parse) !Node.Index { + if (p.eatToken(.keyword_async)) |_| { + var res = try p.expectPrimaryTypeExpr(); + while (true) { + const node = try p.parseSuffixOp(res); + if (node == 0) break; + res = node; + } + const lparen = p.eatToken(.l_paren) orelse { + try p.warn(.expected_param_list); + return res; + }; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + switch (params.len) { + 0 => return p.addNode(.{ + .tag = if (comma) .async_call_one_comma else .async_call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .async_call_one_comma else .async_call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = params[0], + }, + }), + else => return p.addNode(.{ + .tag = if (comma) .async_call_comma else .async_call, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = try p.addExtra(try p.listToSpan(params)), + }, + }), + } + } + + var res = try p.parsePrimaryTypeExpr(); + if (res == 0) return res; + while (true) { + const suffix_op = try p.parseSuffixOp(res); + if (suffix_op != 0) { + res = suffix_op; + continue; + } + const lparen = p.eatToken(.l_paren) orelse return res; + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + res = switch (params.len) { + 0 => try p.addNode(.{ + .tag = if (comma) .call_one_comma else .call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = 0, + }, + }), + 1 => try p.addNode(.{ + .tag = if (comma) .call_one_comma else .call_one, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = params[0], + }, + }), + else => try p.addNode(.{ + .tag = if (comma) .call_comma else .call, + .main_token = lparen, + .data = .{ + .lhs = res, + .rhs = try p.addExtra(try p.listToSpan(params)), + }, + }), + }; + } +} + +/// PrimaryTypeExpr +/// <- BUILTINIDENTIFIER FnCallArguments +/// / CHAR_LITERAL +/// / ContainerDecl +/// / DOT IDENTIFIER +/// / DOT InitList +/// / ErrorSetDecl +/// / FLOAT +/// / FnProto +/// / GroupedExpr +/// / LabeledTypeExpr +/// / IDENTIFIER +/// / IfTypeExpr +/// / INTEGER +/// / KEYWORD_comptime TypeExpr +/// / KEYWORD_error DOT IDENTIFIER +/// / KEYWORD_anyframe +/// / KEYWORD_unreachable +/// / STRINGLITERAL +/// / SwitchExpr +/// +/// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto +/// +/// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE +/// +/// InitList +/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE +/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE +/// / LBRACE RBRACE +/// +/// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE +/// +/// GroupedExpr <- LPAREN Expr RPAREN +/// +/// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? +/// +/// LabeledTypeExpr +/// <- BlockLabel Block +/// / BlockLabel? LoopTypeExpr +/// +/// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) +fn parsePrimaryTypeExpr(p: *Parse) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .char_literal => return p.addNode(.{ + .tag = .char_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .number_literal => return p.addNode(.{ + .tag = .number_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .keyword_unreachable => return p.addNode(.{ + .tag = .unreachable_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .keyword_anyframe => return p.addNode(.{ + .tag = .anyframe_literal, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + .string_literal => { + const main_token = p.nextToken(); + return p.addNode(.{ + .tag = .string_literal, + .main_token = main_token, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + }, + + .builtin => return p.parseBuiltinCall(), + .keyword_fn => return p.parseFnProto(), + .keyword_if => return p.parseIf(expectTypeExpr), + .keyword_switch => return p.expectSwitchExpr(), + + .keyword_extern, + .keyword_packed, + => { + p.tok_i += 1; + return p.parseContainerDeclAuto(); + }, + + .keyword_struct, + .keyword_opaque, + .keyword_enum, + .keyword_union, + => return p.parseContainerDeclAuto(), + + .keyword_comptime => return p.addNode(.{ + .tag = .@"comptime", + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectTypeExpr(), + .rhs = undefined, + }, + }), + .multiline_string_literal_line => { + const first_line = p.nextToken(); + while (p.token_tags[p.tok_i] == .multiline_string_literal_line) { + p.tok_i += 1; + } + return p.addNode(.{ + .tag = .multiline_string_literal, + .main_token = first_line, + .data = .{ + .lhs = first_line, + .rhs = p.tok_i - 1, + }, + }); + }, + .identifier => switch (p.token_tags[p.tok_i + 1]) { + .colon => switch (p.token_tags[p.tok_i + 2]) { + .keyword_inline => { + p.tok_i += 3; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => { + p.tok_i += 2; + return p.parseForTypeExpr(); + }, + .keyword_while => { + p.tok_i += 2; + return p.parseWhileTypeExpr(); + }, + .l_brace => { + p.tok_i += 2; + return p.parseBlock(); + }, + else => return p.addNode(.{ + .tag = .identifier, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + }, + else => return p.addNode(.{ + .tag = .identifier, + .main_token = p.nextToken(), + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }), + }, + .keyword_inline => { + p.tok_i += 1; + switch (p.token_tags[p.tok_i]) { + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + else => return p.fail(.expected_inlinable), + } + }, + .keyword_for => return p.parseForTypeExpr(), + .keyword_while => return p.parseWhileTypeExpr(), + .period => switch (p.token_tags[p.tok_i + 1]) { + .identifier => return p.addNode(.{ + .tag = .enum_literal, + .data = .{ + .lhs = p.nextToken(), // dot + .rhs = undefined, + }, + .main_token = p.nextToken(), // identifier + }), + .l_brace => { + const lbrace = p.tok_i + 1; + p.tok_i = lbrace + 1; + + // If there are 0, 1, or 2 items, we can use ArrayInitDotTwo/StructInitDotTwo; + // otherwise we use the full ArrayInitDot/StructInitDot. + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const field_init = try p.parseFieldInit(); + if (field_init != 0) { + try p.scratch.append(p.gpa, field_init); + while (true) { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + if (p.eatToken(.r_brace)) |_| break; + const next = try p.expectFieldInit(); + try p.scratch.append(p.gpa, next); + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => unreachable, + 1 => return p.addNode(.{ + .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .struct_init_dot_two_comma else .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = inits[1], + }, + }), + else => { + const span = try p.listToSpan(inits); + return p.addNode(.{ + .tag = if (comma) .struct_init_dot_comma else .struct_init_dot, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } + } + + while (true) { + if (p.eatToken(.r_brace)) |_| break; + const elem_init = try p.expectExpr(); + try p.scratch.append(p.gpa, elem_init); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_initializer), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const inits = p.scratch.items[scratch_top..]; + switch (inits.len) { + 0 => return p.addNode(.{ + .tag = .struct_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .array_init_dot_two_comma else .array_init_dot_two, + .main_token = lbrace, + .data = .{ + .lhs = inits[0], + .rhs = inits[1], + }, + }), + else => { + const span = try p.listToSpan(inits); + return p.addNode(.{ + .tag = if (comma) .array_init_dot_comma else .array_init_dot, + .main_token = lbrace, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } + }, + else => return null_node, + }, + .keyword_error => switch (p.token_tags[p.tok_i + 1]) { + .l_brace => { + const error_token = p.tok_i; + p.tok_i += 2; + while (true) { + if (p.eatToken(.r_brace)) |_| break; + _ = try p.eatDocComments(); + _ = try p.expectToken(.identifier); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_brace => { + p.tok_i += 1; + break; + }, + .colon, .r_paren, .r_bracket => return p.failExpected(.r_brace), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_field), + } + } + return p.addNode(.{ + .tag = .error_set_decl, + .main_token = error_token, + .data = .{ + .lhs = undefined, + .rhs = p.tok_i - 1, // rbrace + }, + }); + }, + else => { + const main_token = p.nextToken(); + const period = p.eatToken(.period); + if (period == null) try p.warnExpected(.period); + const identifier = p.eatToken(.identifier); + if (identifier == null) try p.warnExpected(.identifier); + return p.addNode(.{ + .tag = .error_value, + .main_token = main_token, + .data = .{ + .lhs = period orelse 0, + .rhs = identifier orelse 0, + }, + }); + }, + }, + .l_paren => return p.addNode(.{ + .tag = .grouped_expression, + .main_token = p.nextToken(), + .data = .{ + .lhs = try p.expectExpr(), + .rhs = try p.expectToken(.r_paren), + }, + }), + else => return null_node, + } +} + +fn expectPrimaryTypeExpr(p: *Parse) !Node.Index { + const node = try p.parsePrimaryTypeExpr(); + if (node == 0) { + return p.fail(.expected_primary_type_expr); + } + return node; +} + +/// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)? +fn parseForTypeExpr(p: *Parse) !Node.Index { + const for_token = p.eatToken(.keyword_for) orelse return null_node; + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + const inputs = try p.forPrefix(); + + const then_expr = try p.expectTypeExpr(); + var has_else = false; + if (p.eatToken(.keyword_else)) |_| { + try p.scratch.append(p.gpa, then_expr); + const else_expr = try p.expectTypeExpr(); + try p.scratch.append(p.gpa, else_expr); + has_else = true; + } else if (inputs == 1) { + return p.addNode(.{ + .tag = .for_simple, + .main_token = for_token, + .data = .{ + .lhs = p.scratch.items[scratch_top], + .rhs = then_expr, + }, + }); + } else { + try p.scratch.append(p.gpa, then_expr); + } + return p.addNode(.{ + .tag = .@"for", + .main_token = for_token, + .data = .{ + .lhs = (try p.listToSpan(p.scratch.items[scratch_top..])).start, + .rhs = @as(u32, @bitCast(Node.For{ + .inputs = @as(u31, @intCast(inputs)), + .has_else = has_else, + })), + }, + }); +} + +/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? +/// +/// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? +fn parseWhileTypeExpr(p: *Parse) !Node.Index { + const while_token = p.eatToken(.keyword_while) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + const cont_expr = try p.parseWhileContinueExpr(); + + const then_expr = try p.expectTypeExpr(); + _ = p.eatToken(.keyword_else) orelse { + if (cont_expr == 0) { + return p.addNode(.{ + .tag = .while_simple, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + } else { + return p.addNode(.{ + .tag = .while_cont, + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.WhileCont{ + .cont_expr = cont_expr, + .then_expr = then_expr, + }), + }, + }); + } + }; + _ = try p.parsePayload(); + const else_expr = try p.expectTypeExpr(); + return p.addNode(.{ + .tag = .@"while", + .main_token = while_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.While{ + .cont_expr = cont_expr, + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE +fn expectSwitchExpr(p: *Parse) !Node.Index { + const switch_token = p.assertToken(.keyword_switch); + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.expectToken(.l_brace); + const cases = try p.parseSwitchProngList(); + const trailing_comma = p.token_tags[p.tok_i - 1] == .comma; + _ = try p.expectToken(.r_brace); + + return p.addNode(.{ + .tag = if (trailing_comma) .switch_comma else .@"switch", + .main_token = switch_token, + .data = .{ + .lhs = expr_node, + .rhs = try p.addExtra(Node.SubRange{ + .start = cases.start, + .end = cases.end, + }), + }, + }); +} + +/// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN +/// +/// AsmOutput <- COLON AsmOutputList AsmInput? +/// +/// AsmInput <- COLON AsmInputList AsmClobbers? +/// +/// AsmClobbers <- COLON StringList +/// +/// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? +/// +/// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? +/// +/// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? +fn expectAsmExpr(p: *Parse) !Node.Index { + const asm_token = p.assertToken(.keyword_asm); + _ = p.eatToken(.keyword_volatile); + _ = try p.expectToken(.l_paren); + const template = try p.expectExpr(); + + if (p.eatToken(.r_paren)) |rparen| { + return p.addNode(.{ + .tag = .asm_simple, + .main_token = asm_token, + .data = .{ + .lhs = template, + .rhs = rparen, + }, + }); + } + + _ = try p.expectToken(.colon); + + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const output_item = try p.parseAsmOutputItem(); + if (output_item == 0) break; + try p.scratch.append(p.gpa, output_item); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + if (p.eatToken(.colon)) |_| { + while (true) { + const input_item = try p.parseAsmInputItem(); + if (input_item == 0) break; + try p.scratch.append(p.gpa, input_item); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + if (p.eatToken(.colon)) |_| { + while (p.eatToken(.string_literal)) |_| { + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warnExpected(.comma), + } + } + } + } + const rparen = try p.expectToken(.r_paren); + const span = try p.listToSpan(p.scratch.items[scratch_top..]); + return p.addNode(.{ + .tag = .@"asm", + .main_token = asm_token, + .data = .{ + .lhs = template, + .rhs = try p.addExtra(Node.Asm{ + .items_start = span.start, + .items_end = span.end, + .rparen = rparen, + }), + }, + }); +} + +/// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN +fn parseAsmOutputItem(p: *Parse) !Node.Index { + _ = p.eatToken(.l_bracket) orelse return null_node; + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.r_bracket); + _ = try p.expectToken(.string_literal); + _ = try p.expectToken(.l_paren); + const type_expr: Node.Index = blk: { + if (p.eatToken(.arrow)) |_| { + break :blk try p.expectTypeExpr(); + } else { + _ = try p.expectToken(.identifier); + break :blk null_node; + } + }; + const rparen = try p.expectToken(.r_paren); + return p.addNode(.{ + .tag = .asm_output, + .main_token = identifier, + .data = .{ + .lhs = type_expr, + .rhs = rparen, + }, + }); +} + +/// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN +fn parseAsmInputItem(p: *Parse) !Node.Index { + _ = p.eatToken(.l_bracket) orelse return null_node; + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.r_bracket); + _ = try p.expectToken(.string_literal); + _ = try p.expectToken(.l_paren); + const expr = try p.expectExpr(); + const rparen = try p.expectToken(.r_paren); + return p.addNode(.{ + .tag = .asm_input, + .main_token = identifier, + .data = .{ + .lhs = expr, + .rhs = rparen, + }, + }); +} + +/// BreakLabel <- COLON IDENTIFIER +fn parseBreakLabel(p: *Parse) !TokenIndex { + _ = p.eatToken(.colon) orelse return @as(TokenIndex, 0); + return p.expectToken(.identifier); +} + +/// BlockLabel <- IDENTIFIER COLON +fn parseBlockLabel(p: *Parse) TokenIndex { + if (p.token_tags[p.tok_i] == .identifier and + p.token_tags[p.tok_i + 1] == .colon) + { + const identifier = p.tok_i; + p.tok_i += 2; + return identifier; + } + return null_node; +} + +/// FieldInit <- DOT IDENTIFIER EQUAL Expr +fn parseFieldInit(p: *Parse) !Node.Index { + if (p.token_tags[p.tok_i + 0] == .period and + p.token_tags[p.tok_i + 1] == .identifier and + p.token_tags[p.tok_i + 2] == .equal) + { + p.tok_i += 3; + return p.expectExpr(); + } else { + return null_node; + } +} + +fn expectFieldInit(p: *Parse) !Node.Index { + if (p.token_tags[p.tok_i] != .period or + p.token_tags[p.tok_i + 1] != .identifier or + p.token_tags[p.tok_i + 2] != .equal) + return p.fail(.expected_initializer); + + p.tok_i += 3; + return p.expectExpr(); +} + +/// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN +fn parseWhileContinueExpr(p: *Parse) !Node.Index { + _ = p.eatToken(.colon) orelse { + if (p.token_tags[p.tok_i] == .l_paren and + p.tokensOnSameLine(p.tok_i - 1, p.tok_i)) + return p.fail(.expected_continue_expr); + return null_node; + }; + _ = try p.expectToken(.l_paren); + const node = try p.parseAssignExpr(); + if (node == 0) return p.fail(.expected_expr_or_assignment); + _ = try p.expectToken(.r_paren); + return node; +} + +/// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN +fn parseLinkSection(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_linksection) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN +fn parseCallconv(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_callconv) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// AddrSpace <- KEYWORD_addrspace LPAREN Expr RPAREN +fn parseAddrSpace(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_addrspace) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr_node = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr_node; +} + +/// This function can return null nodes and then still return nodes afterwards, +/// such as in the case of anytype and `...`. Caller must look for rparen to find +/// out when there are no more param decls left. +/// +/// ParamDecl +/// <- doc_comment? (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType +/// / DOT3 +/// +/// ParamType +/// <- KEYWORD_anytype +/// / TypeExpr +fn expectParamDecl(p: *Parse) !Node.Index { + _ = try p.eatDocComments(); + switch (p.token_tags[p.tok_i]) { + .keyword_noalias, .keyword_comptime => p.tok_i += 1, + .ellipsis3 => { + p.tok_i += 1; + return null_node; + }, + else => {}, + } + if (p.token_tags[p.tok_i] == .identifier and + p.token_tags[p.tok_i + 1] == .colon) + { + p.tok_i += 2; + } + switch (p.token_tags[p.tok_i]) { + .keyword_anytype => { + p.tok_i += 1; + return null_node; + }, + else => return p.expectTypeExpr(), + } +} + +/// Payload <- PIPE IDENTIFIER PIPE +fn parsePayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.pipe); + return identifier; +} + +/// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE +fn parsePtrPayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + _ = p.eatToken(.asterisk); + const identifier = try p.expectToken(.identifier); + _ = try p.expectToken(.pipe); + return identifier; +} + +/// Returns the first identifier token, if any. +/// +/// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE +fn parsePtrIndexPayload(p: *Parse) !TokenIndex { + _ = p.eatToken(.pipe) orelse return @as(TokenIndex, 0); + _ = p.eatToken(.asterisk); + const identifier = try p.expectToken(.identifier); + if (p.eatToken(.comma) != null) { + _ = try p.expectToken(.identifier); + } + _ = try p.expectToken(.pipe); + return identifier; +} + +/// SwitchProng <- KEYWORD_inline? SwitchCase EQUALRARROW PtrIndexPayload? AssignExpr +/// +/// SwitchCase +/// <- SwitchItem (COMMA SwitchItem)* COMMA? +/// / KEYWORD_else +fn parseSwitchProng(p: *Parse) !Node.Index { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + const is_inline = p.eatToken(.keyword_inline) != null; + + if (p.eatToken(.keyword_else) == null) { + while (true) { + const item = try p.parseSwitchItem(); + if (item == 0) break; + try p.scratch.append(p.gpa, item); + if (p.eatToken(.comma) == null) break; + } + if (scratch_top == p.scratch.items.len) { + if (is_inline) p.tok_i -= 1; + return null_node; + } + } + const arrow_token = try p.expectToken(.equal_angle_bracket_right); + _ = try p.parsePtrIndexPayload(); + + const items = p.scratch.items[scratch_top..]; + switch (items.len) { + 0 => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, + .main_token = arrow_token, + .data = .{ + .lhs = 0, + .rhs = try p.expectSingleAssignExpr(), + }, + }), + 1 => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline_one else .switch_case_one, + .main_token = arrow_token, + .data = .{ + .lhs = items[0], + .rhs = try p.expectSingleAssignExpr(), + }, + }), + else => return p.addNode(.{ + .tag = if (is_inline) .switch_case_inline else .switch_case, + .main_token = arrow_token, + .data = .{ + .lhs = try p.addExtra(try p.listToSpan(items)), + .rhs = try p.expectSingleAssignExpr(), + }, + }), + } +} + +/// SwitchItem <- Expr (DOT3 Expr)? +fn parseSwitchItem(p: *Parse) !Node.Index { + const expr = try p.parseExpr(); + if (expr == 0) return null_node; + + if (p.eatToken(.ellipsis3)) |token| { + return p.addNode(.{ + .tag = .switch_range, + .main_token = token, + .data = .{ + .lhs = expr, + .rhs = try p.expectExpr(), + }, + }); + } + return expr; +} + +const PtrModifiers = struct { + align_node: Node.Index, + addrspace_node: Node.Index, + bit_range_start: Node.Index, + bit_range_end: Node.Index, +}; + +fn parsePtrModifiers(p: *Parse) !PtrModifiers { + var result: PtrModifiers = .{ + .align_node = 0, + .addrspace_node = 0, + .bit_range_start = 0, + .bit_range_end = 0, + }; + var saw_const = false; + var saw_volatile = false; + var saw_allowzero = false; + var saw_addrspace = false; + while (true) { + switch (p.token_tags[p.tok_i]) { + .keyword_align => { + if (result.align_node != 0) { + try p.warn(.extra_align_qualifier); + } + p.tok_i += 1; + _ = try p.expectToken(.l_paren); + result.align_node = try p.expectExpr(); + + if (p.eatToken(.colon)) |_| { + result.bit_range_start = try p.expectExpr(); + _ = try p.expectToken(.colon); + result.bit_range_end = try p.expectExpr(); + } + + _ = try p.expectToken(.r_paren); + }, + .keyword_const => { + if (saw_const) { + try p.warn(.extra_const_qualifier); + } + p.tok_i += 1; + saw_const = true; + }, + .keyword_volatile => { + if (saw_volatile) { + try p.warn(.extra_volatile_qualifier); + } + p.tok_i += 1; + saw_volatile = true; + }, + .keyword_allowzero => { + if (saw_allowzero) { + try p.warn(.extra_allowzero_qualifier); + } + p.tok_i += 1; + saw_allowzero = true; + }, + .keyword_addrspace => { + if (saw_addrspace) { + try p.warn(.extra_addrspace_qualifier); + } + result.addrspace_node = try p.parseAddrSpace(); + }, + else => return result, + } + } +} + +/// SuffixOp +/// <- LBRACKET Expr (DOT2 (Expr? (COLON Expr)?)?)? RBRACKET +/// / DOT IDENTIFIER +/// / DOTASTERISK +/// / DOTQUESTIONMARK +fn parseSuffixOp(p: *Parse, lhs: Node.Index) !Node.Index { + switch (p.token_tags[p.tok_i]) { + .l_bracket => { + const lbracket = p.nextToken(); + const index_expr = try p.expectExpr(); + + if (p.eatToken(.ellipsis2)) |_| { + const end_expr = try p.parseExpr(); + if (p.eatToken(.colon)) |_| { + const sentinel = try p.expectExpr(); + _ = try p.expectToken(.r_bracket); + return p.addNode(.{ + .tag = .slice_sentinel, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(Node.SliceSentinel{ + .start = index_expr, + .end = end_expr, + .sentinel = sentinel, + }), + }, + }); + } + _ = try p.expectToken(.r_bracket); + if (end_expr == 0) { + return p.addNode(.{ + .tag = .slice_open, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = index_expr, + }, + }); + } + return p.addNode(.{ + .tag = .slice, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = try p.addExtra(Node.Slice{ + .start = index_expr, + .end = end_expr, + }), + }, + }); + } + _ = try p.expectToken(.r_bracket); + return p.addNode(.{ + .tag = .array_access, + .main_token = lbracket, + .data = .{ + .lhs = lhs, + .rhs = index_expr, + }, + }); + }, + .period_asterisk => return p.addNode(.{ + .tag = .deref, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = undefined, + }, + }), + .invalid_periodasterisks => { + try p.warn(.asterisk_after_ptr_deref); + return p.addNode(.{ + .tag = .deref, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = undefined, + }, + }); + }, + .period => switch (p.token_tags[p.tok_i + 1]) { + .identifier => return p.addNode(.{ + .tag = .field_access, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = p.nextToken(), + }, + }), + .question_mark => return p.addNode(.{ + .tag = .unwrap_optional, + .main_token = p.nextToken(), + .data = .{ + .lhs = lhs, + .rhs = p.nextToken(), + }, + }), + .l_brace => { + // this a misplaced `.{`, handle the error somewhere else + return null_node; + }, + else => { + p.tok_i += 1; + try p.warn(.expected_suffix_op); + return null_node; + }, + }, + else => return null_node, + } +} + +/// Caller must have already verified the first token. +/// +/// ContainerDeclAuto <- ContainerDeclType LBRACE container_doc_comment? ContainerMembers RBRACE +/// +/// ContainerDeclType +/// <- KEYWORD_struct (LPAREN Expr RPAREN)? +/// / KEYWORD_opaque +/// / KEYWORD_enum (LPAREN Expr RPAREN)? +/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? +fn parseContainerDeclAuto(p: *Parse) !Node.Index { + const main_token = p.nextToken(); + const arg_expr = switch (p.token_tags[main_token]) { + .keyword_opaque => null_node, + .keyword_struct, .keyword_enum => blk: { + if (p.eatToken(.l_paren)) |_| { + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + break :blk expr; + } else { + break :blk null_node; + } + }, + .keyword_union => blk: { + if (p.eatToken(.l_paren)) |_| { + if (p.eatToken(.keyword_enum)) |_| { + if (p.eatToken(.l_paren)) |_| { + const enum_tag_expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.expectToken(.r_paren); + + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + const members_span = try members.toSpan(p); + _ = try p.expectToken(.r_brace); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_enum_tag_trailing, + false => .tagged_union_enum_tag, + }, + .main_token = main_token, + .data = .{ + .lhs = enum_tag_expr, + .rhs = try p.addExtra(members_span), + }, + }); + } else { + _ = try p.expectToken(.r_paren); + + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + if (members.len <= 2) { + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_two_trailing, + false => .tagged_union_two, + }, + .main_token = main_token, + .data = .{ + .lhs = members.lhs, + .rhs = members.rhs, + }, + }); + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .tagged_union_trailing, + false => .tagged_union, + }, + .main_token = main_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + } + } + } else { + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + break :blk expr; + } + } else { + break :blk null_node; + } + }, + else => { + p.tok_i -= 1; + return p.fail(.expected_container); + }, + }; + _ = try p.expectToken(.l_brace); + const members = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + if (arg_expr == 0) { + if (members.len <= 2) { + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_two_trailing, + false => .container_decl_two, + }, + .main_token = main_token, + .data = .{ + .lhs = members.lhs, + .rhs = members.rhs, + }, + }); + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_trailing, + false => .container_decl, + }, + .main_token = main_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + } + } else { + const span = try members.toSpan(p); + return p.addNode(.{ + .tag = switch (members.trailing) { + true => .container_decl_arg_trailing, + false => .container_decl_arg, + }, + .main_token = main_token, + .data = .{ + .lhs = arg_expr, + .rhs = try p.addExtra(Node.SubRange{ + .start = span.start, + .end = span.end, + }), + }, + }); + } +} + +/// Give a helpful error message for those transitioning from +/// C's 'struct Foo {};' to Zig's 'const Foo = struct {};'. +fn parseCStyleContainer(p: *Parse) Error!bool { + const main_token = p.tok_i; + switch (p.token_tags[p.tok_i]) { + .keyword_enum, .keyword_union, .keyword_struct => {}, + else => return false, + } + const identifier = p.tok_i + 1; + if (p.token_tags[identifier] != .identifier) return false; + p.tok_i += 2; + + try p.warnMsg(.{ + .tag = .c_style_container, + .token = identifier, + .extra = .{ .expected_tag = p.token_tags[main_token] }, + }); + try p.warnMsg(.{ + .tag = .zig_style_container, + .is_note = true, + .token = identifier, + .extra = .{ .expected_tag = p.token_tags[main_token] }, + }); + + _ = try p.expectToken(.l_brace); + _ = try p.parseContainerMembers(); + _ = try p.expectToken(.r_brace); + try p.expectSemicolon(.expected_semi_after_decl, true); + return true; +} + +/// Holds temporary data until we are ready to construct the full ContainerDecl AST node. +/// +/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN +fn parseByteAlign(p: *Parse) !Node.Index { + _ = p.eatToken(.keyword_align) orelse return null_node; + _ = try p.expectToken(.l_paren); + const expr = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + return expr; +} + +/// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? +fn parseSwitchProngList(p: *Parse) !Node.SubRange { + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + + while (true) { + const item = try parseSwitchProng(p); + if (item == 0) break; + + try p.scratch.append(p.gpa, item); + + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + // All possible delimiters. + .colon, .r_paren, .r_brace, .r_bracket => break, + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_switch_prong), + } + } + return p.listToSpan(p.scratch.items[scratch_top..]); +} + +/// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? +fn parseParamDeclList(p: *Parse) !SmallSpan { + _ = try p.expectToken(.l_paren); + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + var varargs: union(enum) { none, seen, nonfinal: TokenIndex } = .none; + while (true) { + if (p.eatToken(.r_paren)) |_| break; + if (varargs == .seen) varargs = .{ .nonfinal = p.tok_i }; + const param = try p.expectParamDecl(); + if (param != 0) { + try p.scratch.append(p.gpa, param); + } else if (p.token_tags[p.tok_i - 1] == .ellipsis3) { + if (varargs == .none) varargs = .seen; + } + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + .colon, .r_brace, .r_bracket => return p.failExpected(.r_paren), + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_param), + } + } + if (varargs == .nonfinal) { + try p.warnMsg(.{ .tag = .varargs_nonfinal, .token = varargs.nonfinal }); + } + const params = p.scratch.items[scratch_top..]; + return switch (params.len) { + 0 => SmallSpan{ .zero_or_one = 0 }, + 1 => SmallSpan{ .zero_or_one = params[0] }, + else => SmallSpan{ .multi = try p.listToSpan(params) }, + }; +} + +/// FnCallArguments <- LPAREN ExprList RPAREN +/// +/// ExprList <- (Expr COMMA)* Expr? +fn parseBuiltinCall(p: *Parse) !Node.Index { + const builtin_token = p.assertToken(.builtin); + if (p.token_tags[p.nextToken()] != .l_paren) { + p.tok_i -= 1; + try p.warn(.expected_param_list); + // Pretend this was an identifier so we can continue parsing. + return p.addNode(.{ + .tag = .identifier, + .main_token = builtin_token, + .data = .{ + .lhs = undefined, + .rhs = undefined, + }, + }); + } + const scratch_top = p.scratch.items.len; + defer p.scratch.shrinkRetainingCapacity(scratch_top); + while (true) { + if (p.eatToken(.r_paren)) |_| break; + const param = try p.expectExpr(); + try p.scratch.append(p.gpa, param); + switch (p.token_tags[p.tok_i]) { + .comma => p.tok_i += 1, + .r_paren => { + p.tok_i += 1; + break; + }, + // Likely just a missing comma; give error but continue parsing. + else => try p.warn(.expected_comma_after_arg), + } + } + const comma = (p.token_tags[p.tok_i - 2] == .comma); + const params = p.scratch.items[scratch_top..]; + switch (params.len) { + 0 => return p.addNode(.{ + .tag = .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = 0, + .rhs = 0, + }, + }), + 1 => return p.addNode(.{ + .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = params[0], + .rhs = 0, + }, + }), + 2 => return p.addNode(.{ + .tag = if (comma) .builtin_call_two_comma else .builtin_call_two, + .main_token = builtin_token, + .data = .{ + .lhs = params[0], + .rhs = params[1], + }, + }), + else => { + const span = try p.listToSpan(params); + return p.addNode(.{ + .tag = if (comma) .builtin_call_comma else .builtin_call, + .main_token = builtin_token, + .data = .{ + .lhs = span.start, + .rhs = span.end, + }, + }); + }, + } +} + +/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? +fn parseIf(p: *Parse, comptime bodyParseFn: fn (p: *Parse) Error!Node.Index) !Node.Index { + const if_token = p.eatToken(.keyword_if) orelse return null_node; + _ = try p.expectToken(.l_paren); + const condition = try p.expectExpr(); + _ = try p.expectToken(.r_paren); + _ = try p.parsePtrPayload(); + + const then_expr = try bodyParseFn(p); + assert(then_expr != 0); + + _ = p.eatToken(.keyword_else) orelse return p.addNode(.{ + .tag = .if_simple, + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = then_expr, + }, + }); + _ = try p.parsePayload(); + const else_expr = try bodyParseFn(p); + assert(else_expr != 0); + + return p.addNode(.{ + .tag = .@"if", + .main_token = if_token, + .data = .{ + .lhs = condition, + .rhs = try p.addExtra(Node.If{ + .then_expr = then_expr, + .else_expr = else_expr, + }), + }, + }); +} + +/// Skips over doc comment tokens. Returns the first one, if any. +fn eatDocComments(p: *Parse) !?TokenIndex { + if (p.eatToken(.doc_comment)) |tok| { + var first_line = tok; + if (tok > 0 and tokensOnSameLine(p, tok - 1, tok)) { + try p.warnMsg(.{ + .tag = .same_line_doc_comment, + .token = tok, + }); + first_line = p.eatToken(.doc_comment) orelse return null; + } + while (p.eatToken(.doc_comment)) |_| {} + return first_line; + } + return null; +} + +fn tokensOnSameLine(p: *Parse, token1: TokenIndex, token2: TokenIndex) bool { + return std.mem.indexOfScalar(u8, p.source[p.token_starts[token1]..p.token_starts[token2]], '\n') == null; +} + +fn eatToken(p: *Parse, tag: Token.Tag) ?TokenIndex { + return if (p.token_tags[p.tok_i] == tag) p.nextToken() else null; +} + +fn assertToken(p: *Parse, tag: Token.Tag) TokenIndex { + const token = p.nextToken(); + assert(p.token_tags[token] == tag); + return token; +} + +fn expectToken(p: *Parse, tag: Token.Tag) Error!TokenIndex { + if (p.token_tags[p.tok_i] != tag) { + return p.failMsg(.{ + .tag = .expected_token, + .token = p.tok_i, + .extra = .{ .expected_tag = tag }, + }); + } + return p.nextToken(); +} + +fn expectSemicolon(p: *Parse, error_tag: AstError.Tag, recoverable: bool) Error!void { + if (p.token_tags[p.tok_i] == .semicolon) { + _ = p.nextToken(); + return; + } + try p.warn(error_tag); + if (!recoverable) return error.ParseError; +} + +fn nextToken(p: *Parse) TokenIndex { + const result = p.tok_i; + p.tok_i += 1; + return result; +} + +const null_node: Node.Index = 0; + +const Parse = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const Ast = std.zig.Ast; +const Node = Ast.Node; +const AstError = Ast.Error; +const TokenIndex = Ast.TokenIndex; +const Token = std.zig.Token; + +test { + _ = @import("parser_test.zig"); +} diff --git a/src/stage2/parser_test.zig b/src/stage2/parser_test.zig new file mode 100644 index 0000000000..203976ec1a --- /dev/null +++ b/src/stage2/parser_test.zig @@ -0,0 +1,6233 @@ +test "zig fmt: remove extra whitespace at start and end of file with comment between" { + try testTransform( + \\ + \\ + \\// hello + \\ + \\ + , + \\// hello + \\ + ); +} + +test "zig fmt: tuple struct" { + try testCanonical( + \\const T = struct { + \\ /// doc comment on tuple field + \\ comptime comptime u32, + \\ /// another doc comment on tuple field + \\ *u32 = 1, + \\ // needs to be wrapped in parentheses to not be parsed as a function decl + \\ (fn () void) align(1), + \\}; + \\ + ); +} + +test "zig fmt: preserves clobbers in inline asm with stray comma" { + try testCanonical( + \\fn foo() void { + \\ asm volatile ("" + \\ : [_] "" (-> type), + \\ : + \\ : "clobber" + \\ ); + \\ asm volatile ("" + \\ : + \\ : [_] "" (type), + \\ : "clobber" + \\ ); + \\} + \\ + ); +} + +test "zig fmt: remove trailing comma at the end of assembly clobber" { + try testTransform( + \\fn foo() void { + \\ asm volatile ("" + \\ : [_] "" (-> type), + \\ : + \\ : "clobber1", "clobber2", + \\ ); + \\} + \\ + , + \\fn foo() void { + \\ asm volatile ("" + \\ : [_] "" (-> type), + \\ : + \\ : "clobber1", "clobber2" + \\ ); + \\} + \\ + ); +} + +test "zig fmt: respect line breaks in struct field value declaration" { + try testCanonical( + \\const Foo = struct { + \\ bar: u32 = + \\ 42, + \\ bar: u32 = + \\ // a comment + \\ 42, + \\ bar: u32 = + \\ 42, + \\ // a comment + \\ bar: []const u8 = + \\ \\ foo + \\ \\ bar + \\ \\ baz + \\ , + \\ bar: u32 = + \\ blk: { + \\ break :blk 42; + \\ }, + \\}; + \\ + ); +} + +test "zig fmt: respect line breaks before functions" { + try testCanonical( + \\const std = @import("std"); + \\ + \\inline fn foo() void {} + \\ + \\noinline fn foo() void {} + \\ + \\export fn foo() void {} + \\ + \\extern fn foo() void; + \\ + \\extern "foo" fn foo() void; + \\ + ); +} + +test "zig fmt: rewrite callconv(.Inline) to the inline keyword" { + try testTransform( + \\fn foo() callconv(.Inline) void {} + \\const bar = .Inline; + \\fn foo() callconv(bar) void {} + \\ + , + \\inline fn foo() void {} + \\const bar = .Inline; + \\fn foo() callconv(bar) void {} + \\ + ); +} + +test "zig fmt: simple top level comptime block" { + try testCanonical( + \\// line comment + \\comptime {} + \\ + ); +} + +test "zig fmt: two spaced line comments before decl" { + try testCanonical( + \\// line comment + \\ + \\// another + \\comptime {} + \\ + ); +} + +test "zig fmt: respect line breaks after var declarations" { + try testCanonical( + \\const crc = + \\ lookup_tables[0][p[7]] ^ + \\ lookup_tables[1][p[6]] ^ + \\ lookup_tables[2][p[5]] ^ + \\ lookup_tables[3][p[4]] ^ + \\ lookup_tables[4][@as(u8, self.crc >> 24)] ^ + \\ lookup_tables[5][@as(u8, self.crc >> 16)] ^ + \\ lookup_tables[6][@as(u8, self.crc >> 8)] ^ + \\ lookup_tables[7][@as(u8, self.crc >> 0)]; + \\ + ); +} + +test "zig fmt: multiline string mixed with comments" { + try testCanonical( + \\const s1 = + \\ //\\one + \\ \\two) + \\ \\three + \\; + \\const s2 = + \\ \\one + \\ \\two) + \\ //\\three + \\; + \\const s3 = + \\ \\one + \\ //\\two) + \\ \\three + \\; + \\const s4 = + \\ \\one + \\ //\\two + \\ \\three + \\ //\\four + \\ \\five + \\; + \\const a = + \\ 1; + \\ + ); +} + +test "zig fmt: empty file" { + try testCanonical( + \\ + ); +} + +test "zig fmt: file ends in comment" { + try testTransform( + \\ //foobar + , + \\//foobar + \\ + ); +} + +test "zig fmt: file ends in multi line comment" { + try testTransform( + \\ \\foobar + , + \\\\foobar + \\ + ); +} + +test "zig fmt: file ends in comment after var decl" { + try testTransform( + \\const x = 42; + \\ //foobar + , + \\const x = 42; + \\//foobar + \\ + ); +} + +test "zig fmt: if statement" { + try testCanonical( + \\test "" { + \\ if (optional()) |some| + \\ bar = some.foo(); + \\} + \\ + ); +} + +test "zig fmt: top-level fields" { + try testCanonical( + \\a: did_you_know, + \\b: all_files_are, + \\structs: ?x, + \\ + ); +} + +test "zig fmt: top-level tuple function call type" { + try testCanonical( + \\foo() + \\ + ); +} + +test "zig fmt: top-level enum missing 'const name ='" { + try testError( + \\enum(u32) + \\ + , &[_]Error{.expected_token}); +} + +test "zig fmt: top-level for/while loop" { + try testCanonical( + \\for (foo) |_| foo + \\ + ); + try testCanonical( + \\while (foo) |_| foo + \\ + ); +} + +test "zig fmt: top-level bare asterisk+identifier" { + try testCanonical( + \\*x + \\ + ); +} + +test "zig fmt: top-level bare asterisk+asterisk+identifier" { + try testCanonical( + \\**x + \\ + ); +} + +test "zig fmt: C style containers" { + try testError( + \\struct Foo { + \\ a: u32, + \\}; + , &[_]Error{ + .c_style_container, + .zig_style_container, + }); + try testError( + \\test { + \\ struct Foo { + \\ a: u32, + \\ }; + \\} + , &[_]Error{ + .c_style_container, + .zig_style_container, + }); +} + +test "zig fmt: decl between fields" { + try testError( + \\const S = struct { + \\ const foo = 2; + \\ const bar = 2; + \\ const baz = 2; + \\ a: usize, + \\ const foo1 = 2; + \\ const bar1 = 2; + \\ const baz1 = 2; + \\ b: usize, + \\}; + , &[_]Error{ + .decl_between_fields, + .previous_field, + .next_field, + }); +} + +test "zig fmt: errdefer with payload" { + try testCanonical( + \\pub fn main() anyerror!void { + \\ errdefer |a| x += 1; + \\ errdefer |a| {} + \\ errdefer |a| { + \\ x += 1; + \\ } + \\} + \\ + ); +} + +test "zig fmt: nosuspend block" { + try testCanonical( + \\pub fn main() anyerror!void { + \\ nosuspend { + \\ var foo: Foo = .{ .bar = 42 }; + \\ } + \\} + \\ + ); +} + +test "zig fmt: nosuspend await" { + try testCanonical( + \\fn foo() void { + \\ x = nosuspend await y; + \\} + \\ + ); +} + +test "zig fmt: container declaration, single line" { + try testCanonical( + \\const X = struct { foo: i32 }; + \\const X = struct { foo: i32, bar: i32 }; + \\const X = struct { foo: i32 = 1, bar: i32 = 2 }; + \\const X = struct { foo: i32 align(4), bar: i32 align(4) }; + \\const X = struct { foo: i32 align(4) = 1, bar: i32 align(4) = 2 }; + \\ + ); +} + +test "zig fmt: container declaration, one item, multi line trailing comma" { + try testCanonical( + \\test "" { + \\ comptime { + \\ const X = struct { + \\ x: i32, + \\ }; + \\ } + \\} + \\ + ); +} + +test "zig fmt: container declaration, no trailing comma on separate line" { + try testTransform( + \\test "" { + \\ comptime { + \\ const X = struct { + \\ x: i32 + \\ }; + \\ } + \\} + \\ + , + \\test "" { + \\ comptime { + \\ const X = struct { x: i32 }; + \\ } + \\} + \\ + ); +} + +test "zig fmt: container declaration, line break, no trailing comma" { + try testTransform( + \\const X = struct { + \\ foo: i32, bar: i8 }; + , + \\const X = struct { foo: i32, bar: i8 }; + \\ + ); +} + +test "zig fmt: container declaration, transform trailing comma" { + try testTransform( + \\const X = struct { + \\ foo: i32, bar: i8, }; + , + \\const X = struct { + \\ foo: i32, + \\ bar: i8, + \\}; + \\ + ); +} + +test "zig fmt: container declaration, comment, add trailing comma" { + try testTransform( + \\const X = struct { + \\ foo: i32, // foo + \\ bar: i8 + \\}; + , + \\const X = struct { + \\ foo: i32, // foo + \\ bar: i8, + \\}; + \\ + ); + try testTransform( + \\const X = struct { + \\ foo: i32 // foo + \\}; + , + \\const X = struct { + \\ foo: i32, // foo + \\}; + \\ + ); +} + +test "zig fmt: container declaration, multiline string, add trailing comma" { + try testTransform( + \\const X = struct { + \\ foo: []const u8 = + \\ \\ foo + \\ , + \\ bar: i8 + \\}; + , + \\const X = struct { + \\ foo: []const u8 = + \\ \\ foo + \\ , + \\ bar: i8, + \\}; + \\ + ); +} + +test "zig fmt: container declaration, doc comment on member, add trailing comma" { + try testTransform( + \\pub const Pos = struct { + \\ /// X-axis. + \\ x: u32, + \\ /// Y-axis. + \\ y: u32 + \\}; + , + \\pub const Pos = struct { + \\ /// X-axis. + \\ x: u32, + \\ /// Y-axis. + \\ y: u32, + \\}; + \\ + ); +} + +test "zig fmt: remove empty lines at start/end of container decl" { + try testTransform( + \\const X = struct { + \\ + \\ foo: i32, + \\ + \\ bar: i8, + \\ + \\}; + \\ + , + \\const X = struct { + \\ foo: i32, + \\ + \\ bar: i8, + \\}; + \\ + ); +} + +test "zig fmt: remove empty lines at start/end of block" { + try testTransform( + \\test { + \\ + \\ if (foo) { + \\ foo(); + \\ } + \\ + \\} + \\ + , + \\test { + \\ if (foo) { + \\ foo(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: allow empty line before comment at start of block" { + try testCanonical( + \\test { + \\ + \\ // foo + \\ const x = 42; + \\} + \\ + ); +} + +test "zig fmt: trailing comma in fn parameter list" { + try testCanonical( + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) align(8) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) addrspace(.generic) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) linksection(".text") i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) callconv(.C) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) align(8) linksection(".text") i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) align(8) callconv(.C) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) align(8) linksection(".text") callconv(.C) i32 {} + \\pub fn f( + \\ a: i32, + \\ b: i32, + \\) linksection(".text") callconv(.C) i32 {} + \\ + ); +} + +test "zig fmt: comptime struct field" { + try testCanonical( + \\const Foo = struct { + \\ a: i32, + \\ comptime b: i32 = 1234, + \\}; + \\ + ); +} + +test "zig fmt: break from block" { + try testCanonical( + \\const a = blk: { + \\ break :blk 42; + \\}; + \\const b = blk: { + \\ break :blk; + \\}; + \\const c = { + \\ break 42; + \\}; + \\const d = { + \\ break; + \\}; + \\ + ); +} + +test "zig fmt: grouped expressions (parentheses)" { + try testCanonical( + \\const r = (x + y) * (a + b); + \\ + ); +} + +test "zig fmt: c pointer type" { + try testCanonical( + \\pub extern fn repro() [*c]const u8; + \\ + ); +} + +test "zig fmt: builtin call with trailing comma" { + try testCanonical( + \\pub fn main() void { + \\ @breakpoint(); + \\ _ = @intFromBool(a); + \\ _ = @call( + \\ a, + \\ b, + \\ c, + \\ ); + \\} + \\ + ); +} + +test "zig fmt: asm expression with comptime content" { + try testCanonical( + \\comptime { + \\ asm ("foo" ++ "bar"); + \\} + \\pub fn main() void { + \\ asm volatile ("foo" ++ "bar"); + \\ asm volatile ("foo" ++ "bar" + \\ : [_] "" (x), + \\ ); + \\ asm volatile ("foo" ++ "bar" + \\ : [_] "" (x), + \\ : [_] "" (y), + \\ ); + \\ asm volatile ("foo" ++ "bar" + \\ : [_] "" (x), + \\ : [_] "" (y), + \\ : "h", "e", "l", "l", "o" + \\ ); + \\} + \\ + ); +} + +test "zig fmt: array types last token" { + try testCanonical( + \\test { + \\ const x = [40]u32; + \\} + \\ + \\test { + \\ const x = [40:0]u32; + \\} + \\ + ); +} + +test "zig fmt: sentinel-terminated array type" { + try testCanonical( + \\pub fn cStrToPrefixedFileW(s: [*:0]const u8) ![PATH_MAX_WIDE:0]u16 { + \\ return sliceToPrefixedFileW(mem.toSliceConst(u8, s)); + \\} + \\ + ); +} + +test "zig fmt: sentinel-terminated slice type" { + try testCanonical( + \\pub fn toSlice(self: Buffer) [:0]u8 { + \\ return self.list.toSlice()[0..self.len()]; + \\} + \\ + ); +} + +test "zig fmt: pointer-to-one with modifiers" { + try testCanonical( + \\const x: *u32 = undefined; + \\const y: *allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\const z: *allowzero align(8:4:2) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: pointer-to-many with modifiers" { + try testCanonical( + \\const x: [*]u32 = undefined; + \\const y: [*]allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\const z: [*]allowzero align(8:4:2) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: sentinel pointer with modifiers" { + try testCanonical( + \\const x: [*:42]u32 = undefined; + \\const y: [*:42]allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\const y: [*:42]allowzero align(8:4:2) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: c pointer with modifiers" { + try testCanonical( + \\const x: [*c]u32 = undefined; + \\const y: [*c]allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\const z: [*c]allowzero align(8:4:2) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: slice with modifiers" { + try testCanonical( + \\const x: []u32 = undefined; + \\const y: []allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: sentinel slice with modifiers" { + try testCanonical( + \\const x: [:42]u32 = undefined; + \\const y: [:42]allowzero align(8) addrspace(.generic) const volatile u32 = undefined; + \\ + ); +} + +test "zig fmt: anon literal in array" { + try testCanonical( + \\var arr: [2]Foo = .{ + \\ .{ .a = 2 }, + \\ .{ .b = 3 }, + \\}; + \\ + ); +} + +test "zig fmt: alignment in anonymous literal" { + try testTransform( + \\const a = .{ + \\ "U", "L", "F", + \\ "U'", + \\ "L'", + \\ "F'", + \\}; + \\ + , + \\const a = .{ + \\ "U", "L", "F", + \\ "U'", "L'", "F'", + \\}; + \\ + ); +} + +test "zig fmt: anon struct literal 0 element" { + try testCanonical( + \\test { + \\ const x = .{}; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 1 element" { + try testCanonical( + \\test { + \\ const x = .{ .a = b }; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 1 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ .a = b, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 2 element" { + try testCanonical( + \\test { + \\ const x = .{ .a = b, .c = d }; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 2 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ .a = b, + \\ .c = d, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 3 element" { + try testCanonical( + \\test { + \\ const x = .{ .a = b, .c = d, .e = f }; + \\} + \\ + ); +} + +test "zig fmt: anon struct literal 3 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ .a = b, + \\ .c = d, + \\ .e = f, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: struct literal 0 element" { + try testCanonical( + \\test { + \\ const x = X{}; + \\} + \\ + ); +} + +test "zig fmt: struct literal 1 element" { + try testCanonical( + \\test { + \\ const x = X{ .a = b }; + \\} + \\ + ); +} + +test "zig fmt: Unicode code point literal larger than u8" { + try testCanonical( + \\test { + \\ const x = X{ + \\ .a = b, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: struct literal 2 element" { + try testCanonical( + \\test { + \\ const x = X{ .a = b, .c = d }; + \\} + \\ + ); +} + +test "zig fmt: struct literal 2 element comma" { + try testCanonical( + \\test { + \\ const x = X{ + \\ .a = b, + \\ .c = d, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: struct literal 3 element" { + try testCanonical( + \\test { + \\ const x = X{ .a = b, .c = d, .e = f }; + \\} + \\ + ); +} + +test "zig fmt: struct literal 3 element comma" { + try testCanonical( + \\test { + \\ const x = X{ + \\ .a = b, + \\ .c = d, + \\ .e = f, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 1 element" { + try testCanonical( + \\test { + \\ const x = .{a}; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 1 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ a, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 2 element" { + try testCanonical( + \\test { + \\ const x = .{ a, b }; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 2 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ a, + \\ b, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 3 element" { + try testCanonical( + \\test { + \\ const x = .{ a, b, c }; + \\} + \\ + ); +} + +test "zig fmt: anon list literal 3 element comma" { + try testCanonical( + \\test { + \\ const x = .{ + \\ a, + \\ // foo + \\ b, + \\ + \\ c, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: array literal 0 element" { + try testCanonical( + \\test { + \\ const x = [_]u32{}; + \\} + \\ + ); +} + +test "zig fmt: array literal 1 element" { + try testCanonical( + \\test { + \\ const x = [_]u32{a}; + \\} + \\ + ); +} + +test "zig fmt: array literal 1 element comma" { + try testCanonical( + \\test { + \\ const x = [1]u32{ + \\ a, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: array literal 2 element" { + try testCanonical( + \\test { + \\ const x = [_]u32{ a, b }; + \\} + \\ + ); +} + +test "zig fmt: array literal 2 element comma" { + try testCanonical( + \\test { + \\ const x = [2]u32{ + \\ a, + \\ b, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: array literal 3 element" { + try testCanonical( + \\test { + \\ const x = [_]u32{ a, b, c }; + \\} + \\ + ); +} + +test "zig fmt: array literal 3 element comma" { + try testCanonical( + \\test { + \\ const x = [3]u32{ + \\ a, + \\ b, + \\ c, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: sentinel array literal 1 element" { + try testCanonical( + \\test { + \\ const x = [_:9000]u32{a}; + \\} + \\ + ); +} + +test "zig fmt: slices" { + try testCanonical( + \\const a = b[0..]; + \\const c = d[0..1]; + \\const d = f[0.. :0]; + \\const e = f[0..1 :0]; + \\ + ); +} + +test "zig fmt: slices with spaces in bounds" { + try testCanonical( + \\const a = b[0 + 0 ..]; + \\const c = d[0 + 0 .. 1]; + \\const c = d[0 + 0 .. :0]; + \\const e = f[0 .. 1 + 1 :0]; + \\ + ); +} + +test "zig fmt: block in slice expression" { + try testCanonical( + \\const a = b[{ + \\ _ = x; + \\}..]; + \\const c = d[0..{ + \\ _ = x; + \\ _ = y; + \\}]; + \\const e = f[0..1 :{ + \\ _ = x; + \\ _ = y; + \\ _ = z; + \\}]; + \\ + ); +} + +test "zig fmt: async function" { + try testCanonical( + \\pub const Server = struct { + \\ handleRequestFn: fn (*Server, *const std.net.Address, File) callconv(.Async) void, + \\}; + \\test "hi" { + \\ var ptr: fn (i32) callconv(.Async) void = @ptrCast(other); + \\} + \\ + ); +} + +test "zig fmt: whitespace fixes" { + try testTransform("test \"\" {\r\n\tconst hi = x;\r\n}\n// zig fmt: off\ntest \"\"{\r\n\tconst a = b;}\r\n", + \\test "" { + \\ const hi = x; + \\} + \\// zig fmt: off + \\test ""{ + \\ const a = b;} + \\ + ); +} + +test "zig fmt: while else err prong with no block" { + try testCanonical( + \\test "" { + \\ const result = while (returnError()) |value| { + \\ break value; + \\ } else |err| @as(i32, 2); + \\ try expect(result == 2); + \\} + \\ + ); +} + +test "zig fmt: tagged union with enum values" { + try testCanonical( + \\const MultipleChoice2 = union(enum(u32)) { + \\ Unspecified1: i32, + \\ A: f32 = 20, + \\ Unspecified2: void, + \\ B: bool = 40, + \\ Unspecified3: i32, + \\ C: i8 = 60, + \\ Unspecified4: void, + \\ D: void = 1000, + \\ Unspecified5: i32, + \\}; + \\ + ); +} + +test "zig fmt: tagged union enum tag last token" { + try testCanonical( + \\test { + \\ const U = union(enum(u32)) {}; + \\} + \\ + \\test { + \\ const U = union(enum(u32)) { foo }; + \\} + \\ + \\test { + \\ const U = union(enum(u32)) { + \\ foo, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: allowzero pointer" { + try testCanonical( + \\const T = [*]allowzero const u8; + \\ + ); +} + +test "zig fmt: empty enum decls" { + try testCanonical( + \\const A = enum {}; + \\const B = enum(u32) {}; + \\const C = extern enum(c_int) {}; + \\const D = packed enum(u8) {}; + \\ + ); +} + +test "zig fmt: empty union decls" { + try testCanonical( + \\const A = union {}; + \\const B = union(enum) {}; + \\const C = union(Foo) {}; + \\const D = extern union {}; + \\const E = packed union {}; + \\ + ); +} + +test "zig fmt: enum literal" { + try testCanonical( + \\const x = .hi; + \\ + ); +} + +test "zig fmt: enum literal inside array literal" { + try testCanonical( + \\test "enums in arrays" { + \\ var colors = []Color{.Green}; + \\ colors = []Colors{ .Green, .Cyan }; + \\ colors = []Colors{ + \\ .Grey, + \\ .Green, + \\ .Cyan, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: character literal larger than u8" { + try testCanonical( + \\const x = '\u{01f4a9}'; + \\ + ); +} + +test "zig fmt: infix operator and then multiline string literal" { + try testCanonical( + \\const x = "" ++ + \\ \\ hi + \\; + \\ + ); +} + +test "zig fmt: infix operator and then multiline string literal over multiple lines" { + try testCanonical( + \\const x = "" ++ + \\ \\ hi0 + \\ \\ hi1 + \\ \\ hi2 + \\; + \\ + ); +} + +test "zig fmt: C pointers" { + try testCanonical( + \\const Ptr = [*c]i32; + \\ + ); +} + +test "zig fmt: threadlocal" { + try testCanonical( + \\threadlocal var x: i32 = 1234; + \\ + ); +} + +test "zig fmt: linksection" { + try testCanonical( + \\export var aoeu: u64 linksection(".text.derp") = 1234; + \\export fn _start() linksection(".text.boot") callconv(.Naked) noreturn {} + \\ + ); +} + +test "zig fmt: addrspace" { + try testCanonical( + \\export var python_length: u64 align(1) addrspace(.generic); + \\export var python_color: Color addrspace(.generic) = .green; + \\export var python_legs: u0 align(8) addrspace(.generic) linksection(".python") = 0; + \\export fn python_hiss() align(8) addrspace(.generic) linksection(".python") void; + \\ + ); +} + +test "zig fmt: correctly space struct fields with doc comments" { + try testTransform( + \\pub const S = struct { + \\ /// A + \\ a: u8, + \\ /// B + \\ /// B (cont) + \\ b: u8, + \\ + \\ + \\ /// C + \\ c: u8, + \\}; + \\ + , + \\pub const S = struct { + \\ /// A + \\ a: u8, + \\ /// B + \\ /// B (cont) + \\ b: u8, + \\ + \\ /// C + \\ c: u8, + \\}; + \\ + ); +} + +test "zig fmt: doc comments on param decl" { + try testCanonical( + \\pub const Allocator = struct { + \\ shrinkFn: fn ( + \\ self: Allocator, + \\ /// Guaranteed to be the same as what was returned from most recent call to + \\ /// `allocFn`, `reallocFn`, or `shrinkFn`. + \\ old_mem: []u8, + \\ /// Guaranteed to be the same as what was returned from most recent call to + \\ /// `allocFn`, `reallocFn`, or `shrinkFn`. + \\ old_alignment: u29, + \\ /// Guaranteed to be less than or equal to `old_mem.len`. + \\ new_byte_count: usize, + \\ /// Guaranteed to be less than or equal to `old_alignment`. + \\ new_alignment: u29, + \\ ) []u8, + \\}; + \\ + ); +} + +test "zig fmt: aligned struct field" { + try testCanonical( + \\pub const S = struct { + \\ f: i32 align(32), + \\}; + \\ + ); + try testCanonical( + \\pub const S = struct { + \\ f: i32 align(32) = 1, + \\}; + \\ + ); +} + +test "zig fmt: comment to disable/enable zig fmt first" { + try testCanonical( + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + ); +} + +test "zig fmt: 'zig fmt: (off|on)' can be surrounded by arbitrary whitespace" { + try testTransform( + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + \\ + \\// zig fmt: on + , + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + \\ + \\// zig fmt: on + \\ + ); +} + +test "zig fmt: comment to disable/enable zig fmt" { + try testTransform( + \\const a = b; + \\// zig fmt: off + \\const c = d; + \\// zig fmt: on + \\const e = f; + , + \\const a = b; + \\// zig fmt: off + \\const c = d; + \\// zig fmt: on + \\const e = f; + \\ + ); +} + +test "zig fmt: line comment following 'zig fmt: off'" { + try testCanonical( + \\// zig fmt: off + \\// Test + \\const e = f; + ); +} + +test "zig fmt: doc comment following 'zig fmt: off'" { + try testCanonical( + \\// zig fmt: off + \\/// test + \\const e = f; + ); +} + +test "zig fmt: line and doc comment following 'zig fmt: off'" { + try testCanonical( + \\// zig fmt: off + \\// test 1 + \\/// test 2 + \\const e = f; + ); +} + +test "zig fmt: doc and line comment following 'zig fmt: off'" { + try testCanonical( + \\// zig fmt: off + \\/// test 1 + \\// test 2 + \\const e = f; + ); +} + +test "zig fmt: alternating 'zig fmt: off' and 'zig fmt: on'" { + try testCanonical( + \\// zig fmt: off + \\// zig fmt: on + \\// zig fmt: off + \\const e = f; + \\// zig fmt: off + \\// zig fmt: on + \\// zig fmt: off + \\const a = b; + \\// zig fmt: on + \\const c = d; + \\// zig fmt: on + \\ + ); +} + +test "zig fmt: line comment following 'zig fmt: on'" { + try testCanonical( + \\// zig fmt: off + \\const e = f; + \\// zig fmt: on + \\// test + \\const e = f; + \\ + ); +} + +test "zig fmt: doc comment following 'zig fmt: on'" { + try testCanonical( + \\// zig fmt: off + \\const e = f; + \\// zig fmt: on + \\/// test + \\const e = f; + \\ + ); +} + +test "zig fmt: line and doc comment following 'zig fmt: on'" { + try testCanonical( + \\// zig fmt: off + \\const e = f; + \\// zig fmt: on + \\// test1 + \\/// test2 + \\const e = f; + \\ + ); +} + +test "zig fmt: doc and line comment following 'zig fmt: on'" { + try testCanonical( + \\// zig fmt: off + \\const e = f; + \\// zig fmt: on + \\/// test1 + \\// test2 + \\const e = f; + \\ + ); +} + +test "zig fmt: 'zig fmt: (off|on)' works in the middle of code" { + try testTransform( + \\test "" { + \\ const x = 42; + \\ + \\ if (foobar) |y| { + \\ // zig fmt: off + \\ }// zig fmt: on + \\ + \\ const z = 420; + \\} + \\ + , + \\test "" { + \\ const x = 42; + \\ + \\ if (foobar) |y| { + \\ // zig fmt: off + \\ }// zig fmt: on + \\ + \\ const z = 420; + \\} + \\ + ); +} + +test "zig fmt: 'zig fmt: on' indentation is unchanged" { + try testCanonical( + \\fn initOptionsAndLayouts(output: *Output, context: *Context) !void { + \\ // zig fmt: off + \\ try output.main_amount.init(output, "main_amount"); errdefer optput.main_amount.deinit(); + \\ try output.main_factor.init(output, "main_factor"); errdefer optput.main_factor.deinit(); + \\ try output.view_padding.init(output, "view_padding"); errdefer optput.view_padding.deinit(); + \\ try output.outer_padding.init(output, "outer_padding"); errdefer optput.outer_padding.deinit(); + \\ // zig fmt: on + \\ + \\ // zig fmt: off + \\ try output.top.init(output, .top); errdefer optput.top.deinit(); + \\ try output.right.init(output, .right); errdefer optput.right.deinit(); + \\ try output.bottom.init(output, .bottom); errdefer optput.bottom.deinit(); + \\ try output.left.init(output, .left); errdefer optput.left.deinit(); + \\ // zig fmt: on + \\} + \\ + ); +} + +test "zig fmt: pointer of unknown length" { + try testCanonical( + \\fn foo(ptr: [*]u8) void {} + \\ + ); +} + +test "zig fmt: spaces around slice operator" { + try testCanonical( + \\var a = b[c..d]; + \\var a = b[c..d :0]; + \\var a = b[c + 1 .. d]; + \\var a = b[c + 1 ..]; + \\var a = b[c .. d + 1]; + \\var a = b[c .. d + 1 :0]; + \\var a = b[c.a..d.e]; + \\var a = b[c.a..d.e :0]; + \\ + ); +} + +test "zig fmt: async call in if condition" { + try testCanonical( + \\comptime { + \\ if (async b()) { + \\ a(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: 2nd arg multiline string" { + try testCanonical( + \\comptime { + \\ cases.addAsm("hello world linux x86_64", + \\ \\.text + \\ , "Hello, world!\n"); + \\} + \\ + ); + try testTransform( + \\comptime { + \\ cases.addAsm("hello world linux x86_64", + \\ \\.text + \\ , "Hello, world!\n",); + \\} + , + \\comptime { + \\ cases.addAsm( + \\ "hello world linux x86_64", + \\ \\.text + \\ , + \\ "Hello, world!\n", + \\ ); + \\} + \\ + ); +} + +test "zig fmt: 2nd arg multiline string many args" { + try testCanonical( + \\comptime { + \\ cases.addAsm("hello world linux x86_64", + \\ \\.text + \\ , "Hello, world!\n", "Hello, world!\n"); + \\} + \\ + ); +} + +test "zig fmt: final arg multiline string" { + try testCanonical( + \\comptime { + \\ cases.addAsm("hello world linux x86_64", "Hello, world!\n", + \\ \\.text + \\ ); + \\} + \\ + ); +} + +test "zig fmt: if condition wraps" { + try testTransform( + \\comptime { + \\ if (cond and + \\ cond) { + \\ return x; + \\ } + \\ while (cond and + \\ cond) { + \\ return x; + \\ } + \\ if (a == b and + \\ c) { + \\ a = b; + \\ } + \\ while (a == b and + \\ c) { + \\ a = b; + \\ } + \\ if ((cond and + \\ cond)) { + \\ return x; + \\ } + \\ while ((cond and + \\ cond)) { + \\ return x; + \\ } + \\ var a = if (a) |*f| x: { + \\ break :x &a.b; + \\ } else |err| err; + \\ var a = if (cond and + \\ cond) |*f| + \\ x: { + \\ break :x &a.b; + \\ } else |err| err; + \\} + , + \\comptime { + \\ if (cond and + \\ cond) + \\ { + \\ return x; + \\ } + \\ while (cond and + \\ cond) + \\ { + \\ return x; + \\ } + \\ if (a == b and + \\ c) + \\ { + \\ a = b; + \\ } + \\ while (a == b and + \\ c) + \\ { + \\ a = b; + \\ } + \\ if ((cond and + \\ cond)) + \\ { + \\ return x; + \\ } + \\ while ((cond and + \\ cond)) + \\ { + \\ return x; + \\ } + \\ var a = if (a) |*f| x: { + \\ break :x &a.b; + \\ } else |err| err; + \\ var a = if (cond and + \\ cond) |*f| + \\ x: { + \\ break :x &a.b; + \\ } else |err| err; + \\} + \\ + ); +} + +test "zig fmt: if condition has line break but must not wrap" { + try testCanonical( + \\comptime { + \\ if (self.user_input_options.put( + \\ name, + \\ UserInputOption{ + \\ .name = name, + \\ .used = false, + \\ }, + \\ ) catch unreachable) |*prev_value| { + \\ foo(); + \\ bar(); + \\ } + \\ if (put( + \\ a, + \\ b, + \\ )) { + \\ foo(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: if condition has line break but must not wrap (no fn call comma)" { + try testCanonical( + \\comptime { + \\ if (self.user_input_options.put(name, UserInputOption{ + \\ .name = name, + \\ .used = false, + \\ }) catch unreachable) |*prev_value| { + \\ foo(); + \\ bar(); + \\ } + \\ if (put( + \\ a, + \\ b, + \\ )) { + \\ foo(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: function call with multiline argument" { + try testCanonical( + \\comptime { + \\ self.user_input_options.put(name, UserInputOption{ + \\ .name = name, + \\ .used = false, + \\ }); + \\} + \\ + ); +} + +test "zig fmt: if-else with comment before else" { + try testCanonical( + \\comptime { + \\ // cexp(finite|nan +- i inf|nan) = nan + i nan + \\ if ((hx & 0x7fffffff) != 0x7f800000) { + \\ return Complex(f32).init(y - y, y - y); + \\ } // cexp(-inf +- i inf|nan) = 0 + i0 + \\ else if (hx & 0x80000000 != 0) { + \\ return Complex(f32).init(0, 0); + \\ } // cexp(+inf +- i inf|nan) = inf + i nan + \\ else { + \\ return Complex(f32).init(x, y - y); + \\ } + \\} + \\ + ); +} + +test "zig fmt: if nested" { + try testCanonical( + \\pub fn foo() void { + \\ return if ((aInt & bInt) >= 0) + \\ if (aInt < bInt) + \\ GE_LESS + \\ else if (aInt == bInt) + \\ GE_EQUAL + \\ else + \\ GE_GREATER + \\ // comment + \\ else if (aInt > bInt) + \\ GE_LESS + \\ else if (aInt == bInt) + \\ GE_EQUAL + \\ else + \\ GE_GREATER; + \\ // comment + \\} + \\ + ); +} + +test "zig fmt: respect line breaks in if-else" { + try testCanonical( + \\comptime { + \\ return if (cond) a else b; + \\ return if (cond) + \\ a + \\ else + \\ b; + \\ return if (cond) + \\ a + \\ else if (cond) + \\ b + \\ else + \\ c; + \\} + \\ + ); +} + +test "zig fmt: respect line breaks after infix operators" { + try testCanonical( + \\comptime { + \\ self.crc = + \\ lookup_tables[0][p[7]] ^ + \\ lookup_tables[1][p[6]] ^ + \\ lookup_tables[2][p[5]] ^ + \\ lookup_tables[3][p[4]] ^ + \\ lookup_tables[4][@as(u8, self.crc >> 24)] ^ + \\ lookup_tables[5][@as(u8, self.crc >> 16)] ^ + \\ lookup_tables[6][@as(u8, self.crc >> 8)] ^ + \\ lookup_tables[7][@as(u8, self.crc >> 0)]; + \\} + \\ + ); +} + +test "zig fmt: fn decl with trailing comma" { + try testTransform( + \\fn foo(a: i32, b: i32,) void {} + , + \\fn foo( + \\ a: i32, + \\ b: i32, + \\) void {} + \\ + ); +} + +test "zig fmt: enum decl with no trailing comma" { + try testTransform( + \\const StrLitKind = enum {Normal, C}; + , + \\const StrLitKind = enum { Normal, C }; + \\ + ); +} + +test "zig fmt: switch comment before prong" { + try testCanonical( + \\comptime { + \\ switch (a) { + \\ // hi + \\ 0 => {}, + \\ } + \\} + \\ + ); +} + +test "zig fmt: switch comment after prong" { + try testCanonical( + \\comptime { + \\ switch (a) { + \\ 0, + \\ // hi + \\ => {}, + \\ } + \\} + \\ + ); +} + +test "zig fmt: struct literal no trailing comma" { + try testTransform( + \\const a = foo{ .x = 1, .y = 2 }; + \\const a = foo{ .x = 1, + \\ .y = 2 }; + \\const a = foo{ .x = 1, + \\ .y = 2, }; + , + \\const a = foo{ .x = 1, .y = 2 }; + \\const a = foo{ .x = 1, .y = 2 }; + \\const a = foo{ + \\ .x = 1, + \\ .y = 2, + \\}; + \\ + ); +} + +test "zig fmt: struct literal containing a multiline expression" { + try testTransform( + \\const a = A{ .x = if (f1()) 10 else 20 }; + \\const a = A{ .x = if (f1()) 10 else 20, }; + \\const a = A{ .x = if (f1()) + \\ 10 else 20 }; + \\const a = A{ .x = if (f1()) + \\ 10 else 20,}; + \\const a = A{ .x = if (f1()) 10 else 20, .y = f2() + 100 }; + \\const a = A{ .x = if (f1()) 10 else 20, .y = f2() + 100, }; + \\const a = A{ .x = if (f1()) + \\ 10 else 20}; + \\const a = A{ .x = if (f1()) + \\ 10 else 20,}; + \\const a = A{ .x = switch(g) {0 => "ok", else => "no"} }; + \\const a = A{ .x = switch(g) {0 => "ok", else => "no"}, }; + \\ + , + \\const a = A{ .x = if (f1()) 10 else 20 }; + \\const a = A{ + \\ .x = if (f1()) 10 else 20, + \\}; + \\const a = A{ .x = if (f1()) + \\ 10 + \\else + \\ 20 }; + \\const a = A{ + \\ .x = if (f1()) + \\ 10 + \\ else + \\ 20, + \\}; + \\const a = A{ .x = if (f1()) 10 else 20, .y = f2() + 100 }; + \\const a = A{ + \\ .x = if (f1()) 10 else 20, + \\ .y = f2() + 100, + \\}; + \\const a = A{ .x = if (f1()) + \\ 10 + \\else + \\ 20 }; + \\const a = A{ + \\ .x = if (f1()) + \\ 10 + \\ else + \\ 20, + \\}; + \\const a = A{ .x = switch (g) { + \\ 0 => "ok", + \\ else => "no", + \\} }; + \\const a = A{ + \\ .x = switch (g) { + \\ 0 => "ok", + \\ else => "no", + \\ }, + \\}; + \\ + ); +} + +test "zig fmt: array literal with hint" { + try testTransform( + \\const a = []u8{ + \\ 1, 2, // + \\ 3, + \\ 4, + \\ 5, + \\ 6, + \\ 7 }; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, + \\ 4, + \\ 5, + \\ 6, + \\ 7, 8 }; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, + \\ 4, + \\ 5, + \\ 6, // blah + \\ 7, 8 }; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, // + \\ 4, + \\ 5, + \\ 6, + \\ 7 }; + \\const a = []u8{ + \\ 1, + \\ 2, + \\ 3, 4, // + \\ 5, 6, // + \\ 7, 8, // + \\}; + , + \\const a = []u8{ + \\ 1, 2, // + \\ 3, 4, + \\ 5, 6, + \\ 7, + \\}; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, 4, + \\ 5, 6, + \\ 7, 8, + \\}; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, 4, + \\ 5, + \\ 6, // blah + \\ 7, + \\ 8, + \\}; + \\const a = []u8{ + \\ 1, 2, // + \\ 3, // + \\ 4, + \\ 5, + \\ 6, + \\ 7, + \\}; + \\const a = []u8{ + \\ 1, + \\ 2, + \\ 3, 4, // + \\ 5, 6, // + \\ 7, 8, // + \\}; + \\ + ); +} + +test "zig fmt: array literal vertical column alignment" { + try testTransform( + \\const a = []u8{ + \\ 1000, 200, + \\ 30, 4, + \\ 50000, 60, + \\}; + \\const a = []u8{0, 1, 2, 3, 40, + \\ 4,5,600,7, + \\ 80, + \\ 9, 10, 11, 0, 13, 14, 15,}; + \\const a = [12]u8{ + \\ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + \\const a = [12]u8{ + \\ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, }; + \\ + , + \\const a = []u8{ + \\ 1000, 200, + \\ 30, 4, + \\ 50000, 60, + \\}; + \\const a = []u8{ + \\ 0, 1, 2, 3, 40, + \\ 4, 5, 600, 7, 80, + \\ 9, 10, 11, 0, 13, + \\ 14, 15, + \\}; + \\const a = [12]u8{ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + \\const a = [12]u8{ + \\ 31, + \\ 28, + \\ 31, + \\ 30, + \\ 31, + \\ 30, + \\ 31, + \\ 31, + \\ 30, + \\ 31, + \\ 30, + \\ 31, + \\}; + \\ + ); +} + +test "zig fmt: multiline string with backslash at end of line" { + try testCanonical( + \\comptime { + \\ err( + \\ \\\ + \\ ); + \\} + \\ + ); +} + +test "zig fmt: multiline string parameter in fn call with trailing comma" { + try testCanonical( + \\fn foo() void { + \\ try stdout.print( + \\ \\ZIG_CMAKE_BINARY_DIR {s} + \\ \\ZIG_C_HEADER_FILES {s} + \\ \\ZIG_DIA_GUIDS_LIB {s} + \\ \\ + \\ , + \\ std.mem.sliceTo(c.ZIG_CMAKE_BINARY_DIR, 0), + \\ std.mem.sliceTo(c.ZIG_CXX_COMPILER, 0), + \\ std.mem.sliceTo(c.ZIG_DIA_GUIDS_LIB, 0), + \\ ); + \\} + \\ + ); +} + +test "zig fmt: trailing comma on fn call" { + try testCanonical( + \\comptime { + \\ var module = try Module.create( + \\ allocator, + \\ zig_lib_dir, + \\ full_cache_dir, + \\ ); + \\} + \\ + ); +} + +test "zig fmt: multi line arguments without last comma" { + try testTransform( + \\pub fn foo( + \\ a: usize, + \\ b: usize, + \\ c: usize, + \\ d: usize + \\) usize { + \\ return a + b + c + d; + \\} + \\ + , + \\pub fn foo(a: usize, b: usize, c: usize, d: usize) usize { + \\ return a + b + c + d; + \\} + \\ + ); +} + +test "zig fmt: empty block with only comment" { + try testCanonical( + \\comptime { + \\ { + \\ // comment + \\ } + \\} + \\ + ); +} + +test "zig fmt: trailing commas on struct decl" { + try testTransform( + \\const RoundParam = struct { + \\ k: usize, s: u32, t: u32 + \\}; + \\const RoundParam = struct { + \\ k: usize, s: u32, t: u32, + \\}; + , + \\const RoundParam = struct { k: usize, s: u32, t: u32 }; + \\const RoundParam = struct { + \\ k: usize, + \\ s: u32, + \\ t: u32, + \\}; + \\ + ); +} + +test "zig fmt: extra newlines at the end" { + try testTransform( + \\const a = b; + \\ + \\ + \\ + , + \\const a = b; + \\ + ); +} + +test "zig fmt: simple asm" { + try testTransform( + \\comptime { + \\ asm volatile ( + \\ \\.globl aoeu; + \\ \\.type aoeu, @function; + \\ \\.set aoeu, derp; + \\ ); + \\ + \\ asm ("not real assembly" + \\ :[a] "x" (x),); + \\ asm ("not real assembly" + \\ :[a] "x" (->i32),:[a] "x" (1),); + \\ asm ("still not real assembly" + \\ :::"a","b",); + \\} + , + \\comptime { + \\ asm volatile ( + \\ \\.globl aoeu; + \\ \\.type aoeu, @function; + \\ \\.set aoeu, derp; + \\ ); + \\ + \\ asm ("not real assembly" + \\ : [a] "x" (x), + \\ ); + \\ asm ("not real assembly" + \\ : [a] "x" (-> i32), + \\ : [a] "x" (1), + \\ ); + \\ asm ("still not real assembly" ::: "a", "b"); + \\} + \\ + ); +} + +test "zig fmt: nested struct literal with one item" { + try testCanonical( + \\const a = foo{ + \\ .item = bar{ .a = b }, + \\}; + \\ + ); +} + +test "zig fmt: switch cases trailing comma" { + try testTransform( + \\test "switch cases trailing comma"{ + \\ switch (x) { + \\ 1,2,3 => {}, + \\ 4,5, => {}, + \\ 6... 8, => {}, + \\ else => {}, + \\ } + \\} + , + \\test "switch cases trailing comma" { + \\ switch (x) { + \\ 1, 2, 3 => {}, + \\ 4, + \\ 5, + \\ => {}, + \\ 6...8 => {}, + \\ else => {}, + \\ } + \\} + \\ + ); +} + +test "zig fmt: slice align" { + try testCanonical( + \\const A = struct { + \\ items: []align(A) T, + \\}; + \\ + ); +} + +test "zig fmt: add trailing comma to array literal" { + try testTransform( + \\comptime { + \\ return []u16{'m', 's', 'y', 's', '-' // hi + \\ }; + \\ return []u16{'m', 's', 'y', 's', + \\ '-'}; + \\ return []u16{'m', 's', 'y', 's', '-'}; + \\} + , + \\comptime { + \\ return []u16{ + \\ 'm', 's', 'y', 's', '-', // hi + \\ }; + \\ return []u16{ 'm', 's', 'y', 's', '-' }; + \\ return []u16{ 'm', 's', 'y', 's', '-' }; + \\} + \\ + ); +} + +test "zig fmt: first thing in file is line comment" { + try testCanonical( + \\// Introspection and determination of system libraries needed by zig. + \\ + \\// Introspection and determination of system libraries needed by zig. + \\ + \\const std = @import("std"); + \\ + ); +} + +test "zig fmt: line comment after doc comment" { + try testCanonical( + \\/// doc comment + \\// line comment + \\fn foo() void {} + \\ + ); +} + +test "zig fmt: bit field alignment" { + try testCanonical( + \\test { + \\ assert(@TypeOf(&blah.b) == *align(1:3:6) const u3); + \\} + \\ + ); +} + +test "zig fmt: nested switch" { + try testCanonical( + \\test { + \\ switch (state) { + \\ TermState.Start => switch (c) { + \\ '\x1b' => state = TermState.Escape, + \\ else => try out.writeByte(c), + \\ }, + \\ } + \\} + \\ + ); +} + +test "zig fmt: float literal with exponent" { + try testCanonical( + \\pub const f64_true_min = 4.94065645841246544177e-324; + \\const threshold = 0x1.a827999fcef32p+1022; + \\ + ); +} + +test "zig fmt: if-else end of comptime" { + try testCanonical( + \\comptime { + \\ if (a) { + \\ b(); + \\ } else { + \\ b(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: nested blocks" { + try testCanonical( + \\comptime { + \\ { + \\ { + \\ { + \\ a(); + \\ } + \\ } + \\ } + \\} + \\ + ); +} + +test "zig fmt: block with same line comment after end brace" { + try testCanonical( + \\comptime { + \\ { + \\ b(); + \\ } // comment + \\} + \\ + ); +} + +test "zig fmt: statements with comment between" { + try testCanonical( + \\comptime { + \\ a = b; + \\ // comment + \\ a = b; + \\} + \\ + ); +} + +test "zig fmt: statements with empty line between" { + try testCanonical( + \\comptime { + \\ a = b; + \\ + \\ a = b; + \\} + \\ + ); +} + +test "zig fmt: ptr deref operator and unwrap optional operator" { + try testCanonical( + \\const a = b.*; + \\const a = b.?; + \\ + ); +} + +test "zig fmt: comment after if before another if" { + try testCanonical( + \\test "aoeu" { + \\ // comment + \\ if (x) { + \\ bar(); + \\ } + \\} + \\ + \\test "aoeu" { + \\ if (x) { + \\ foo(); + \\ } + \\ // comment + \\ if (x) { + \\ bar(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: line comment between if block and else keyword" { + try testCanonical( + \\test "aoeu" { + \\ // cexp(finite|nan +- i inf|nan) = nan + i nan + \\ if ((hx & 0x7fffffff) != 0x7f800000) { + \\ return Complex(f32).init(y - y, y - y); + \\ } + \\ // cexp(-inf +- i inf|nan) = 0 + i0 + \\ else if (hx & 0x80000000 != 0) { + \\ return Complex(f32).init(0, 0); + \\ } + \\ // cexp(+inf +- i inf|nan) = inf + i nan + \\ // another comment + \\ else { + \\ return Complex(f32).init(x, y - y); + \\ } + \\} + \\ + ); +} + +test "zig fmt: same line comments in expression" { + try testCanonical( + \\test "aoeu" { + \\ const x = ( // a + \\ 0 // b + \\ ); // c + \\} + \\ + ); +} + +test "zig fmt: add comma on last switch prong" { + try testTransform( + \\test "aoeu" { + \\switch (self.init_arg_expr) { + \\ InitArg.Type => |t| { }, + \\ InitArg.None, + \\ InitArg.Enum => { } + \\} + \\ switch (self.init_arg_expr) { + \\ InitArg.Type => |t| { }, + \\ InitArg.None, + \\ InitArg.Enum => { }//line comment + \\ } + \\} + , + \\test "aoeu" { + \\ switch (self.init_arg_expr) { + \\ InitArg.Type => |t| {}, + \\ InitArg.None, InitArg.Enum => {}, + \\ } + \\ switch (self.init_arg_expr) { + \\ InitArg.Type => |t| {}, + \\ InitArg.None, InitArg.Enum => {}, //line comment + \\ } + \\} + \\ + ); +} + +test "zig fmt: same-line comment after a statement" { + try testCanonical( + \\test "" { + \\ a = b; + \\ debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption + \\ a = b; + \\} + \\ + ); +} + +test "zig fmt: same-line comment after var decl in struct" { + try testCanonical( + \\pub const vfs_cap_data = extern struct { + \\ const Data = struct {}; // when on disk. + \\}; + \\ + ); +} + +test "zig fmt: same-line comment after field decl" { + try testCanonical( + \\pub const dirent = extern struct { + \\ d_name: u8, + \\ d_name: u8, // comment 1 + \\ d_name: u8, + \\ d_name: u8, // comment 2 + \\ d_name: u8, + \\}; + \\ + ); +} + +test "zig fmt: same-line comment after switch prong" { + try testCanonical( + \\test "" { + \\ switch (err) { + \\ error.PathAlreadyExists => {}, // comment 2 + \\ else => return err, // comment 1 + \\ } + \\} + \\ + ); +} + +test "zig fmt: same-line comment after non-block if expression" { + try testCanonical( + \\comptime { + \\ if (sr > n_uword_bits - 1) // d > r + \\ return 0; + \\} + \\ + ); +} + +test "zig fmt: same-line comment on comptime expression" { + try testCanonical( + \\test "" { + \\ comptime assert(@typeInfo(T) == .Int); // must pass an integer to absInt + \\} + \\ + ); +} + +test "zig fmt: switch with empty body" { + try testCanonical( + \\test "" { + \\ foo() catch |err| switch (err) {}; + \\} + \\ + ); +} + +test "zig fmt: line comments in struct initializer" { + try testCanonical( + \\fn foo() void { + \\ return Self{ + \\ .a = b, + \\ + \\ // Initialize these two fields to buffer_size so that + \\ // in `readFn` we treat the state as being able to read + \\ .start_index = buffer_size, + \\ .end_index = buffer_size, + \\ + \\ // middle + \\ + \\ .a = b, + \\ + \\ // end + \\ }; + \\} + \\ + ); +} + +test "zig fmt: first line comment in struct initializer" { + try testCanonical( + \\pub fn acquire(self: *Self) HeldLock { + \\ return HeldLock{ + \\ // guaranteed allocation elision + \\ .held = self.lock.acquire(), + \\ .value = &self.private_data, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: doc comments before struct field" { + try testCanonical( + \\pub const Allocator = struct { + \\ /// Allocate byte_count bytes and return them in a slice, with the + \\ /// slice's pointer aligned at least to alignment bytes. + \\ allocFn: fn () void, + \\}; + \\ + ); +} + +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error{ + \\ A, + \\ B, + \\ + \\ C, + \\}; + \\ + \\const Error = error{ + \\ /// no more memory + \\ OutOfMemory, + \\}; + \\ + \\const Error = error{ + \\ /// no more memory + \\ OutOfMemory, + \\ + \\ /// another + \\ Another, + \\ + \\ // end + \\}; + \\ + \\const Error = error{OutOfMemory}; + \\const Error = error{}; + \\ + \\const Error = error{ OutOfMemory, OutOfTime }; + \\ + ); +} + +test "zig fmt: union(enum(u32)) with assigned enum values" { + try testCanonical( + \\const MultipleChoice = union(enum(u32)) { + \\ A = 20, + \\ B = 40, + \\ C = 60, + \\ D = 1000, + \\}; + \\ + ); +} + +test "zig fmt: resume from suspend block" { + try testCanonical( + \\fn foo() void { + \\ suspend { + \\ resume @frame(); + \\ } + \\} + \\ + ); +} + +test "zig fmt: comments before error set decl" { + try testCanonical( + \\const UnexpectedError = error{ + \\ /// The Operating System returned an undocumented error code. + \\ Unexpected, + \\ // another + \\ Another, + \\ + \\ // in between + \\ + \\ // at end + \\}; + \\ + ); +} + +test "zig fmt: comments before switch prong" { + try testCanonical( + \\test "" { + \\ switch (err) { + \\ error.PathAlreadyExists => continue, + \\ + \\ // comment 1 + \\ + \\ // comment 2 + \\ else => return err, + \\ // at end + \\ } + \\} + \\ + ); +} + +test "zig fmt: comments before var decl in struct" { + try testCanonical( + \\pub const vfs_cap_data = extern struct { + \\ // All of these are mandated as little endian + \\ // when on disk. + \\ const Data = struct { + \\ permitted: u32, + \\ inheritable: u32, + \\ }; + \\ + \\ // in between + \\ + \\ /// All of these are mandated as little endian + \\ /// when on disk. + \\ const Data = struct { + \\ permitted: u32, + \\ inheritable: u32, + \\ }; + \\ + \\ // at end + \\}; + \\ + ); +} + +test "zig fmt: array literal with 1 item on 1 line" { + try testCanonical( + \\var s = []const u64{0} ** 25; + \\ + ); +} + +test "zig fmt: comments before global variables" { + try testCanonical( + \\/// Foo copies keys and values before they go into the map, and + \\/// frees them when they get removed. + \\pub const Foo = struct {}; + \\ + ); +} + +test "zig fmt: comments in statements" { + try testCanonical( + \\test "std" { + \\ // statement comment + \\ _ = @import("foo/bar.zig"); + \\ + \\ // middle + \\ // middle2 + \\ + \\ // end + \\} + \\ + ); +} + +test "zig fmt: comments before test decl" { + try testCanonical( + \\// top level normal comment + \\test "hi" {} + \\ + \\// middle + \\ + \\// end + \\ + ); +} + +test "zig fmt: preserve spacing" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = std.io.getStdOut; + \\ var stdout_file = std.io.getStdOut; + \\ + \\ var stdout_file = std.io.getStdOut; + \\ var stdout_file = std.io.getStdOut; + \\} + \\ + ); +} + +test "zig fmt: return types" { + try testCanonical( + \\pub fn main() !void {} + \\pub fn main() FooBar {} + \\pub fn main() i32 {} + \\ + ); +} + +test "zig fmt: imports" { + try testCanonical( + \\const std = @import("std"); + \\const std = @import(); + \\ + ); +} + +test "zig fmt: global declarations" { + try testCanonical( + \\const a = b; + \\pub const a = b; + \\var a = b; + \\pub var a = b; + \\const a: i32 = b; + \\pub const a: i32 = b; + \\var a: i32 = b; + \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; + \\ + ); +} + +test "zig fmt: extern declaration" { + try testCanonical( + \\extern var foo: c_int; + \\ + ); +} + +test "zig fmt: alignment" { + try testCanonical( + \\var foo: c_int align(1); + \\ + ); +} + +test "zig fmt: C main" { + try testCanonical( + \\fn main(argc: c_int, argv: **u8) c_int { + \\ const a = b; + \\} + \\ + ); +} + +test "zig fmt: return" { + try testCanonical( + \\fn foo(argc: c_int, argv: **u8) c_int { + \\ return 0; + \\} + \\ + \\fn bar() void { + \\ return; + \\} + \\ + ); +} + +test "zig fmt: function attributes" { + try testCanonical( + \\export fn foo() void {} + \\pub export fn foo() void {} + \\extern fn foo() void; + \\pub extern fn foo() void; + \\extern "c" fn foo() void; + \\pub extern "c" fn foo() void; + \\noinline fn foo() void {} + \\pub noinline fn foo() void {} + \\ + ); +} + +test "zig fmt: nested pointers with ** tokens" { + try testCanonical( + \\const x: *u32 = undefined; + \\const x: **u32 = undefined; + \\const x: ***u32 = undefined; + \\const x: ****u32 = undefined; + \\const x: *****u32 = undefined; + \\const x: ******u32 = undefined; + \\const x: *******u32 = undefined; + \\ + ); +} + +test "zig fmt: pointer attributes" { + try testCanonical( + \\extern fn f1(s: *align(*u8) u8) c_int; + \\extern fn f2(s: **align(1) *const *volatile u8) c_int; + \\extern fn f3(s: *align(1) const *align(1) volatile *const volatile u8) c_int; + \\extern fn f4(s: *align(1) const volatile u8) c_int; + \\extern fn f5(s: [*:0]align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: slice attributes" { + try testCanonical( + \\extern fn f1(s: []align(*u8) u8) c_int; + \\extern fn f2(s: []align(1) []const []volatile u8) c_int; + \\extern fn f3(s: []align(1) const [:0]align(1) volatile []const volatile u8) c_int; + \\extern fn f4(s: []align(1) const volatile u8) c_int; + \\extern fn f5(s: [:0]align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: test declaration" { + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); +} + +test "zig fmt: infix operators" { + try testCanonical( + \\test { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i orelse i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a == b and c == d; + \\} + \\ + ); +} + +test "zig fmt: prefix operators" { + try testCanonical( + \\test "prefix operators" { + \\ try return --%~!&0; + \\} + \\ + ); +} + +test "zig fmt: call expression" { + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); +} + +test "zig fmt: anytype type" { + try testCanonical( + \\fn print(args: anytype) @This() {} + \\ + ); +} + +test "zig fmt: functions" { + try testCanonical( + \\extern fn puts(s: *const u8) c_int; + \\extern "c" fn puts(s: *const u8) c_int; + \\export fn puts(s: *const u8) c_int; + \\inline fn puts(s: *const u8) c_int; + \\noinline fn puts(s: *const u8) c_int; + \\pub extern fn puts(s: *const u8) c_int; + \\pub extern "c" fn puts(s: *const u8) c_int; + \\pub export fn puts(s: *const u8) c_int; + \\pub inline fn puts(s: *const u8) c_int; + \\pub noinline fn puts(s: *const u8) c_int; + \\pub extern fn puts(s: *const u8) align(2 + 2) c_int; + \\pub extern "c" fn puts(s: *const u8) align(2 + 2) c_int; + \\pub export fn puts(s: *const u8) align(2 + 2) c_int; + \\pub inline fn puts(s: *const u8) align(2 + 2) c_int; + \\pub noinline fn puts(s: *const u8) align(2 + 2) c_int; + \\pub fn callInlineFn(func: fn () callconv(.Inline) void) void { + \\ func(); + \\} + \\ + ); +} + +test "zig fmt: multiline string" { + try testCanonical( + \\test "" { + \\ const s1 = + \\ \\one + \\ \\two) + \\ \\three + \\ ; + \\ const s3 = // hi + \\ \\one + \\ \\two) + \\ \\three + \\ ; + \\} + \\ + ); +} + +test "zig fmt: values" { + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ anyerror; + \\ this; + \\ unreachable; + \\} + \\ + ); +} + +test "zig fmt: indexing" { + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; + \\} + \\ + ); +} + +test "zig fmt: struct declaration" { + try testCanonical( + \\const S = struct { + \\ const Self = @This(); + \\ f1: u8, + \\ f3: u8, + \\ + \\ f2: u8, + \\ + \\ fn method(self: *Self) Self { + \\ return self.*; + \\ } + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + \\const Ps = packed struct(u32) { + \\ a: u1, + \\ b: u2, + \\ + \\ c: u29, + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + ); +} + +test "zig fmt: enum declaration" { + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0, + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird, + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + ); +} + +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool, + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool, + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ None, + \\ Bool, + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool, + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool, + \\}; + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ + \\ 1, + \\ 2, + \\ }; + \\ const a: [2]u8 = []u8{ + \\ 1, + \\ 2, + \\ }; + \\ const a: [0]u8 = []u8{}; + \\ const x: [4:0]u8 = undefined; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a0 = []u8{}; + \\const a1 = []u8{1}; + \\const a2 = []u8{ + \\ 1, + \\ 2, + \\ 3, + \\ 4, + \\}; + \\const s0 = S{}; + \\const s1 = S{ .a = 1 }; + \\const s2 = S{ + \\ .a = 1, + \\ .b = 2, + \\}; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: anyerror!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch + \\ return; + \\ _ = a catch |err| return; + \\ _ = a catch |err| + \\ return; + \\} + \\ + ); +} + +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4...7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ }, + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ 1 => a = 4, + \\ else => 4, + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64, + \\ }; + \\ + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable, + \\ 1 => |a, b| unreachable, + \\ 2 => |*a, b| unreachable, + \\ } + \\} + \\ + ); + + try testTransform( + \\test { + \\ switch (x) { + \\ foo => + \\ "bar", + \\ } + \\} + \\ + , + \\test { + \\ switch (x) { + \\ foo => "bar", + \\ } + \\} + \\ + ); +} + +test "zig fmt: switch multiline string" { + try testCanonical( + \\test "switch multiline string" { + \\ const x: u32 = 0; + \\ const str = switch (x) { + \\ 1 => "one", + \\ 2 => + \\ \\ Comma after the multiline string + \\ \\ is needed + \\ , + \\ 3 => "three", + \\ else => "else", + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64, + \\ }; + \\ + \\ const str = switch (u) { + \\ Union.Int => |int| + \\ \\ Comma after the multiline string + \\ \\ is needed + \\ , + \\ Union.Float => |*float| unreachable, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) unreachable; + \\ + \\ while (10 < 1) unreachable else unreachable; + \\ + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) continue; + \\ + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ const res = while (0 < 10) + \\ break 7 + \\ else + \\ unreachable; + \\ + \\ var a: anyerror!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) : (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| continue; + \\ + \\ for (a) |v| continue else return; + \\ + \\ for (a) |v| { + \\ continue; + \\ } else return; + \\ + \\ for (a) |v| continue else { + \\ return; + \\ } + \\ + \\ for (a) |v| + \\ continue + \\ else + \\ return; + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a, 0..) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a, 0..) |v, i| + \\ continue; + \\ + \\ for (a) |b| switch (b) { + \\ c => {}, + \\ d => {}, + \\ }; + \\ + \\ const res = for (a, 0..) |v, i| { + \\ break v; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var num: usize = 0; + \\ inline for (a, 0..1) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\ + \\ for (a, b) | + \\ long_name, + \\ another_long_name, + \\ | { + \\ continue; + \\ } + \\} + \\ + ); + + try testTransform( + \\test "fix for" { + \\ for (a) |x| + \\ f(x) else continue; + \\} + \\ + , + \\test "fix for" { + \\ for (a) |x| + \\ f(x) + \\ else + \\ continue; + \\} + \\ + ); + + try testTransform( + \\test "fix for" { + \\ for (a, b, c,) |long, another, third,| {} + \\} + \\ + , + \\test "fix for" { + \\ for ( + \\ a, + \\ b, + \\ c, + \\ ) | + \\ long, + \\ another, + \\ third, + \\ | {} + \\} + \\ + ); +} + +test "zig fmt: for if" { + try testCanonical( + \\test { + \\ for (a) |x| if (x) f(x); + \\ + \\ for (a) |x| if (x) + \\ f(x); + \\ + \\ for (a) |x| if (x) { + \\ f(x); + \\ }; + \\ + \\ for (a) |x| + \\ if (x) + \\ f(x); + \\ + \\ for (a) |x| + \\ if (x) { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: if for" { + try testCanonical( + \\test { + \\ if (a) for (x) |x| f(x); + \\ + \\ if (a) for (x) |x| + \\ f(x); + \\ + \\ if (a) for (x) |x| { + \\ f(x); + \\ }; + \\ + \\ if (a) + \\ for (x) |x| + \\ f(x); + \\ + \\ if (a) + \\ for (x) |x| { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: while if" { + try testCanonical( + \\test { + \\ while (a) if (x) f(x); + \\ + \\ while (a) if (x) + \\ f(x); + \\ + \\ while (a) if (x) { + \\ f(x); + \\ }; + \\ + \\ while (a) + \\ if (x) + \\ f(x); + \\ + \\ while (a) + \\ if (x) { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: if while" { + try testCanonical( + \\test { + \\ if (a) while (x) : (cont) f(x); + \\ + \\ if (a) while (x) : (cont) + \\ f(x); + \\ + \\ if (a) while (x) : (cont) { + \\ f(x); + \\ }; + \\ + \\ if (a) + \\ while (x) : (cont) + \\ f(x); + \\ + \\ if (a) + \\ while (x) : (cont) { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: while for" { + try testCanonical( + \\test { + \\ while (a) for (x) |x| f(x); + \\ + \\ while (a) for (x) |x| + \\ f(x); + \\ + \\ while (a) for (x) |x| { + \\ f(x); + \\ }; + \\ + \\ while (a) + \\ for (x) |x| + \\ f(x); + \\ + \\ while (a) + \\ for (x) |x| { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: for while" { + try testCanonical( + \\test { + \\ for (a) |a| while (x) |x| f(x); + \\ + \\ for (a) |a| while (x) |x| + \\ f(x); + \\ + \\ for (a) |a| while (x) |x| { + \\ f(x); + \\ }; + \\ + \\ for (a) |a| + \\ while (x) |x| + \\ f(x); + \\ + \\ for (a) |a| + \\ while (x) |x| { + \\ f(x); + \\ }; + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ const some_number = 1 + if (10 < 0) 2 else 3; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: anyerror!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: fix single statement if/for/while line breaks" { + try testTransform( + \\test { + \\ if (cond) a + \\ else b; + \\ + \\ if (cond) + \\ a + \\ else b; + \\ + \\ for (xs) |x| foo() + \\ else bar(); + \\ + \\ for (xs) |x| + \\ foo() + \\ else bar(); + \\ + \\ while (a) : (b) foo() + \\ else bar(); + \\ + \\ while (a) : (b) + \\ foo() + \\ else bar(); + \\} + \\ + , + \\test { + \\ if (cond) a else b; + \\ + \\ if (cond) + \\ a + \\ else + \\ b; + \\ + \\ for (xs) |x| foo() else bar(); + \\ + \\ for (xs) |x| + \\ foo() + \\ else + \\ bar(); + \\ + \\ while (a) : (b) foo() else bar(); + \\ + \\ while (a) : (b) + \\ foo() + \\ else + \\ bar(); + \\} + \\ + ); +} + +test "zig fmt: anon struct/array literal in if" { + try testCanonical( + \\test { + \\ const a = if (cond) .{ + \\ 1, 2, + \\ 3, 4, + \\ } else .{ + \\ 1, + \\ 2, + \\ 3, + \\ }; + \\ + \\ const rl_and_tag: struct { rl: ResultLoc, tag: zir.Inst.Tag } = if (any_payload_is_ref) .{ + \\ .rl = .ref, + \\ .tag = .switchbr_ref, + \\ } else .{ + \\ .rl = .none, + \\ .tag = .switchbr, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const a: fn (u8) u8 = undefined; + \\const b: fn (u8) callconv(.Naked) u8 = undefined; + \\const ap: fn (u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize), + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1), + \\ : "rcx", "r11" + \\ ); + \\} + \\ + ); +} + +test "zig fmt: async functions" { + try testCanonical( + \\fn simpleAsyncFn() void { + \\ const a = async a.b(); + \\ x += 1; + \\ suspend {} + \\ x += 1; + \\ suspend {} + \\ const p: anyframe->void = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "suspend, resume, await" { + \\ const p: anyframe = async testAsyncSeq(); + \\ resume p; + \\ await p; + \\} + \\ + ); +} + +test "zig fmt: nosuspend" { + try testCanonical( + \\const a = nosuspend foo(); + \\ + ); +} + +test "zig fmt: Block after if" { + try testCanonical( + \\test { + \\ if (true) { + \\ const a = 0; + \\ } + \\ + \\ { + \\ const a = 0; + \\ } + \\} + \\ + ); +} + +test "zig fmt: usingnamespace" { + try testCanonical( + \\usingnamespace @import("std"); + \\pub usingnamespace @import("std"); + \\ + ); +} + +test "zig fmt: string identifier" { + try testCanonical( + \\const @"a b" = @"c d".@"e f"; + \\fn @"g h"() void {} + \\ + ); +} + +test "zig fmt: error return" { + try testCanonical( + \\fn err() anyerror { + \\ call(); + \\ return error.InvalidArgs; + \\} + \\ + ); +} + +test "zig fmt: comptime block in container" { + try testCanonical( + \\pub fn container() type { + \\ return struct { + \\ comptime { + \\ if (false) { + \\ unreachable; + \\ } + \\ } + \\ }; + \\} + \\ + ); +} + +test "zig fmt: inline asm parameter alignment" { + try testCanonical( + \\pub fn main() void { + \\ asm volatile ( + \\ \\ foo + \\ \\ bar + \\ ); + \\ asm volatile ( + \\ \\ foo + \\ \\ bar + \\ : [_] "" (-> usize), + \\ [_] "" (-> usize), + \\ ); + \\ asm volatile ( + \\ \\ foo + \\ \\ bar + \\ : + \\ : [_] "" (0), + \\ [_] "" (0), + \\ ); + \\ asm volatile ( + \\ \\ foo + \\ \\ bar + \\ ::: "", ""); + \\ asm volatile ( + \\ \\ foo + \\ \\ bar + \\ : [_] "" (-> usize), + \\ [_] "" (-> usize), + \\ : [_] "" (0), + \\ [_] "" (0), + \\ : "", "" + \\ ); + \\} + \\ + ); +} + +test "zig fmt: multiline string in array" { + try testCanonical( + \\const Foo = [][]const u8{ + \\ \\aaa + \\ , + \\ \\bbb + \\}; + \\ + \\fn bar() void { + \\ const Foo = [][]const u8{ + \\ \\aaa + \\ , + \\ \\bbb + \\ }; + \\ const Bar = [][]const u8{ // comment here + \\ \\aaa + \\ \\ + \\ , // and another comment can go here + \\ \\bbb + \\ }; + \\} + \\ + ); +} + +test "zig fmt: if type expr" { + try testCanonical( + \\const mycond = true; + \\pub fn foo() if (mycond) i32 else void { + \\ if (mycond) { + \\ return 42; + \\ } + \\} + \\ + ); +} +test "zig fmt: file ends with struct field" { + try testCanonical( + \\a: bool + \\ + ); +} + +test "zig fmt: comment after empty comment" { + try testCanonical( + \\const x = true; // + \\// + \\// + \\//a + \\ + ); +} + +test "zig fmt: line comment in array" { + try testTransform( + \\test "a" { + \\ var arr = [_]u32{ + \\ 0 + \\ // 1, + \\ // 2, + \\ }; + \\} + \\ + , + \\test "a" { + \\ var arr = [_]u32{ + \\ 0, + \\ // 1, + \\ // 2, + \\ }; + \\} + \\ + ); + try testCanonical( + \\test "a" { + \\ var arr = [_]u32{ + \\ 0, + \\ // 1, + \\ // 2, + \\ }; + \\} + \\ + ); +} + +test "zig fmt: comment after params" { + try testTransform( + \\fn a( + \\ b: u32 + \\ // c: u32, + \\ // d: u32, + \\) void {} + \\ + , + \\fn a( + \\ b: u32, + \\ // c: u32, + \\ // d: u32, + \\) void {} + \\ + ); + try testCanonical( + \\fn a( + \\ b: u32, + \\ // c: u32, + \\ // d: u32, + \\) void {} + \\ + ); +} + +test "zig fmt: comment in array initializer/access" { + try testCanonical( + \\test "a" { + \\ var a = x{ //aa + \\ //bb + \\ }; + \\ var a = []x{ //aa + \\ //bb + \\ }; + \\ var b = [ //aa + \\ _ + \\ ]x{ //aa + \\ //bb + \\ 9, + \\ }; + \\ var c = b[ //aa + \\ 0 + \\ ]; + \\ var d = [ + \\ _ + \\ //aa + \\ : + \\ 0 + \\ ]x{ //aa + \\ //bb + \\ 9, + \\ }; + \\ var e = d[ + \\ 0 + \\ //aa + \\ ]; + \\} + \\ + ); +} + +test "zig fmt: comments at several places in struct init" { + try testTransform( + \\var bar = Bar{ + \\ .x = 10, // test + \\ .y = "test" + \\ // test + \\}; + \\ + , + \\var bar = Bar{ + \\ .x = 10, // test + \\ .y = "test", + \\ // test + \\}; + \\ + ); + + try testCanonical( + \\var bar = Bar{ // test + \\ .x = 10, // test + \\ .y = "test", + \\ // test + \\}; + \\ + ); +} + +test "zig fmt: container doc comments" { + try testCanonical( + \\//! tld 1 + \\//! tld 2 + \\//! tld 3 + \\ + \\// comment + \\ + \\/// A doc + \\const A = struct { + \\ //! A tld 1 + \\ //! A tld 2 + \\ //! A tld 3 + \\}; + \\ + \\/// B doc + \\const B = struct { + \\ //! B tld 1 + \\ //! B tld 2 + \\ //! B tld 3 + \\ + \\ /// B doc + \\ b: u32, + \\}; + \\ + \\/// C doc + \\const C = union(enum) { // comment + \\ //! C tld 1 + \\ //! C tld 2 + \\ //! C tld 3 + \\}; + \\ + \\/// D doc + \\const D = union(Foo) { + \\ //! D tld 1 + \\ //! D tld 2 + \\ //! D tld 3 + \\ + \\ /// D doc + \\ b: u32, + \\}; + \\ + ); + try testCanonical( + \\//! Top-level documentation. + \\ + \\/// This is A + \\pub const A = usize; + \\ + ); + try testCanonical( + \\//! Nothing here + \\ + ); +} + +test "zig fmt: remove newlines surrounding doc comment" { + try testTransform( + \\ + \\ + \\ + \\/// doc comment + \\ + \\fn foo() void {} + \\ + , + \\/// doc comment + \\fn foo() void {} + \\ + ); +} + +test "zig fmt: remove newlines surrounding doc comment within container decl" { + try testTransform( + \\const Foo = struct { + \\ + \\ + \\ /// doc comment + \\ + \\ fn foo() void {} + \\}; + \\ + , + \\const Foo = struct { + \\ /// doc comment + \\ fn foo() void {} + \\}; + \\ + ); +} + +test "zig fmt: comptime before comptime field" { + try testError( + \\const Foo = struct { + \\ a: i32, + \\ comptime comptime b: i32 = 1234, + \\}; + \\ + , &[_]Error{ + .expected_comma_after_field, + }); +} + +test "zig fmt: invalid doc comments on comptime and test blocks" { + try testError( + \\/// This is a doc comment for a comptime block. + \\comptime {} + \\/// This is a doc comment for a test + \\test "This is my test" {} + , &[_]Error{ + .comptime_doc_comment, + .test_doc_comment, + }); +} + +test "zig fmt: invalid else branch statement" { + try testError( + \\comptime { + \\ if (true) {} else var a = 0; + \\ if (true) {} else defer {} + \\} + \\comptime { + \\ while (true) {} else var a = 0; + \\ while (true) {} else defer {} + \\} + \\comptime { + \\ for ("") |_| {} else var a = 0; + \\ for ("") |_| {} else defer {} + \\} + , &[_]Error{ + .expected_expr_or_assignment, + .expected_expr_or_assignment, + .expected_expr_or_assignment, + .expected_expr_or_assignment, + .expected_expr_or_assignment, + .expected_expr_or_assignment, + }); +} + +test "zig fmt: anytype struct field" { + try testError( + \\pub const Pointer = struct { + \\ sentinel: anytype, + \\}; + \\ + , &[_]Error{ + .expected_type_expr, + }); +} + +test "zig fmt: extern without container keyword returns error" { + try testError( + \\const container = extern {}; + \\ + , &[_]Error{ + .expected_container, + }); +} + +test "zig fmt: same line doc comment returns error" { + try testError( + \\const Foo = struct{ + \\ bar: u32, /// comment + \\ foo: u32, /// comment + \\ /// comment + \\}; + \\ + \\const a = 42; /// comment + \\ + \\extern fn foo() void; /// comment + \\ + \\/// comment + \\ + , &[_]Error{ + .same_line_doc_comment, + .same_line_doc_comment, + .unattached_doc_comment, + .same_line_doc_comment, + .same_line_doc_comment, + .unattached_doc_comment, + }); +} + +test "zig fmt: integer literals with underscore separators" { + try testTransform( + \\const + \\ x = + \\ 1_234_567 + \\ + (0b0_1-0o7_0+0xff_FF ) + 1_0; + , + \\const x = + \\ 1_234_567 + (0b0_1 - 0o7_0 + 0xff_FF) + 1_0; + \\ + ); +} + +test "zig fmt: hex literals with underscore separators" { + try testTransform( + \\pub fn orMask(a: [ 1_000 ]u64, b: [ 1_000] u64) [1_000]u64 { + \\ var c: [1_000]u64 = [1]u64{ 0xFFFF_FFFF_FFFF_FFFF}**1_000; + \\ for (c [ 1_0 .. ], 0..) |_, i| { + \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA; + \\ } + \\ return c; + \\} + \\ + \\ + , + \\pub fn orMask(a: [1_000]u64, b: [1_000]u64) [1_000]u64 { + \\ var c: [1_000]u64 = [1]u64{0xFFFF_FFFF_FFFF_FFFF} ** 1_000; + \\ for (c[1_0..], 0..) |_, i| { + \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA; + \\ } + \\ return c; + \\} + \\ + ); +} + +test "zig fmt: decimal float literals with underscore separators" { + try testTransform( + \\pub fn main() void { + \\ const a:f64=(10.0e-0+(10.0e+0))+10_00.00_00e-2+20_00.00_10e+4; + \\ const b:f64=1_0.0--10_10.0+1_0_0.0_0+1e2; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + , + \\pub fn main() void { + \\ const a: f64 = (10.0e-0 + (10.0e+0)) + 10_00.00_00e-2 + 20_00.00_10e+4; + \\ const b: f64 = 1_0.0 - -10_10.0 + 1_0_0.0_0 + 1e2; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + \\ + ); +} + +test "zig fmt: hexadeciaml float literals with underscore separators" { + try testTransform( + \\pub fn main() void { + \\ const a: f64 = (0x10.0p-0+(0x10.0p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16; + \\ const b: f64 = 0x0010.0--0x00_10.0+0x10.00+0x1p4; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + , + \\pub fn main() void { + \\ const a: f64 = (0x10.0p-0 + (0x10.0p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16; + \\ const b: f64 = 0x0010.0 - -0x00_10.0 + 0x10.00 + 0x1p4; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + \\ + ); +} + +test "zig fmt: C var args" { + try testCanonical( + \\pub extern "c" fn printf(format: [*:0]const u8, ...) c_int; + \\ + ); +} + +test "zig fmt: Only indent multiline string literals in function calls" { + try testCanonical( + \\test "zig fmt:" { + \\ try testTransform( + \\ \\const X = struct { + \\ \\ foo: i32, bar: i8 }; + \\ , + \\ \\const X = struct { + \\ \\ foo: i32, bar: i8 + \\ \\}; + \\ \\ + \\ ); + \\} + \\ + ); +} + +test "zig fmt: Don't add extra newline after if" { + try testCanonical( + \\pub fn atomicSymLink(allocator: Allocator, existing_path: []const u8, new_path: []const u8) !void { + \\ if (cwd().symLink(existing_path, new_path, .{})) { + \\ return; + \\ } + \\} + \\ + ); +} + +test "zig fmt: comments in ternary ifs" { + try testCanonical( + \\const x = if (true) { + \\ 1; + \\} else if (false) + \\ // Comment + \\ 0; + \\const y = if (true) + \\ // Comment + \\ 1 + \\else + \\ // Comment + \\ 0; + \\ + \\pub extern "c" fn printf(format: [*:0]const u8, ...) c_int; + \\ + ); +} + +test "zig fmt: while statement in blockless if" { + try testCanonical( + \\pub fn main() void { + \\ const zoom_node = if (focused_node == layout_first) + \\ while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null + \\ else + \\ focused_node; + \\} + \\ + ); +} + +test "zig fmt: test comments in field access chain" { + try testCanonical( + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ .more().more() // + \\ .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more.more // + \\ .more() // + \\ // .more() // + \\ // .more() // + \\ // .more() // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + \\pub const str = struct { + \\ pub const Thing = more // + \\ .more // + \\ .more() // + \\ .more(); + \\ data: Data, + \\}; + \\ + ); +} + +test "zig fmt: allow line break before field access" { + try testCanonical( + \\test { + \\ const w = foo.bar().zippy(zag).iguessthisisok(); + \\ + \\ const x = foo + \\ .bar() + \\ . // comment + \\ // comment + \\ swooop().zippy(zag) + \\ .iguessthisisok(); + \\ + \\ const y = view.output.root.server.input_manager.default_seat.wlr_seat.name; + \\ + \\ const z = view.output.root.server + \\ .input_manager // + \\ .default_seat + \\ . // comment + \\ // another comment + \\ wlr_seat.name; + \\} + \\ + ); + try testTransform( + \\test { + \\ const x = foo. + \\ bar() + \\ .zippy(zag).iguessthisisok(); + \\ + \\ const z = view.output.root.server. + \\ input_manager. + \\ default_seat.wlr_seat.name; + \\} + \\ + , + \\test { + \\ const x = foo + \\ .bar() + \\ .zippy(zag).iguessthisisok(); + \\ + \\ const z = view.output.root.server + \\ .input_manager + \\ .default_seat.wlr_seat.name; + \\} + \\ + ); +} + +test "zig fmt: Indent comma correctly after multiline string literals in arg list (trailing comma)" { + try testCanonical( + \\fn foo() void { + \\ z.display_message_dialog( + \\ *const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , + \\ g.GtkMessageType.GTK_MESSAGE_WARNING, + \\ null, + \\ ); + \\ + \\ z.display_message_dialog(*const [323:0]u8, + \\ \\Message Text + \\ \\------------ + \\ \\xxxxxxxxxxxx + \\ \\xxxxxxxxxxxx + \\ , g.GtkMessageType.GTK_MESSAGE_WARNING, null); + \\} + \\ + ); +} + +test "zig fmt: Control flow statement as body of blockless if" { + try testCanonical( + \\pub fn main() void { + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null + \\ else + \\ focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null else focused_node; + \\ + \\ const zoom_node = if (focused_node == layout_first) + \\ if (it.next()) { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ } else null; + \\ + \\ const zoom_node = if (focused_node == layout_first) while (it.next()) |node| { + \\ if (!node.view.pending.float and !node.view.pending.fullscreen) break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) for (nodes) |node| { + \\ break node; + \\ }; + \\ + \\ const zoom_node = if (focused_node == layout_first) switch (nodes) { + \\ 0 => 0, + \\ } else focused_node; + \\} + \\ + ); +} + +test "zig fmt: regression test for #5722" { + try testCanonical( + \\pub fn sendViewTags(self: Self) void { + \\ var it = ViewStack(View).iterator(self.output.views.first, std.math.maxInt(u32)); + \\ while (it.next()) |node| + \\ view_tags.append(node.view.current_tags) catch { + \\ c.wl_resource_post_no_memory(self.wl_resource); + \\ log.err(.river_status, "out of memory", .{}); + \\ return; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: regression test for #8974" { + try testCanonical( + \\pub const VARIABLE; + \\ + ); +} + +test "zig fmt: allow trailing line comments to do manual array formatting" { + try testCanonical( + \\fn foo() void { + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ 0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ }); + \\ + \\ di_buf.appendAssumeCapacity(&[_]u8{ + \\ 1, DW.TAG_compile_unit, DW.CHILDREN_no, // header + \\ DW.AT_stmt_list, DW_FORM_data4, // form value pairs + \\ DW.AT_low_pc, DW_FORM_addr, + \\ DW.AT_high_pc, DW_FORM_addr, + \\ DW.AT_name, DW_FORM_strp, + \\ DW.AT_comp_dir, DW_FORM_strp, + \\ DW.AT_producer, DW_FORM_strp, + \\ DW.AT_language, DW_FORM_data2, + \\ 0, 0, // sentinel + \\ }); + \\ + \\ self.code.appendSliceAssumeCapacity(&[_]u8{ + \\ 0x55, // push rbp + \\ 0x48, 0x89, 0xe5, // mov rbp, rsp + \\ // How do we handle this? + \\ //0x48, 0x81, 0xec, // sub rsp, imm32 (with reloc) + \\ // Here's a blank line, should that be allowed? + \\ + \\ 0x48, 0x89, 0xe5, + \\ 0x33, 0x45, + \\ // Now the comment breaks a single line -- how do we handle this? + \\ 0x88, + \\ }); + \\} + \\ + ); +} + +test "zig fmt: multiline string literals should play nice with array initializers" { + try testCanonical( + \\fn main() void { + \\ var a = .{.{.{.{.{.{.{.{ + \\ 0, + \\ }}}}}}}}; + \\ myFunc(.{ + \\ "aaaaaaa", "bbbbbb", "ccccc", + \\ "dddd", ("eee"), ("fff"), + \\ ("gggg"), + \\ // Line comment + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ \\Multiline String Literals can be quite long + \\ , + \\ ( + \\ \\Multiline String Literals can be quite long + \\ ), + \\ .{ + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ .{( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ )}, + \\ .{ + \\ "xxxxxxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxx", + \\ "xxx", + \\ }, + \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, + \\ .{ "xxxxxxx", "xxx", "xxx", "xxx" }, + \\ "aaaaaaa", "bbbbbb", "ccccc", // - + \\ "dddd", ("eee"), ("fff"), + \\ .{ + \\ "xxx", "xxx", + \\ ( + \\ \\ xxx + \\ ), + \\ "xxxxxxxxxxxxxx", + \\ "xxx", + \\ }, + \\ .{ + \\ ( + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ ), + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }, + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ \\xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + \\ }); + \\} + \\ + ); +} + +test "zig fmt: use of comments and multiline string literals may force the parameters over multiple lines" { + try testCanonical( + \\pub fn makeMemUndefined(qzz: []u8) i1 { + \\ cases.add( // fixed bug foo + \\ "compile diagnostic string for top level decl type", + \\ \\export fn entry() void { + \\ \\ var foo: u32 = @This(){}; + \\ \\} + \\ , &[_][]const u8{ + \\ "tmp.zig:2:27: error: type 'u32' does not support array initialization", + \\ }); + \\ @compileError( + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ \\ unknown-length pointers and C pointers cannot be hashed deeply. + \\ \\ Consider providing your own hash function. + \\ ); + \\ return @intCast(doMemCheckClientRequestExpr(0, // default return + \\ .MakeMemUndefined, @intFromPtr(qzz.ptr), qzz.len, 0, 0, 0)); + \\} + \\ + \\// This looks like garbage don't do this + \\const rparen = tree.prevToken( + \\// the first token for the annotation expressions is the left + \\// parenthesis, hence the need for two prevToken + \\if (fn_proto.getAlignExpr()) |align_expr| + \\ tree.prevToken(tree.prevToken(align_expr.firstToken())) + \\else if (fn_proto.getSectionExpr()) |section_expr| + \\ tree.prevToken(tree.prevToken(section_expr.firstToken())) + \\else if (fn_proto.getCallconvExpr()) |callconv_expr| + \\ tree.prevToken(tree.prevToken(callconv_expr.firstToken())) + \\else switch (fn_proto.return_type) { + \\ .Explicit => |node| node.firstToken(), + \\ .InferErrorSet => |node| tree.prevToken(node.firstToken()), + \\ .Invalid => unreachable, + \\}); + \\ + ); +} + +test "zig fmt: single argument trailing commas in @builtins()" { + try testCanonical( + \\pub fn foo(qzz: []u8) i1 { + \\ @panic( + \\ foo, + \\ ); + \\ panic( + \\ foo, + \\ ); + \\ @panic( + \\ foo, + \\ bar, + \\ ); + \\} + \\ + ); +} + +test "zig fmt: trailing comma should force multiline 1 column" { + try testTransform( + \\pub const UUID_NULL: uuid_t = [16]u8{0,0,0,0,}; + \\ + , + \\pub const UUID_NULL: uuid_t = [16]u8{ + \\ 0, + \\ 0, + \\ 0, + \\ 0, + \\}; + \\ + ); +} + +test "zig fmt: function params should align nicely" { + try testCanonical( + \\pub fn foo() void { + \\ cases.addRuntimeSafety("slicing operator with sentinel", + \\ \\const std = @import("std"); + \\ ++ check_panic_msg ++ + \\ \\pub fn main() void { + \\ \\ var buf = [4]u8{'a','b','c',0}; + \\ \\ const slice = buf[0..:0]; + \\ \\} + \\ ); + \\} + \\ + ); +} + +test "zig fmt: fn proto end with anytype and comma" { + try testCanonical( + \\pub fn format( + \\ out_stream: anytype, + \\) !void {} + \\ + ); +} + +test "zig fmt: space after top level doc comment" { + try testCanonical( + \\//! top level doc comment + \\ + \\field: i32, + \\ + ); +} + +test "zig fmt: remove trailing whitespace after container doc comment" { + try testTransform( + \\//! top level doc comment + \\ + , + \\//! top level doc comment + \\ + ); +} + +test "zig fmt: remove trailing whitespace after doc comment" { + try testTransform( + \\/// doc comment + \\a = 0, + \\ + , + \\/// doc comment + \\a = 0, + \\ + ); +} + +test "zig fmt: for loop with ptr payload and index" { + try testCanonical( + \\test { + \\ for (self.entries.items, 0..) |*item, i| {} + \\ for (self.entries.items, 0..) |*item, i| + \\ a = b; + \\ for (self.entries.items, 0..) |*item, i| a = b; + \\} + \\ + ); +} + +test "zig fmt: proper indent line comment after multi-line single expr while loop" { + try testCanonical( + \\test { + \\ while (a) : (b) + \\ foo(); + \\ + \\ // bar + \\ baz(); + \\} + \\ + ); +} + +test "zig fmt: function with labeled block as return type" { + try testCanonical( + \\fn foo() t: { + \\ break :t bar; + \\} { + \\ baz(); + \\} + \\ + ); +} + +test "zig fmt: extern function with missing param name" { + try testCanonical( + \\extern fn a( + \\ *b, + \\ c: *d, + \\) e; + \\extern fn f(*g, h: *i) j; + \\ + ); +} + +test "zig fmt: line comment after multiline single expr if statement with multiline string" { + try testCanonical( + \\test { + \\ if (foo) + \\ x = + \\ \\hello + \\ \\hello + \\ \\ + \\ ; + \\ + \\ // bar + \\ baz(); + \\ + \\ if (foo) + \\ x = + \\ \\hello + \\ \\hello + \\ \\ + \\ else + \\ y = + \\ \\hello + \\ \\hello + \\ \\ + \\ ; + \\ + \\ // bar + \\ baz(); + \\} + \\ + ); +} + +test "zig fmt: respect extra newline between fn and pub usingnamespace" { + try testCanonical( + \\fn foo() void { + \\ bar(); + \\} + \\ + \\pub usingnamespace baz; + \\ + ); +} + +test "zig fmt: respect extra newline between switch items" { + try testCanonical( + \\const a = switch (b) { + \\ .c => {}, + \\ + \\ .d, + \\ .e, + \\ => f, + \\}; + \\ + ); +} + +test "zig fmt: assignment with inline for and inline while" { + try testCanonical( + \\const tmp = inline for (items) |item| {}; + \\ + ); + + try testCanonical( + \\const tmp2 = inline while (true) {}; + \\ + ); +} + +test "zig fmt: saturating arithmetic" { + try testCanonical( + \\test { + \\ const actual = switch (op) { + \\ .add => a +| b, + \\ .sub => a -| b, + \\ .mul => a *| b, + \\ .shl => a <<| b, + \\ }; + \\ switch (op) { + \\ .add => actual +|= b, + \\ .sub => actual -|= b, + \\ .mul => actual *|= b, + \\ .shl => actual <<|= b, + \\ } + \\} + \\ + ); +} + +test "zig fmt: insert trailing comma if there are comments between switch values" { + try testTransform( + \\const a = switch (b) { + \\ .c => {}, + \\ + \\ .d, // foobar + \\ .e + \\ => f, + \\ + \\ .g, .h + \\ // comment + \\ => i, + \\}; + \\ + , + \\const a = switch (b) { + \\ .c => {}, + \\ + \\ .d, // foobar + \\ .e, + \\ => f, + \\ + \\ .g, + \\ .h, + \\ // comment + \\ => i, + \\}; + \\ + ); +} + +test "zig fmt: insert trailing comma if comments in array init" { + try testTransform( + \\var a = .{ + \\ "foo", // + \\ "bar" + \\}; + \\var a = .{ + \\ "foo", + \\ "bar" // + \\}; + \\var a = .{ + \\ "foo", + \\ "//" + \\}; + \\var a = .{ + \\ "foo", + \\ "//" // + \\}; + \\ + , + \\var a = .{ + \\ "foo", // + \\ "bar", + \\}; + \\var a = .{ + \\ "foo", + \\ "bar", // + \\}; + \\var a = .{ "foo", "//" }; + \\var a = .{ + \\ "foo", + \\ "//", // + \\}; + \\ + ); +} + +test "zig fmt: make single-line if no trailing comma" { + try testTransform( + \\test "function call no trailing comma" { + \\ foo( + \\ 1, + \\ 2 + \\ ); + \\} + \\ + , + \\test "function call no trailing comma" { + \\ foo(1, 2); + \\} + \\ + ); + + try testTransform( + \\test "struct no trailing comma" { + \\ const a = .{ + \\ .foo = 1, + \\ .bar = 2 + \\ }; + \\} + \\ + , + \\test "struct no trailing comma" { + \\ const a = .{ .foo = 1, .bar = 2 }; + \\} + \\ + ); + + try testTransform( + \\test "array no trailing comma" { + \\ var stream = multiOutStream(.{ + \\ fbs1.outStream(), + \\ fbs2.outStream() + \\ }); + \\} + \\ + , + \\test "array no trailing comma" { + \\ var stream = multiOutStream(.{ fbs1.outStream(), fbs2.outStream() }); + \\} + \\ + ); +} + +test "zig fmt: preserve container doc comment in container without trailing comma" { + try testTransform( + \\const A = enum(u32) { + \\//! comment + \\_ }; + \\ + , + \\const A = enum(u32) { + \\ //! comment + \\ _, + \\}; + \\ + ); +} + +test "zig fmt: make single-line if no trailing comma, fmt: off" { + try testCanonical( + \\// Test trailing comma syntax + \\// zig fmt: off + \\ + \\extern var a: c_int; + \\extern "c" var b: c_int; + \\export var c: c_int = 0; + \\threadlocal var d: c_int = 0; + \\extern threadlocal var e: c_int; + \\extern "c" threadlocal var f: c_int; + \\export threadlocal var g: c_int = 0; + \\ + \\const struct_trailing_comma = struct { x: i32, y: i32, }; + \\const struct_no_comma = struct { x: i32, y: i32 }; + \\const struct_fn_no_comma = struct { fn m() void {} y: i32 }; + \\ + \\const enum_no_comma = enum { A, B }; + \\ + \\fn container_init() void { + \\ const S = struct { x: i32, y: i32 }; + \\ _ = S { .x = 1, .y = 2 }; + \\ _ = S { .x = 1, .y = 2, }; + \\} + \\ + \\fn type_expr_return1() if (true) A {} + \\fn type_expr_return2() for (true) |_| A {} + \\fn type_expr_return3() while (true) A {} + \\ + \\fn switch_cases(x: i32) void { + \\ switch (x) { + \\ 1,2,3 => {}, + \\ 4,5, => {}, + \\ 6...8, => {}, + \\ else => {}, + \\ } + \\} + \\ + \\fn switch_prongs(x: i32) void { + \\ switch (x) { + \\ 0 => {}, + \\ else => {}, + \\ } + \\ switch (x) { + \\ 0 => {}, + \\ else => {} + \\ } + \\} + \\ + \\const fn_no_comma = fn (i32, i32) void; + \\const fn_trailing_comma = fn (i32, i32,) void; + \\ + \\fn fn_calls() void { + \\ fn add(x: i32, y: i32,) i32 { x + y }; + \\ _ = add(1, 2); + \\ _ = add(1, 2,); + \\} + \\ + \\fn asm_lists() void { + \\ if (false) { // Build AST but don't analyze + \\ asm ("not real assembly" + \\ :[a] "x" (x),); + \\ asm ("not real assembly" + \\ :[a] "x" (->i32),:[a] "x" (1),); + \\ asm volatile ("still not real assembly" + \\ :::"a","b",); + \\ } + \\} + ); +} + +test "zig fmt: variable initialized with ==" { + try testError( + \\comptime { + \\ var z: u32 == 12 + 1; + \\} + , &.{.wrong_equal_var_decl}); +} + +test "zig fmt: missing const/var before local variable in comptime block" { + try testError( + \\comptime { + \\ z: u32; + \\} + \\comptime { + \\ z: u32 align(1); + \\} + \\comptime { + \\ z: u32 addrspace(.generic); + \\} + \\comptime { + \\ z: u32 linksection("foo"); + \\} + \\comptime { + \\ z: u32 = 1; + \\} + , &.{ + .expected_labelable, + .expected_var_const, + .expected_var_const, + .expected_var_const, + .expected_var_const, + }); +} + +test "zig fmt: missing const/var before local variable" { + try testError( + \\std = foo, + \\std = foo; + \\*u32 = foo; + , &.{ + .expected_comma_after_field, + .var_const_decl, + .expected_comma_after_field, + }); +} + +test "zig fmt: while continue expr" { + try testCanonical( + \\test { + \\ while (i > 0) + \\ (i * 2); + \\} + \\ + ); + try testError( + \\test { + \\ while (i > 0) (i -= 1) { + \\ print("test123", .{}); + \\ } + \\} + , &[_]Error{ + .expected_continue_expr, + }); +} + +test "zig fmt: canonicalize symbols (simple)" { + try testTransform( + \\const val_normal: Normal = .{}; + \\const @"val_unesc_me": @"UnescMe" = .{}; + \\const @"val_esc!": @"Esc!" = .{}; + \\ + \\fn fnNormal() void {} + \\fn @"fnUnescMe"() void {} + \\fn @"fnEsc!"() void {} + \\ + \\extern fn protoNormal() void; + \\extern fn @"protoUnescMe"() void; + \\extern fn @"protoEsc!"() void; + \\ + \\fn fnWithArgs(normal: Normal, @"unesc_me": @"UnescMe", @"esc!": @"Esc!") void { + \\ _ = normal; + \\ _ = @"unesc_me"; + \\ _ = @"esc!"; + \\} + \\ + \\const Normal = struct {}; + \\const @"UnescMe" = struct { + \\ @"x": @"X", + \\ const X = union(@"EnumUnesc") { + \\ normal, + \\ @"unesc_me", + \\ @"esc!", + \\ }; + \\ const @"EnumUnesc" = enum { + \\ normal, + \\ @"unesc_me", + \\ @"esc!", + \\ }; + \\}; + \\const @"Esc!" = struct { + \\ normal: bool = false, + \\ @"unesc_me": bool = false, + \\ @"esc!": bool = false, + \\}; + \\ + \\pub fn main() void { + \\ _ = val_normal; + \\ _ = @"val_normal"; + \\ _ = val_unesc_me; + \\ _ = @"val_unesc_me"; + \\ _ = @"val_esc!"; + \\ + \\ fnNormal(); + \\ @"fnNormal"(); + \\ fnUnescMe(); + \\ @"fnUnescMe"(); + \\ @"fnEsc!"(); + \\ + \\ fnWithArgs(1, Normal{}, UnescMe{}, @"Esc!"{}); + \\ fnWithArgs(1, @"Normal"{}, @"UnescMe"{}, @"Esc!"{}); + \\ fnWithArgs(1, @"Normal"{}, @"Normal"{}, @"Esc!"{}); + \\ + \\ const local_val1: @"Normal" = .{}; + \\ const @"local_val2": UnescMe = .{ + \\ .@"x" = .@"unesc_me", + \\ }; + \\ fnWithArgs(@"local_val1", @"local_val2", .{ .@"normal" = true, .@"unesc_me" = true, .@"esc!" = true }); + \\ fnWithArgs(local_val1, local_val2, .{ .normal = true, .unesc_me = true, .@"esc!" = true }); + \\ + \\ var x: u8 = 'x'; + \\ switch (@"x") { + \\ @"x" => {}, + \\ } + \\ + \\ _ = @import("std"); // Don't mess with @builtins + \\ // @"comment" + \\} + \\ + , + \\const val_normal: Normal = .{}; + \\const val_unesc_me: UnescMe = .{}; + \\const @"val_esc!": @"Esc!" = .{}; + \\ + \\fn fnNormal() void {} + \\fn fnUnescMe() void {} + \\fn @"fnEsc!"() void {} + \\ + \\extern fn protoNormal() void; + \\extern fn protoUnescMe() void; + \\extern fn @"protoEsc!"() void; + \\ + \\fn fnWithArgs(normal: Normal, unesc_me: UnescMe, @"esc!": @"Esc!") void { + \\ _ = normal; + \\ _ = unesc_me; + \\ _ = @"esc!"; + \\} + \\ + \\const Normal = struct {}; + \\const UnescMe = struct { + \\ x: X, + \\ const X = union(EnumUnesc) { + \\ normal, + \\ unesc_me, + \\ @"esc!", + \\ }; + \\ const EnumUnesc = enum { + \\ normal, + \\ unesc_me, + \\ @"esc!", + \\ }; + \\}; + \\const @"Esc!" = struct { + \\ normal: bool = false, + \\ unesc_me: bool = false, + \\ @"esc!": bool = false, + \\}; + \\ + \\pub fn main() void { + \\ _ = val_normal; + \\ _ = val_normal; + \\ _ = val_unesc_me; + \\ _ = val_unesc_me; + \\ _ = @"val_esc!"; + \\ + \\ fnNormal(); + \\ fnNormal(); + \\ fnUnescMe(); + \\ fnUnescMe(); + \\ @"fnEsc!"(); + \\ + \\ fnWithArgs(1, Normal{}, UnescMe{}, @"Esc!"{}); + \\ fnWithArgs(1, Normal{}, UnescMe{}, @"Esc!"{}); + \\ fnWithArgs(1, Normal{}, Normal{}, @"Esc!"{}); + \\ + \\ const local_val1: Normal = .{}; + \\ const local_val2: UnescMe = .{ + \\ .x = .unesc_me, + \\ }; + \\ fnWithArgs(local_val1, local_val2, .{ .normal = true, .unesc_me = true, .@"esc!" = true }); + \\ fnWithArgs(local_val1, local_val2, .{ .normal = true, .unesc_me = true, .@"esc!" = true }); + \\ + \\ var x: u8 = 'x'; + \\ switch (x) { + \\ x => {}, + \\ } + \\ + \\ _ = @import("std"); // Don't mess with @builtins + \\ // @"comment" + \\} + \\ + ); +} + +// Contextually unescape when shadowing primitive types and values. +test "zig fmt: canonicalize symbols (primitive types)" { + try testTransform( + \\const @"anyopaque" = struct { + \\ @"u8": @"type" = true, + \\ @"_": @"false" = @"true", + \\ const @"type" = bool; + \\ const @"false" = bool; + \\ const @"true" = false; + \\}; + \\ + \\const U = union(@"null") { + \\ @"type", + \\ const @"null" = enum { + \\ @"type", + \\ }; + \\}; + \\ + \\test { + \\ const E = enum { @"anyopaque" }; + \\ _ = U{ .@"type" = {} }; + \\ _ = U.@"type"; + \\ _ = E.@"anyopaque"; + \\} + \\ + \\fn @"i10"(@"void": @"anyopaque", @"type": @"anyopaque".@"type") error{@"null"}!void { + \\ var @"f32" = @"void"; + \\ @"f32".@"u8" = false; + \\ _ = @"type"; + \\ _ = type; + \\ if (@"f32".@"u8") { + \\ return @"i10"(.{ .@"u8" = true, .@"_" = false }, false); + \\ } else { + \\ return error.@"null"; + \\ } + \\} + \\ + \\test @"i10" { + \\ try @"i10"(.{}, true); + \\ _ = @"void": while (null) |@"u3"| { + \\ break :@"void" @"u3"; + \\ }; + \\ _ = @"void": { + \\ break :@"void"; + \\ }; + \\ for ("hi", 0..) |@"u3", @"i4"| { + \\ _ = @"u3"; + \\ _ = @"i4"; + \\ } + \\ if (false) {} else |@"bool"| { + \\ _ = @"bool"; + \\ } + \\} + \\ + , + \\const @"anyopaque" = struct { + \\ u8: @"type" = true, + \\ _: @"false" = @"true", + \\ const @"type" = bool; + \\ const @"false" = bool; + \\ const @"true" = false; + \\}; + \\ + \\const U = union(@"null") { + \\ type, + \\ const @"null" = enum { + \\ type, + \\ }; + \\}; + \\ + \\test { + \\ const E = enum { anyopaque }; + \\ _ = U{ .type = {} }; + \\ _ = U.type; + \\ _ = E.anyopaque; + \\} + \\ + \\fn @"i10"(@"void": @"anyopaque", @"type": @"anyopaque".type) error{null}!void { + \\ var @"f32" = @"void"; + \\ @"f32".u8 = false; + \\ _ = @"type"; + \\ _ = type; + \\ if (@"f32".u8) { + \\ return @"i10"(.{ .u8 = true, ._ = false }, false); + \\ } else { + \\ return error.null; + \\ } + \\} + \\ + \\test @"i10" { + \\ try @"i10"(.{}, true); + \\ _ = void: while (null) |@"u3"| { + \\ break :void @"u3"; + \\ }; + \\ _ = void: { + \\ break :void; + \\ }; + \\ for ("hi", 0..) |@"u3", @"i4"| { + \\ _ = @"u3"; + \\ _ = @"i4"; + \\ } + \\ if (false) {} else |@"bool"| { + \\ _ = @"bool"; + \\ } + \\} + \\ + ); +} + +// Never unescape names spelled like keywords. +test "zig fmt: canonicalize symbols (keywords)" { + try testCanonical( + \\const @"enum" = struct { + \\ @"error": @"struct" = true, + \\ const @"struct" = bool; + \\}; + \\ + \\fn @"usingnamespace"(@"union": @"enum") error{@"try"}!void { + \\ var @"struct" = @"union"; + \\ @"struct".@"error" = false; + \\ if (@"struct".@"error") { + \\ return @"usingnamespace"(.{ .@"error" = false }); + \\ } else { + \\ return error.@"try"; + \\ } + \\} + \\ + \\test @"usingnamespace" { + \\ try @"usingnamespace"(.{}); + \\ _ = @"return": { + \\ break :@"return" 4; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: no space before newline before multiline string" { + try testCanonical( + \\const S = struct { + \\ text: []const u8, + \\ comment: []const u8, + \\}; + \\ + \\test { + \\ const s1 = .{ + \\ .text = + \\ \\hello + \\ \\world + \\ , + \\ .comment = "test", + \\ }; + \\ _ = s1; + \\ const s2 = .{ + \\ .comment = "test", + \\ .text = + \\ \\hello + \\ \\world + \\ , + \\ }; + \\ _ = s2; + \\} + \\ + ); +} + +// Normalize \xNN and \u{NN} escapes and unicode inside @"" escapes. +test "zig fmt: canonicalize symbols (character escapes)" { + try testTransform( + \\const @"\x46\x6f\x6f\x64" = struct { + \\ @"\x62\x61\x72\x6E": @"\x43\x72\x61\x62" = false, + \\ @"\u{67}\u{6C}o\u{70}\xFF": @"Cra\x62" = false, + \\ @"\x65\x72\x72\x6F\x72": Crab = true, + \\ @"\x74\x72\x79": Crab = true, + \\ @"\u{74}\u{79}\u{70}\u{65}": @"any\u{6F}\u{70}\u{61}\u{71}\u{75}\u{65}", + \\ + \\ const @"\x43\x72\x61\x62" = bool; + \\ const @"\x61\x6E\x79\x6F\x70\x61que" = void; + \\}; + \\ + \\test "unicode" { + \\ const @"cąbbäge ⚡" = 2; + \\ _ = @"cąbbäge ⚡"; + \\ const @"\u{01f422} friend\u{f6}" = 4; + \\ _ = @"🐢 friendö"; + \\} + \\ + , + \\const Food = struct { + \\ barn: Crab = false, + \\ @"glop\xFF": Crab = false, + \\ @"error": Crab = true, + \\ @"try": Crab = true, + \\ type: @"anyopaque", + \\ + \\ const Crab = bool; + \\ const @"anyopaque" = void; + \\}; + \\ + \\test "unicode" { + \\ const @"cąbbäge ⚡" = 2; + \\ _ = @"cąbbäge ⚡"; + \\ const @"\u{01f422} friend\u{f6}" = 4; + \\ _ = @"🐢 friendö"; + \\} + \\ + ); +} + +test "zig fmt: canonicalize symbols (asm)" { + try testTransform( + \\test "asm" { + \\ const @"null" = usize; + \\ const @"try": usize = 808; + \\ const arg: usize = 2; + \\ _ = asm volatile ("syscall" + \\ : [@"void"] "={rax}" (-> @"null"), + \\ : [@"error"] "{rax}" (@"try"), + \\ [@"arg1"] "{rdi}" (arg), + \\ [arg2] "{rsi}" (arg), + \\ [arg3] "{rdx}" (arg), + \\ : "rcx", "r11" + \\ ); + \\ + \\ const @"false": usize = 10; + \\ const @"true" = "explode"; + \\ _ = asm volatile (@"true" + \\ : [one] "={rax}" (@"false"), + \\ : [two] "{rax}" (@"false"), + \\ ); + \\} + \\ + , + \\test "asm" { + \\ const @"null" = usize; + \\ const @"try": usize = 808; + \\ const arg: usize = 2; + \\ _ = asm volatile ("syscall" + \\ : [void] "={rax}" (-> @"null"), + \\ : [@"error"] "{rax}" (@"try"), + \\ [arg1] "{rdi}" (arg), + \\ [arg2] "{rsi}" (arg), + \\ [arg3] "{rdx}" (arg), + \\ : "rcx", "r11" + \\ ); + \\ + \\ const @"false": usize = 10; + \\ const @"true" = "explode"; + \\ _ = asm volatile (@"true" + \\ : [one] "={rax}" (false), + \\ : [two] "{rax}" (@"false"), + \\ ); + \\} + \\ + ); +} + +test "zig fmt: don't canonicalize _ in enums" { + try testTransform( + \\const A = enum { + \\ first, + \\ second, + \\ third, + \\ _, + \\}; + \\const B = enum { + \\ @"_", + \\ @"__", + \\ @"___", + \\ @"____", + \\}; + \\const C = struct { + \\ @"_": u8, + \\ @"__": u8, + \\ @"___": u8, + \\ @"____": u8, + \\}; + \\const D = union { + \\ @"_": u8, + \\ @"__": u8, + \\ @"___": u8, + \\ @"____": u8, + \\}; + \\ + , + \\const A = enum { + \\ first, + \\ second, + \\ third, + \\ _, + \\}; + \\const B = enum { + \\ @"_", + \\ __, + \\ ___, + \\ ____, + \\}; + \\const C = struct { + \\ _: u8, + \\ __: u8, + \\ ___: u8, + \\ ____: u8, + \\}; + \\const D = union { + \\ _: u8, + \\ __: u8, + \\ ___: u8, + \\ ____: u8, + \\}; + \\ + ); +} + +test "zig fmt: error for missing sentinel value in sentinel slice" { + try testError( + \\const foo = foo[0..:]; + , &[_]Error{ + .expected_expr, + }); +} + +test "zig fmt: error for invalid bit range" { + try testError( + \\var x: []align(0:0:0)u8 = bar; + , &[_]Error{ + .invalid_bit_range, + }); +} + +test "zig fmt: error for ptr mod on array child type" { + try testError( + \\var a: [10]align(10) u8 = e; + \\var b: [10]const u8 = f; + \\var c: [10]volatile u8 = g; + \\var d: [10]allowzero u8 = h; + , &[_]Error{ + .ptr_mod_on_array_child_type, + .ptr_mod_on_array_child_type, + .ptr_mod_on_array_child_type, + .ptr_mod_on_array_child_type, + }); +} + +test "recovery: top level" { + try testError( + \\test "" {inline} + \\test "" {inline} + , &[_]Error{ + .expected_inlinable, + .expected_inlinable, + }); +} + +test "recovery: block statements" { + try testError( + \\test "" { + \\ foo + +; + \\ inline; + \\} + , &[_]Error{ + .expected_expr, + .expected_semi_after_stmt, + .expected_statement, + .expected_inlinable, + }); +} + +test "recovery: missing comma" { + try testError( + \\test "" { + \\ switch (foo) { + \\ 2 => {} + \\ 3 => {} + \\ else => { + \\ foo & bar +; + \\ } + \\ } + \\} + , &[_]Error{ + .expected_comma_after_switch_prong, + .expected_comma_after_switch_prong, + .expected_expr, + }); +} + +test "recovery: non-associative operators" { + try testError( + \\const x = a == b == c; + \\const x = a == b != c; + , &[_]Error{ + .chained_comparison_operators, + .chained_comparison_operators, + }); +} + +test "recovery: extra qualifier" { + try testError( + \\const a: *const const u8; + \\test "" + , &[_]Error{ + .extra_const_qualifier, + .expected_block, + }); +} + +test "recovery: missing return type" { + try testError( + \\fn foo() { + \\ a & b; + \\} + \\test "" + , &[_]Error{ + .expected_return_type, + .expected_block, + }); +} + +test "recovery: continue after invalid decl" { + try testError( + \\fn foo { + \\ inline; + \\} + \\pub test "" { + \\ async a & b; + \\} + , &[_]Error{ + .expected_token, + .expected_pub_item, + .expected_param_list, + }); + try testError( + \\threadlocal test "" { + \\ @a & b; + \\} + , &[_]Error{ + .expected_var_decl, + .expected_param_list, + }); +} + +test "recovery: invalid extern/inline" { + try testError( + \\inline test "" { a & b; } + , &[_]Error{ + .expected_fn, + }); + try testError( + \\extern "" test "" { a & b; } + , &[_]Error{ + .expected_var_decl_or_fn, + }); +} + +test "recovery: missing semicolon" { + try testError( + \\test "" { + \\ comptime a & b + \\ c & d + \\ @foo + \\} + , &[_]Error{ + .expected_semi_after_stmt, + .expected_semi_after_stmt, + .expected_param_list, + .expected_semi_after_stmt, + }); +} + +test "recovery: invalid container members" { + try testError( + \\usingnamespace; + \\@foo()+ + \\@bar()@, + \\while (a == 2) { test "" {}} + \\test "" { + \\ a & b + \\} + , &[_]Error{ + .expected_expr, + .expected_comma_after_field, + .expected_type_expr, + .expected_semi_after_stmt, + }); +} + +// TODO after https://github.com/ziglang/zig/issues/35 is implemented, +// we should be able to recover from this *at any indentation level*, +// reporting a parse error and yet also parsing all the decls even +// inside structs. +test "recovery: extra '}' at top level" { + try testError( + \\}}} + \\test "" { + \\ a & b; + \\} + , &[_]Error{ + .expected_token, + }); +} + +test "recovery: mismatched bracket at top level" { + try testError( + \\const S = struct { + \\ arr: 128]?G + \\}; + , &[_]Error{ + .expected_comma_after_field, + }); +} + +test "recovery: invalid global error set access" { + try testError( + \\test "" { + \\ error & foo; + \\} + , &[_]Error{ + .expected_token, + .expected_token, + }); +} + +test "recovery: invalid asterisk after pointer dereference" { + try testError( + \\test "" { + \\ var sequence = "repeat".*** 10; + \\} + , &[_]Error{ + .asterisk_after_ptr_deref, + .mismatched_binary_op_whitespace, + }); + try testError( + \\test "" { + \\ var sequence = "repeat".** 10&a; + \\} + , &[_]Error{ + .asterisk_after_ptr_deref, + .mismatched_binary_op_whitespace, + }); +} + +test "recovery: missing semicolon after if, for, while stmt" { + try testError( + \\test "" { + \\ if (foo) bar + \\ for (foo) |a| bar + \\ while (foo) bar + \\ a & b; + \\} + , &[_]Error{ + .expected_semi_or_else, + .expected_semi_or_else, + .expected_semi_or_else, + }); +} + +test "recovery: invalid comptime" { + try testError( + \\comptime + , &[_]Error{ + .expected_type_expr, + }); +} + +test "recovery: missing block after suspend" { + try testError( + \\fn foo() void { + \\ suspend; + \\ nosuspend; + \\} + , &[_]Error{ + .expected_block_or_expr, + .expected_block_or_expr, + }); +} + +test "recovery: missing block after for/while loops" { + try testError( + \\test "" { while (foo) } + , &[_]Error{ + .expected_block_or_assignment, + }); + try testError( + \\test "" { for (foo) |bar| } + , &[_]Error{ + .expected_block_or_assignment, + }); +} + +test "recovery: missing for payload" { + try testError( + \\comptime { + \\ const a = for(a) {}; + \\ const a: for(a) blk: {} = {}; + \\ for(a) {} + \\} + , &[_]Error{ + .expected_loop_payload, + .expected_loop_payload, + .expected_loop_payload, + }); +} + +test "recovery: missing comma in params" { + try testError( + \\fn foo(comptime bool what what) void { } + \\fn bar(a: i32, b: i32 c) void { } + \\ + , &[_]Error{ + .expected_comma_after_param, + .expected_comma_after_param, + .expected_comma_after_param, + }); +} + +test "recovery: missing while rbrace" { + try testError( + \\fn a() b { + \\ while (d) { + \\} + , &[_]Error{ + .expected_statement, + }); +} + +test "recovery: nonfinal varargs" { + try testError( + \\extern fn f(a: u32, ..., b: u32) void; + \\extern fn g(a: u32, ..., b: anytype) void; + \\extern fn h(a: u32, ..., ...) void; + , &[_]Error{ + .varargs_nonfinal, + .varargs_nonfinal, + .varargs_nonfinal, + }); +} + +test "recovery: eof in c pointer" { + try testError( + \\const Ptr = [*c + , &[_]Error{ + .expected_token, + }); +} + +test "matching whitespace on minus op" { + try testError( + \\ _ = 2 -1, + \\ _ = 2- 1, + \\ _ = 2- + \\ 2, + \\ _ = 2 + \\ -2, + , &[_]Error{ + .mismatched_binary_op_whitespace, + .mismatched_binary_op_whitespace, + .mismatched_binary_op_whitespace, + .mismatched_binary_op_whitespace, + }); + + try testError( + \\ _ = - 1, + \\ _ = -1, + \\ _ = 2 - -1, + \\ _ = 2 - 1, + \\ _ = 2-1, + \\ _ = 2 - + \\1, + \\ _ = 2 + \\ - 1, + , &[_]Error{}); +} + +test "ampersand" { + try testError( + \\ _ = bar && foo, + \\ _ = bar&&foo, + \\ _ = bar& & foo, + \\ _ = bar& &foo, + , &.{ + .invalid_ampersand_ampersand, + .invalid_ampersand_ampersand, + .mismatched_binary_op_whitespace, + .mismatched_binary_op_whitespace, + }); + + try testError( + \\ _ = bar & &foo, + \\ _ = bar & &&foo, + \\ _ = &&foo, + , &.{}); +} + +const std = @import("std"); +const mem = std.mem; +const print = std.debug.print; +const io = std.io; +const maxInt = std.math.maxInt; + +var fixed_buffer_mem: [100 * 1024]u8 = undefined; + +fn testParse(source: [:0]const u8, allocator: mem.Allocator, anything_changed: *bool) ![]u8 { + const stderr = io.getStdErr().writer(); + + var tree = try std.zig.Ast.parse(allocator, source, .zig); + defer tree.deinit(allocator); + + for (tree.errors) |parse_error| { + const loc = tree.tokenLocation(0, parse_error.token); + try stderr.print("(memory buffer):{d}:{d}: error: ", .{ loc.line + 1, loc.column + 1 }); + try tree.renderError(parse_error, stderr); + try stderr.print("\n{s}\n", .{source[loc.line_start..loc.line_end]}); + { + var i: usize = 0; + while (i < loc.column) : (i += 1) { + try stderr.writeAll(" "); + } + try stderr.writeAll("^"); + } + try stderr.writeAll("\n"); + } + if (tree.errors.len != 0) { + return error.ParseError; + } + + const formatted = try tree.render(allocator); + anything_changed.* = !mem.eql(u8, formatted, source); + return formatted; +} +fn testTransformImpl(allocator: mem.Allocator, fba: *std.heap.FixedBufferAllocator, source: [:0]const u8, expected_source: []const u8) !void { + // reset the fixed buffer allocator each run so that it can be re-used for each + // iteration of the failing index + fba.reset(); + var anything_changed: bool = undefined; + const result_source = try testParse(source, allocator, &anything_changed); + try std.testing.expectEqualStrings(expected_source, result_source); + const changes_expected = source.ptr != expected_source.ptr; + if (anything_changed != changes_expected) { + print("std.zig.render returned {} instead of {}\n", .{ anything_changed, changes_expected }); + return error.TestFailed; + } + try std.testing.expect(anything_changed == changes_expected); + allocator.free(result_source); +} +fn testTransform(source: [:0]const u8, expected_source: []const u8) !void { + var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + return std.testing.checkAllAllocationFailures(fixed_allocator.allocator(), testTransformImpl, .{ &fixed_allocator, source, expected_source }); +} +fn testCanonical(source: [:0]const u8) !void { + return testTransform(source, source); +} + +const Error = std.zig.Ast.Error.Tag; + +fn testError(source: [:0]const u8, expected_errors: []const Error) !void { + var tree = try std.zig.Ast.parse(std.testing.allocator, source, .zig); + defer tree.deinit(std.testing.allocator); + + std.testing.expectEqual(expected_errors.len, tree.errors.len) catch |err| { + std.debug.print("errors found: {any}\n", .{tree.errors}); + return err; + }; + for (expected_errors, 0..) |expected, i| { + try std.testing.expectEqual(expected, tree.errors[i].tag); + } +} diff --git a/src/stage2/render.zig b/src/stage2/render.zig new file mode 100644 index 0000000000..897bd1780c --- /dev/null +++ b/src/stage2/render.zig @@ -0,0 +1,3304 @@ +const std = @import("std"); +const assert = std.debug.assert; +const mem = std.mem; +const Allocator = std.mem.Allocator; +const meta = std.meta; +const Ast = std.zig.Ast; +const Token = std.zig.Token; +const primitives = std.zig.primitives; + +const indent_delta = 4; +const asm_indent_delta = 2; + +pub const Error = Ast.RenderError; + +const Ais = AutoIndentingStream(std.ArrayList(u8).Writer); + +pub fn renderTree(buffer: *std.ArrayList(u8), tree: Ast) Error!void { + assert(tree.errors.len == 0); // Cannot render an invalid tree. + var auto_indenting_stream = Ais{ + .indent_delta = indent_delta, + .underlying_writer = buffer.writer(), + }; + const ais = &auto_indenting_stream; + + // Render all the line comments at the beginning of the file. + const comment_end_loc = tree.tokens.items(.start)[0]; + _ = try renderComments(ais, tree, 0, comment_end_loc); + + if (tree.tokens.items(.tag)[0] == .container_doc_comment) { + try renderContainerDocComments(ais, tree, 0); + } + + try renderMembers(buffer.allocator, ais, tree, tree.rootDecls()); + + if (ais.disabled_offset) |disabled_offset| { + try writeFixingWhitespace(ais.underlying_writer, tree.source[disabled_offset..]); + } +} + +/// Render all members in the given slice, keeping empty lines where appropriate +fn renderMembers(gpa: Allocator, ais: *Ais, tree: Ast, members: []const Ast.Node.Index) Error!void { + if (members.len == 0) return; + const container: Container = for (members) |member| { + if (tree.fullContainerField(member)) |field| if (!field.ast.tuple_like) break .other; + } else .tuple; + try renderMember(gpa, ais, tree, container, members[0], .newline); + for (members[1..]) |member| { + try renderExtraNewline(ais, tree, member); + try renderMember(gpa, ais, tree, container, member, .newline); + } +} + +const Container = enum { + @"enum", + tuple, + other, +}; + +fn renderMember( + gpa: Allocator, + ais: *Ais, + tree: Ast, + container: Container, + decl: Ast.Node.Index, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + const main_tokens = tree.nodes.items(.main_token); + const datas = tree.nodes.items(.data); + try renderDocComments(ais, tree, tree.firstToken(decl)); + switch (tree.nodes.items(.tag)[decl]) { + .fn_decl => { + // Some examples: + // pub extern "foo" fn ... + // export fn ... + const fn_proto = datas[decl].lhs; + const fn_token = main_tokens[fn_proto]; + // Go back to the first token we should render here. + var i = fn_token; + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, + .keyword_export, + .keyword_pub, + .string_literal, + .keyword_inline, + .keyword_noinline, + => continue, + + else => { + i += 1; + break; + }, + } + } + while (i < fn_token) : (i += 1) { + try renderToken(ais, tree, i, .space); + } + switch (tree.nodes.items(.tag)[fn_proto]) { + .fn_proto_one, .fn_proto => { + const callconv_expr = if (tree.nodes.items(.tag)[fn_proto] == .fn_proto_one) + tree.extraData(datas[fn_proto].lhs, Ast.Node.FnProtoOne).callconv_expr + else + tree.extraData(datas[fn_proto].lhs, Ast.Node.FnProto).callconv_expr; + if (callconv_expr != 0 and tree.nodes.items(.tag)[callconv_expr] == .enum_literal) { + if (mem.eql(u8, "Inline", tree.tokenSlice(main_tokens[callconv_expr]))) { + try ais.writer().writeAll("inline "); + } + } + }, + .fn_proto_simple, .fn_proto_multi => {}, + else => unreachable, + } + assert(datas[decl].rhs != 0); + try renderExpression(gpa, ais, tree, fn_proto, .space); + return renderExpression(gpa, ais, tree, datas[decl].rhs, space); + }, + .fn_proto_simple, + .fn_proto_multi, + .fn_proto_one, + .fn_proto, + => { + // Extern function prototypes are parsed as these tags. + // Go back to the first token we should render here. + const fn_token = main_tokens[decl]; + var i = fn_token; + while (i > 0) { + i -= 1; + switch (token_tags[i]) { + .keyword_extern, + .keyword_export, + .keyword_pub, + .string_literal, + .keyword_inline, + .keyword_noinline, + => continue, + + else => { + i += 1; + break; + }, + } + } + while (i < fn_token) : (i += 1) { + try renderToken(ais, tree, i, .space); + } + try renderExpression(gpa, ais, tree, decl, .none); + return renderToken(ais, tree, tree.lastToken(decl) + 1, space); // semicolon + }, + + .@"usingnamespace" => { + const main_token = main_tokens[decl]; + const expr = datas[decl].lhs; + if (main_token > 0 and token_tags[main_token - 1] == .keyword_pub) { + try renderToken(ais, tree, main_token - 1, .space); // pub + } + try renderToken(ais, tree, main_token, .space); // usingnamespace + try renderExpression(gpa, ais, tree, expr, .none); + return renderToken(ais, tree, tree.lastToken(expr) + 1, space); // ; + }, + + .global_var_decl, + .local_var_decl, + .simple_var_decl, + .aligned_var_decl, + => return renderVarDecl(gpa, ais, tree, tree.fullVarDecl(decl).?, false, .semicolon), + + .test_decl => { + const test_token = main_tokens[decl]; + try renderToken(ais, tree, test_token, .space); + const test_name_tag = token_tags[test_token + 1]; + switch (test_name_tag) { + .string_literal => try renderToken(ais, tree, test_token + 1, .space), + .identifier => try renderIdentifier(ais, tree, test_token + 1, .space, .preserve_when_shadowing), + else => {}, + } + try renderExpression(gpa, ais, tree, datas[decl].rhs, space); + }, + + .container_field_init, + .container_field_align, + .container_field, + => return renderContainerField(gpa, ais, tree, container, tree.fullContainerField(decl).?, space), + + .@"comptime" => return renderExpression(gpa, ais, tree, decl, space), + + .root => unreachable, + else => unreachable, + } +} + +/// Render all expressions in the slice, keeping empty lines where appropriate +fn renderExpressions(gpa: Allocator, ais: *Ais, tree: Ast, expressions: []const Ast.Node.Index, space: Space) Error!void { + if (expressions.len == 0) return; + try renderExpression(gpa, ais, tree, expressions[0], space); + for (expressions[1..]) |expression| { + try renderExtraNewline(ais, tree, expression); + try renderExpression(gpa, ais, tree, expression, space); + } +} + +fn renderExpression(gpa: Allocator, ais: *Ais, tree: Ast, node: Ast.Node.Index, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + const main_tokens = tree.nodes.items(.main_token); + const node_tags = tree.nodes.items(.tag); + const datas = tree.nodes.items(.data); + switch (node_tags[node]) { + .identifier => { + const token_index = main_tokens[node]; + return renderIdentifier(ais, tree, token_index, space, .preserve_when_shadowing); + }, + + .number_literal, + .char_literal, + .unreachable_literal, + .anyframe_literal, + .string_literal, + => return renderToken(ais, tree, main_tokens[node], space), + + .multiline_string_literal => { + var locked_indents = ais.lockOneShotIndent(); + try ais.maybeInsertNewline(); + + var i = datas[node].lhs; + while (i <= datas[node].rhs) : (i += 1) try renderToken(ais, tree, i, .newline); + + while (locked_indents > 0) : (locked_indents -= 1) ais.popIndent(); + + switch (space) { + .none, .space, .newline, .skip => {}, + .semicolon => if (token_tags[i] == .semicolon) try renderToken(ais, tree, i, .newline), + .comma => if (token_tags[i] == .comma) try renderToken(ais, tree, i, .newline), + .comma_space => if (token_tags[i] == .comma) try renderToken(ais, tree, i, .space), + } + }, + + .error_value => { + try renderToken(ais, tree, main_tokens[node], .none); + try renderToken(ais, tree, main_tokens[node] + 1, .none); + return renderIdentifier(ais, tree, main_tokens[node] + 2, space, .eagerly_unquote); + }, + + .block_two, + .block_two_semicolon, + => { + const statements = [2]Ast.Node.Index{ datas[node].lhs, datas[node].rhs }; + if (datas[node].lhs == 0) { + return renderBlock(gpa, ais, tree, node, statements[0..0], space); + } else if (datas[node].rhs == 0) { + return renderBlock(gpa, ais, tree, node, statements[0..1], space); + } else { + return renderBlock(gpa, ais, tree, node, statements[0..2], space); + } + }, + .block, + .block_semicolon, + => { + const statements = tree.extra_data[datas[node].lhs..datas[node].rhs]; + return renderBlock(gpa, ais, tree, node, statements, space); + }, + + .@"errdefer" => { + const defer_token = main_tokens[node]; + const payload_token = datas[node].lhs; + const expr = datas[node].rhs; + + try renderToken(ais, tree, defer_token, .space); + if (payload_token != 0) { + try renderToken(ais, tree, payload_token - 1, .none); // | + try renderIdentifier(ais, tree, payload_token, .none, .preserve_when_shadowing); // identifier + try renderToken(ais, tree, payload_token + 1, .space); // | + } + return renderExpression(gpa, ais, tree, expr, space); + }, + + .@"defer" => { + const defer_token = main_tokens[node]; + const expr = datas[node].rhs; + try renderToken(ais, tree, defer_token, .space); + return renderExpression(gpa, ais, tree, expr, space); + }, + .@"comptime", .@"nosuspend" => { + const comptime_token = main_tokens[node]; + const block = datas[node].lhs; + try renderToken(ais, tree, comptime_token, .space); + return renderExpression(gpa, ais, tree, block, space); + }, + + .@"suspend" => { + const suspend_token = main_tokens[node]; + const body = datas[node].lhs; + try renderToken(ais, tree, suspend_token, .space); + return renderExpression(gpa, ais, tree, body, space); + }, + + .@"catch" => { + const main_token = main_tokens[node]; + const fallback_first = tree.firstToken(datas[node].rhs); + + const same_line = tree.tokensOnSameLine(main_token, fallback_first); + const after_op_space = if (same_line) Space.space else Space.newline; + + try renderExpression(gpa, ais, tree, datas[node].lhs, .space); // target + + if (token_tags[fallback_first - 1] == .pipe) { + try renderToken(ais, tree, main_token, .space); // catch keyword + try renderToken(ais, tree, main_token + 1, .none); // pipe + try renderIdentifier(ais, tree, main_token + 2, .none, .preserve_when_shadowing); // payload identifier + try renderToken(ais, tree, main_token + 3, after_op_space); // pipe + } else { + assert(token_tags[fallback_first - 1] == .keyword_catch); + try renderToken(ais, tree, main_token, after_op_space); // catch keyword + } + + ais.pushIndentOneShot(); + try renderExpression(gpa, ais, tree, datas[node].rhs, space); // fallback + }, + + .field_access => { + const main_token = main_tokens[node]; + const field_access = datas[node]; + + try renderExpression(gpa, ais, tree, field_access.lhs, .none); + + // Allow a line break between the lhs and the dot if the lhs and rhs + // are on different lines. + const lhs_last_token = tree.lastToken(field_access.lhs); + const same_line = tree.tokensOnSameLine(lhs_last_token, main_token + 1); + if (!same_line) { + if (!hasComment(tree, lhs_last_token, main_token)) try ais.insertNewline(); + ais.pushIndentOneShot(); + } + + try renderToken(ais, tree, main_token, .none); // . + + // This check ensures that zag() is indented in the following example: + // const x = foo + // .bar() + // . // comment + // zag(); + if (!same_line and hasComment(tree, main_token, main_token + 1)) { + ais.pushIndentOneShot(); + } + + return renderIdentifier(ais, tree, field_access.rhs, space, .eagerly_unquote); // field + }, + + .error_union, + .switch_range, + => { + const infix = datas[node]; + try renderExpression(gpa, ais, tree, infix.lhs, .none); + try renderToken(ais, tree, main_tokens[node], .none); + return renderExpression(gpa, ais, tree, infix.rhs, space); + }, + .for_range => { + const infix = datas[node]; + try renderExpression(gpa, ais, tree, infix.lhs, .none); + if (infix.rhs != 0) { + try renderToken(ais, tree, main_tokens[node], .none); + return renderExpression(gpa, ais, tree, infix.rhs, space); + } else { + return renderToken(ais, tree, main_tokens[node], space); + } + }, + + .add, + .add_wrap, + .add_sat, + .array_cat, + .array_mult, + .assign, + .assign_bit_and, + .assign_bit_or, + .assign_shl, + .assign_shl_sat, + .assign_shr, + .assign_bit_xor, + .assign_div, + .assign_sub, + .assign_sub_wrap, + .assign_sub_sat, + .assign_mod, + .assign_add, + .assign_add_wrap, + .assign_add_sat, + .assign_mul, + .assign_mul_wrap, + .assign_mul_sat, + .bang_equal, + .bit_and, + .bit_or, + .shl, + .shl_sat, + .shr, + .bit_xor, + .bool_and, + .bool_or, + .div, + .equal_equal, + .greater_or_equal, + .greater_than, + .less_or_equal, + .less_than, + .merge_error_sets, + .mod, + .mul, + .mul_wrap, + .mul_sat, + .sub, + .sub_wrap, + .sub_sat, + .@"orelse", + => { + const infix = datas[node]; + try renderExpression(gpa, ais, tree, infix.lhs, .space); + const op_token = main_tokens[node]; + if (tree.tokensOnSameLine(op_token, op_token + 1)) { + try renderToken(ais, tree, op_token, .space); + } else { + ais.pushIndent(); + try renderToken(ais, tree, op_token, .newline); + ais.popIndent(); + } + ais.pushIndentOneShot(); + return renderExpression(gpa, ais, tree, infix.rhs, space); + }, + + .assign_destructure => { + const lhs_count = tree.extra_data[datas[node].lhs]; + assert(lhs_count > 1); + const lhs_exprs = tree.extra_data[datas[node].lhs + 1 ..][0..lhs_count]; + const rhs = datas[node].rhs; + + const maybe_comptime_token = tree.firstToken(node) - 1; + if (token_tags[maybe_comptime_token] == .keyword_comptime) { + try renderToken(ais, tree, maybe_comptime_token, .space); + } + + for (lhs_exprs, 0..) |lhs_node, i| { + const lhs_space: Space = if (i == lhs_exprs.len - 1) .space else .comma_space; + switch (node_tags[lhs_node]) { + .global_var_decl, + .local_var_decl, + .simple_var_decl, + .aligned_var_decl, + => { + try renderVarDecl(gpa, ais, tree, tree.fullVarDecl(lhs_node).?, true, lhs_space); + }, + else => try renderExpression(gpa, ais, tree, lhs_node, lhs_space), + } + } + const equal_token = main_tokens[node]; + if (tree.tokensOnSameLine(equal_token, equal_token + 1)) { + try renderToken(ais, tree, equal_token, .space); + } else { + ais.pushIndent(); + try renderToken(ais, tree, equal_token, .newline); + ais.popIndent(); + } + ais.pushIndentOneShot(); + return renderExpression(gpa, ais, tree, rhs, space); + }, + + .bit_not, + .bool_not, + .negation, + .negation_wrap, + .optional_type, + .address_of, + => { + try renderToken(ais, tree, main_tokens[node], .none); + return renderExpression(gpa, ais, tree, datas[node].lhs, space); + }, + + .@"try", + .@"resume", + .@"await", + => { + try renderToken(ais, tree, main_tokens[node], .space); + return renderExpression(gpa, ais, tree, datas[node].lhs, space); + }, + + .array_type, + .array_type_sentinel, + => return renderArrayType(gpa, ais, tree, tree.fullArrayType(node).?, space), + + .ptr_type_aligned, + .ptr_type_sentinel, + .ptr_type, + .ptr_type_bit_range, + => return renderPtrType(gpa, ais, tree, tree.fullPtrType(node).?, space), + + .array_init_one, + .array_init_one_comma, + .array_init_dot_two, + .array_init_dot_two_comma, + .array_init_dot, + .array_init_dot_comma, + .array_init, + .array_init_comma, + => { + var elements: [2]Ast.Node.Index = undefined; + return renderArrayInit(gpa, ais, tree, tree.fullArrayInit(&elements, node).?, space); + }, + + .struct_init_one, + .struct_init_one_comma, + .struct_init_dot_two, + .struct_init_dot_two_comma, + .struct_init_dot, + .struct_init_dot_comma, + .struct_init, + .struct_init_comma, + => { + var buf: [2]Ast.Node.Index = undefined; + return renderStructInit(gpa, ais, tree, node, tree.fullStructInit(&buf, node).?, space); + }, + + .call_one, + .call_one_comma, + .async_call_one, + .async_call_one_comma, + .call, + .call_comma, + .async_call, + .async_call_comma, + => { + var buf: [1]Ast.Node.Index = undefined; + return renderCall(gpa, ais, tree, tree.fullCall(&buf, node).?, space); + }, + + .array_access => { + const suffix = datas[node]; + const lbracket = tree.firstToken(suffix.rhs) - 1; + const rbracket = tree.lastToken(suffix.rhs) + 1; + const one_line = tree.tokensOnSameLine(lbracket, rbracket); + const inner_space = if (one_line) Space.none else Space.newline; + try renderExpression(gpa, ais, tree, suffix.lhs, .none); + ais.pushIndentNextLine(); + try renderToken(ais, tree, lbracket, inner_space); // [ + try renderExpression(gpa, ais, tree, suffix.rhs, inner_space); + ais.popIndent(); + return renderToken(ais, tree, rbracket, space); // ] + }, + + .slice_open, .slice, .slice_sentinel => return renderSlice(gpa, ais, tree, node, tree.fullSlice(node).?, space), + + .deref => { + try renderExpression(gpa, ais, tree, datas[node].lhs, .none); + return renderToken(ais, tree, main_tokens[node], space); + }, + + .unwrap_optional => { + try renderExpression(gpa, ais, tree, datas[node].lhs, .none); + try renderToken(ais, tree, main_tokens[node], .none); + return renderToken(ais, tree, datas[node].rhs, space); + }, + + .@"break" => { + const main_token = main_tokens[node]; + const label_token = datas[node].lhs; + const target = datas[node].rhs; + if (label_token == 0 and target == 0) { + try renderToken(ais, tree, main_token, space); // break keyword + } else if (label_token == 0 and target != 0) { + try renderToken(ais, tree, main_token, .space); // break keyword + try renderExpression(gpa, ais, tree, target, space); + } else if (label_token != 0 and target == 0) { + try renderToken(ais, tree, main_token, .space); // break keyword + try renderToken(ais, tree, label_token - 1, .none); // colon + try renderIdentifier(ais, tree, label_token, space, .eagerly_unquote); // identifier + } else if (label_token != 0 and target != 0) { + try renderToken(ais, tree, main_token, .space); // break keyword + try renderToken(ais, tree, label_token - 1, .none); // colon + try renderIdentifier(ais, tree, label_token, .space, .eagerly_unquote); // identifier + try renderExpression(gpa, ais, tree, target, space); + } + }, + + .@"continue" => { + const main_token = main_tokens[node]; + const label = datas[node].lhs; + if (label != 0) { + try renderToken(ais, tree, main_token, .space); // continue + try renderToken(ais, tree, label - 1, .none); // : + return renderIdentifier(ais, tree, label, space, .eagerly_unquote); // label + } else { + return renderToken(ais, tree, main_token, space); // continue + } + }, + + .@"return" => { + if (datas[node].lhs != 0) { + try renderToken(ais, tree, main_tokens[node], .space); + try renderExpression(gpa, ais, tree, datas[node].lhs, space); + } else { + try renderToken(ais, tree, main_tokens[node], space); + } + }, + + .grouped_expression => { + try renderToken(ais, tree, main_tokens[node], .none); // lparen + ais.pushIndentOneShot(); + try renderExpression(gpa, ais, tree, datas[node].lhs, .none); + return renderToken(ais, tree, datas[node].rhs, space); // rparen + }, + + .container_decl, + .container_decl_trailing, + .container_decl_arg, + .container_decl_arg_trailing, + .container_decl_two, + .container_decl_two_trailing, + .tagged_union, + .tagged_union_trailing, + .tagged_union_enum_tag, + .tagged_union_enum_tag_trailing, + .tagged_union_two, + .tagged_union_two_trailing, + => { + var buf: [2]Ast.Node.Index = undefined; + return renderContainerDecl(gpa, ais, tree, node, tree.fullContainerDecl(&buf, node).?, space); + }, + + .error_set_decl => { + const error_token = main_tokens[node]; + const lbrace = error_token + 1; + const rbrace = datas[node].rhs; + + try renderToken(ais, tree, error_token, .none); + + if (lbrace + 1 == rbrace) { + // There is nothing between the braces so render condensed: `error{}` + try renderToken(ais, tree, lbrace, .none); + return renderToken(ais, tree, rbrace, space); + } else if (lbrace + 2 == rbrace and token_tags[lbrace + 1] == .identifier) { + // There is exactly one member and no trailing comma or + // comments, so render without surrounding spaces: `error{Foo}` + try renderToken(ais, tree, lbrace, .none); + try renderIdentifier(ais, tree, lbrace + 1, .none, .eagerly_unquote); // identifier + return renderToken(ais, tree, rbrace, space); + } else if (token_tags[rbrace - 1] == .comma) { + // There is a trailing comma so render each member on a new line. + ais.pushIndentNextLine(); + try renderToken(ais, tree, lbrace, .newline); + var i = lbrace + 1; + while (i < rbrace) : (i += 1) { + if (i > lbrace + 1) try renderExtraNewlineToken(ais, tree, i); + switch (token_tags[i]) { + .doc_comment => try renderToken(ais, tree, i, .newline), + .identifier => try renderIdentifier(ais, tree, i, .comma, .eagerly_unquote), + .comma => {}, + else => unreachable, + } + } + ais.popIndent(); + return renderToken(ais, tree, rbrace, space); + } else { + // There is no trailing comma so render everything on one line. + try renderToken(ais, tree, lbrace, .space); + var i = lbrace + 1; + while (i < rbrace) : (i += 1) { + switch (token_tags[i]) { + .doc_comment => unreachable, // TODO + .identifier => try renderIdentifier(ais, tree, i, .comma_space, .eagerly_unquote), + .comma => {}, + else => unreachable, + } + } + return renderToken(ais, tree, rbrace, space); + } + }, + + .builtin_call_two, .builtin_call_two_comma => { + if (datas[node].lhs == 0) { + return renderBuiltinCall(gpa, ais, tree, main_tokens[node], &.{}, space); + } else if (datas[node].rhs == 0) { + return renderBuiltinCall(gpa, ais, tree, main_tokens[node], &.{datas[node].lhs}, space); + } else { + return renderBuiltinCall(gpa, ais, tree, main_tokens[node], &.{ datas[node].lhs, datas[node].rhs }, space); + } + }, + .builtin_call, .builtin_call_comma => { + const params = tree.extra_data[datas[node].lhs..datas[node].rhs]; + return renderBuiltinCall(gpa, ais, tree, main_tokens[node], params, space); + }, + + .fn_proto_simple, + .fn_proto_multi, + .fn_proto_one, + .fn_proto, + => { + var buf: [1]Ast.Node.Index = undefined; + return renderFnProto(gpa, ais, tree, tree.fullFnProto(&buf, node).?, space); + }, + + .anyframe_type => { + const main_token = main_tokens[node]; + if (datas[node].rhs != 0) { + try renderToken(ais, tree, main_token, .none); // anyframe + try renderToken(ais, tree, main_token + 1, .none); // -> + return renderExpression(gpa, ais, tree, datas[node].rhs, space); + } else { + return renderToken(ais, tree, main_token, space); // anyframe + } + }, + + .@"switch", + .switch_comma, + => { + const switch_token = main_tokens[node]; + const condition = datas[node].lhs; + const extra = tree.extraData(datas[node].rhs, Ast.Node.SubRange); + const cases = tree.extra_data[extra.start..extra.end]; + const rparen = tree.lastToken(condition) + 1; + + try renderToken(ais, tree, switch_token, .space); // switch keyword + try renderToken(ais, tree, switch_token + 1, .none); // lparen + try renderExpression(gpa, ais, tree, condition, .none); // condition expression + try renderToken(ais, tree, rparen, .space); // rparen + + ais.pushIndentNextLine(); + if (cases.len == 0) { + try renderToken(ais, tree, rparen + 1, .none); // lbrace + } else { + try renderToken(ais, tree, rparen + 1, .newline); // lbrace + try renderExpressions(gpa, ais, tree, cases, .comma); + } + ais.popIndent(); + return renderToken(ais, tree, tree.lastToken(node), space); // rbrace + }, + + .switch_case_one, + .switch_case_inline_one, + .switch_case, + .switch_case_inline, + => return renderSwitchCase(gpa, ais, tree, tree.fullSwitchCase(node).?, space), + + .while_simple, + .while_cont, + .@"while", + => return renderWhile(gpa, ais, tree, tree.fullWhile(node).?, space), + + .for_simple, + .@"for", + => return renderFor(gpa, ais, tree, tree.fullFor(node).?, space), + + .if_simple, + .@"if", + => return renderIf(gpa, ais, tree, tree.fullIf(node).?, space), + + .asm_simple, + .@"asm", + => return renderAsm(gpa, ais, tree, tree.fullAsm(node).?, space), + + .enum_literal => { + try renderToken(ais, tree, main_tokens[node] - 1, .none); // . + return renderIdentifier(ais, tree, main_tokens[node], space, .eagerly_unquote); // name + }, + + .fn_decl => unreachable, + .container_field => unreachable, + .container_field_init => unreachable, + .container_field_align => unreachable, + .root => unreachable, + .global_var_decl => unreachable, + .local_var_decl => unreachable, + .simple_var_decl => unreachable, + .aligned_var_decl => unreachable, + .@"usingnamespace" => unreachable, + .test_decl => unreachable, + .asm_output => unreachable, + .asm_input => unreachable, + } +} + +fn renderArrayType( + gpa: Allocator, + ais: *Ais, + tree: Ast, + array_type: Ast.full.ArrayType, + space: Space, +) Error!void { + const rbracket = tree.firstToken(array_type.ast.elem_type) - 1; + const one_line = tree.tokensOnSameLine(array_type.ast.lbracket, rbracket); + const inner_space = if (one_line) Space.none else Space.newline; + ais.pushIndentNextLine(); + try renderToken(ais, tree, array_type.ast.lbracket, inner_space); // lbracket + try renderExpression(gpa, ais, tree, array_type.ast.elem_count, inner_space); + if (array_type.ast.sentinel != 0) { + try renderToken(ais, tree, tree.firstToken(array_type.ast.sentinel) - 1, inner_space); // colon + try renderExpression(gpa, ais, tree, array_type.ast.sentinel, inner_space); + } + ais.popIndent(); + try renderToken(ais, tree, rbracket, .none); // rbracket + return renderExpression(gpa, ais, tree, array_type.ast.elem_type, space); +} + +fn renderPtrType( + gpa: Allocator, + ais: *Ais, + tree: Ast, + ptr_type: Ast.full.PtrType, + space: Space, +) Error!void { + switch (ptr_type.size) { + .One => { + // Since ** tokens exist and the same token is shared by two + // nested pointer types, we check to see if we are the parent + // in such a relationship. If so, skip rendering anything for + // this pointer type and rely on the child to render our asterisk + // as well when it renders the ** token. + if (tree.tokens.items(.tag)[ptr_type.ast.main_token] == .asterisk_asterisk and + ptr_type.ast.main_token == tree.nodes.items(.main_token)[ptr_type.ast.child_type]) + { + return renderExpression(gpa, ais, tree, ptr_type.ast.child_type, space); + } + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // asterisk + }, + .Many => { + if (ptr_type.ast.sentinel == 0) { + try renderToken(ais, tree, ptr_type.ast.main_token - 1, .none); // lbracket + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // asterisk + try renderToken(ais, tree, ptr_type.ast.main_token + 1, .none); // rbracket + } else { + try renderToken(ais, tree, ptr_type.ast.main_token - 1, .none); // lbracket + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // asterisk + try renderToken(ais, tree, ptr_type.ast.main_token + 1, .none); // colon + try renderExpression(gpa, ais, tree, ptr_type.ast.sentinel, .none); + try renderToken(ais, tree, tree.lastToken(ptr_type.ast.sentinel) + 1, .none); // rbracket + } + }, + .C => { + try renderToken(ais, tree, ptr_type.ast.main_token - 1, .none); // lbracket + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // asterisk + try renderToken(ais, tree, ptr_type.ast.main_token + 1, .none); // c + try renderToken(ais, tree, ptr_type.ast.main_token + 2, .none); // rbracket + }, + .Slice => { + if (ptr_type.ast.sentinel == 0) { + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // lbracket + try renderToken(ais, tree, ptr_type.ast.main_token + 1, .none); // rbracket + } else { + try renderToken(ais, tree, ptr_type.ast.main_token, .none); // lbracket + try renderToken(ais, tree, ptr_type.ast.main_token + 1, .none); // colon + try renderExpression(gpa, ais, tree, ptr_type.ast.sentinel, .none); + try renderToken(ais, tree, tree.lastToken(ptr_type.ast.sentinel) + 1, .none); // rbracket + } + }, + } + + if (ptr_type.allowzero_token) |allowzero_token| { + try renderToken(ais, tree, allowzero_token, .space); + } + + if (ptr_type.ast.align_node != 0) { + const align_first = tree.firstToken(ptr_type.ast.align_node); + try renderToken(ais, tree, align_first - 2, .none); // align + try renderToken(ais, tree, align_first - 1, .none); // lparen + try renderExpression(gpa, ais, tree, ptr_type.ast.align_node, .none); + if (ptr_type.ast.bit_range_start != 0) { + assert(ptr_type.ast.bit_range_end != 0); + try renderToken(ais, tree, tree.firstToken(ptr_type.ast.bit_range_start) - 1, .none); // colon + try renderExpression(gpa, ais, tree, ptr_type.ast.bit_range_start, .none); + try renderToken(ais, tree, tree.firstToken(ptr_type.ast.bit_range_end) - 1, .none); // colon + try renderExpression(gpa, ais, tree, ptr_type.ast.bit_range_end, .none); + try renderToken(ais, tree, tree.lastToken(ptr_type.ast.bit_range_end) + 1, .space); // rparen + } else { + try renderToken(ais, tree, tree.lastToken(ptr_type.ast.align_node) + 1, .space); // rparen + } + } + + if (ptr_type.ast.addrspace_node != 0) { + const addrspace_first = tree.firstToken(ptr_type.ast.addrspace_node); + try renderToken(ais, tree, addrspace_first - 2, .none); // addrspace + try renderToken(ais, tree, addrspace_first - 1, .none); // lparen + try renderExpression(gpa, ais, tree, ptr_type.ast.addrspace_node, .none); + try renderToken(ais, tree, tree.lastToken(ptr_type.ast.addrspace_node) + 1, .space); // rparen + } + + if (ptr_type.const_token) |const_token| { + try renderToken(ais, tree, const_token, .space); + } + + if (ptr_type.volatile_token) |volatile_token| { + try renderToken(ais, tree, volatile_token, .space); + } + + try renderExpression(gpa, ais, tree, ptr_type.ast.child_type, space); +} + +fn renderSlice( + gpa: Allocator, + ais: *Ais, + tree: Ast, + slice_node: Ast.Node.Index, + slice: Ast.full.Slice, + space: Space, +) Error!void { + const node_tags = tree.nodes.items(.tag); + const after_start_space_bool = nodeCausesSliceOpSpace(node_tags[slice.ast.start]) or + if (slice.ast.end != 0) nodeCausesSliceOpSpace(node_tags[slice.ast.end]) else false; + const after_start_space = if (after_start_space_bool) Space.space else Space.none; + const after_dots_space = if (slice.ast.end != 0) + after_start_space + else if (slice.ast.sentinel != 0) Space.space else Space.none; + + try renderExpression(gpa, ais, tree, slice.ast.sliced, .none); + try renderToken(ais, tree, slice.ast.lbracket, .none); // lbracket + + const start_last = tree.lastToken(slice.ast.start); + try renderExpression(gpa, ais, tree, slice.ast.start, after_start_space); + try renderToken(ais, tree, start_last + 1, after_dots_space); // ellipsis2 ("..") + + if (slice.ast.end != 0) { + const after_end_space = if (slice.ast.sentinel != 0) Space.space else Space.none; + try renderExpression(gpa, ais, tree, slice.ast.end, after_end_space); + } + + if (slice.ast.sentinel != 0) { + try renderToken(ais, tree, tree.firstToken(slice.ast.sentinel) - 1, .none); // colon + try renderExpression(gpa, ais, tree, slice.ast.sentinel, .none); + } + + try renderToken(ais, tree, tree.lastToken(slice_node), space); // rbracket +} + +fn renderAsmOutput( + gpa: Allocator, + ais: *Ais, + tree: Ast, + asm_output: Ast.Node.Index, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + const node_tags = tree.nodes.items(.tag); + const main_tokens = tree.nodes.items(.main_token); + const datas = tree.nodes.items(.data); + assert(node_tags[asm_output] == .asm_output); + const symbolic_name = main_tokens[asm_output]; + + try renderToken(ais, tree, symbolic_name - 1, .none); // lbracket + try renderIdentifier(ais, tree, symbolic_name, .none, .eagerly_unquote); // ident + try renderToken(ais, tree, symbolic_name + 1, .space); // rbracket + try renderToken(ais, tree, symbolic_name + 2, .space); // "constraint" + try renderToken(ais, tree, symbolic_name + 3, .none); // lparen + + if (token_tags[symbolic_name + 4] == .arrow) { + try renderToken(ais, tree, symbolic_name + 4, .space); // -> + try renderExpression(gpa, ais, tree, datas[asm_output].lhs, Space.none); + return renderToken(ais, tree, datas[asm_output].rhs, space); // rparen + } else { + try renderIdentifier(ais, tree, symbolic_name + 4, .none, .eagerly_unquote); // ident + return renderToken(ais, tree, symbolic_name + 5, space); // rparen + } +} + +fn renderAsmInput( + gpa: Allocator, + ais: *Ais, + tree: Ast, + asm_input: Ast.Node.Index, + space: Space, +) Error!void { + const node_tags = tree.nodes.items(.tag); + const main_tokens = tree.nodes.items(.main_token); + const datas = tree.nodes.items(.data); + assert(node_tags[asm_input] == .asm_input); + const symbolic_name = main_tokens[asm_input]; + + try renderToken(ais, tree, symbolic_name - 1, .none); // lbracket + try renderIdentifier(ais, tree, symbolic_name, .none, .eagerly_unquote); // ident + try renderToken(ais, tree, symbolic_name + 1, .space); // rbracket + try renderToken(ais, tree, symbolic_name + 2, .space); // "constraint" + try renderToken(ais, tree, symbolic_name + 3, .none); // lparen + try renderExpression(gpa, ais, tree, datas[asm_input].lhs, Space.none); + return renderToken(ais, tree, datas[asm_input].rhs, space); // rparen +} + +fn renderVarDecl( + gpa: Allocator, + ais: *Ais, + tree: Ast, + var_decl: Ast.full.VarDecl, + /// Destructures intentionally ignore leading `comptime` tokens. + ignore_comptime_token: bool, + /// `comma_space` and `space` are used for destructure LHS decls. + space: Space, +) Error!void { + if (var_decl.visib_token) |visib_token| { + try renderToken(ais, tree, visib_token, Space.space); // pub + } + + if (var_decl.extern_export_token) |extern_export_token| { + try renderToken(ais, tree, extern_export_token, Space.space); // extern + + if (var_decl.lib_name) |lib_name| { + try renderToken(ais, tree, lib_name, Space.space); // "lib" + } + } + + if (var_decl.threadlocal_token) |thread_local_token| { + try renderToken(ais, tree, thread_local_token, Space.space); // threadlocal + } + + if (!ignore_comptime_token) { + if (var_decl.comptime_token) |comptime_token| { + try renderToken(ais, tree, comptime_token, Space.space); // comptime + } + } + + try renderToken(ais, tree, var_decl.ast.mut_token, .space); // var + + if (var_decl.ast.type_node != 0 or var_decl.ast.align_node != 0 or + var_decl.ast.addrspace_node != 0 or var_decl.ast.section_node != 0 or + var_decl.ast.init_node != 0) + { + const name_space = if (var_decl.ast.type_node == 0 and + (var_decl.ast.align_node != 0 or + var_decl.ast.addrspace_node != 0 or + var_decl.ast.section_node != 0 or + var_decl.ast.init_node != 0)) + Space.space + else + Space.none; + + try renderIdentifier(ais, tree, var_decl.ast.mut_token + 1, name_space, .preserve_when_shadowing); // name + } else { + return renderIdentifier(ais, tree, var_decl.ast.mut_token + 1, space, .preserve_when_shadowing); // name + } + + if (var_decl.ast.type_node != 0) { + try renderToken(ais, tree, var_decl.ast.mut_token + 2, Space.space); // : + if (var_decl.ast.align_node != 0 or var_decl.ast.addrspace_node != 0 or + var_decl.ast.section_node != 0 or var_decl.ast.init_node != 0) + { + try renderExpression(gpa, ais, tree, var_decl.ast.type_node, .space); + } else { + return renderExpression(gpa, ais, tree, var_decl.ast.type_node, space); + } + } + + if (var_decl.ast.align_node != 0) { + const lparen = tree.firstToken(var_decl.ast.align_node) - 1; + const align_kw = lparen - 1; + const rparen = tree.lastToken(var_decl.ast.align_node) + 1; + try renderToken(ais, tree, align_kw, Space.none); // align + try renderToken(ais, tree, lparen, Space.none); // ( + try renderExpression(gpa, ais, tree, var_decl.ast.align_node, Space.none); + if (var_decl.ast.addrspace_node != 0 or var_decl.ast.section_node != 0 or + var_decl.ast.init_node != 0) + { + try renderToken(ais, tree, rparen, .space); // ) + } else { + return renderToken(ais, tree, rparen, space); // ) + } + } + + if (var_decl.ast.addrspace_node != 0) { + const lparen = tree.firstToken(var_decl.ast.addrspace_node) - 1; + const addrspace_kw = lparen - 1; + const rparen = tree.lastToken(var_decl.ast.addrspace_node) + 1; + try renderToken(ais, tree, addrspace_kw, Space.none); // addrspace + try renderToken(ais, tree, lparen, Space.none); // ( + try renderExpression(gpa, ais, tree, var_decl.ast.addrspace_node, Space.none); + if (var_decl.ast.section_node != 0 or var_decl.ast.init_node != 0) { + try renderToken(ais, tree, rparen, .space); // ) + } else { + try renderToken(ais, tree, rparen, .none); // ) + return renderToken(ais, tree, rparen + 1, Space.newline); // ; + } + } + + if (var_decl.ast.section_node != 0) { + const lparen = tree.firstToken(var_decl.ast.section_node) - 1; + const section_kw = lparen - 1; + const rparen = tree.lastToken(var_decl.ast.section_node) + 1; + try renderToken(ais, tree, section_kw, Space.none); // linksection + try renderToken(ais, tree, lparen, Space.none); // ( + try renderExpression(gpa, ais, tree, var_decl.ast.section_node, Space.none); + if (var_decl.ast.init_node != 0) { + try renderToken(ais, tree, rparen, .space); // ) + } else { + return renderToken(ais, tree, rparen, space); // ) + } + } + + assert(var_decl.ast.init_node != 0); + + const eq_token = tree.firstToken(var_decl.ast.init_node) - 1; + const eq_space: Space = if (tree.tokensOnSameLine(eq_token, eq_token + 1)) .space else .newline; + { + ais.pushIndent(); + try renderToken(ais, tree, eq_token, eq_space); // = + ais.popIndent(); + } + ais.pushIndentOneShot(); + return renderExpression(gpa, ais, tree, var_decl.ast.init_node, space); // ; +} + +fn renderIf(gpa: Allocator, ais: *Ais, tree: Ast, if_node: Ast.full.If, space: Space) Error!void { + return renderWhile(gpa, ais, tree, .{ + .ast = .{ + .while_token = if_node.ast.if_token, + .cond_expr = if_node.ast.cond_expr, + .cont_expr = 0, + .then_expr = if_node.ast.then_expr, + .else_expr = if_node.ast.else_expr, + }, + .inline_token = null, + .label_token = null, + .payload_token = if_node.payload_token, + .else_token = if_node.else_token, + .error_token = if_node.error_token, + }, space); +} + +/// Note that this function is additionally used to render if expressions, with +/// respective values set to null. +fn renderWhile(gpa: Allocator, ais: *Ais, tree: Ast, while_node: Ast.full.While, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + + if (while_node.label_token) |label| { + try renderIdentifier(ais, tree, label, .none, .eagerly_unquote); // label + try renderToken(ais, tree, label + 1, .space); // : + } + + if (while_node.inline_token) |inline_token| { + try renderToken(ais, tree, inline_token, .space); // inline + } + + try renderToken(ais, tree, while_node.ast.while_token, .space); // if/for/while + try renderToken(ais, tree, while_node.ast.while_token + 1, .none); // lparen + try renderExpression(gpa, ais, tree, while_node.ast.cond_expr, .none); // condition + + var last_prefix_token = tree.lastToken(while_node.ast.cond_expr) + 1; // rparen + + if (while_node.payload_token) |payload_token| { + try renderToken(ais, tree, last_prefix_token, .space); + try renderToken(ais, tree, payload_token - 1, .none); // | + const ident = blk: { + if (token_tags[payload_token] == .asterisk) { + try renderToken(ais, tree, payload_token, .none); // * + break :blk payload_token + 1; + } else { + break :blk payload_token; + } + }; + try renderIdentifier(ais, tree, ident, .none, .preserve_when_shadowing); // identifier + const pipe = blk: { + if (token_tags[ident + 1] == .comma) { + try renderToken(ais, tree, ident + 1, .space); // , + try renderIdentifier(ais, tree, ident + 2, .none, .preserve_when_shadowing); // index + break :blk ident + 3; + } else { + break :blk ident + 1; + } + }; + last_prefix_token = pipe; + } + + if (while_node.ast.cont_expr != 0) { + try renderToken(ais, tree, last_prefix_token, .space); + const lparen = tree.firstToken(while_node.ast.cont_expr) - 1; + try renderToken(ais, tree, lparen - 1, .space); // : + try renderToken(ais, tree, lparen, .none); // lparen + try renderExpression(gpa, ais, tree, while_node.ast.cont_expr, .none); + last_prefix_token = tree.lastToken(while_node.ast.cont_expr) + 1; // rparen + } + + try renderThenElse( + gpa, + ais, + tree, + last_prefix_token, + while_node.ast.then_expr, + while_node.else_token, + while_node.error_token, + while_node.ast.else_expr, + space, + ); +} + +fn renderThenElse( + gpa: Allocator, + ais: *Ais, + tree: Ast, + last_prefix_token: Ast.TokenIndex, + then_expr: Ast.Node.Index, + else_token: Ast.TokenIndex, + maybe_error_token: ?Ast.TokenIndex, + else_expr: Ast.Node.Index, + space: Space, +) Error!void { + const node_tags = tree.nodes.items(.tag); + const then_expr_is_block = nodeIsBlock(node_tags[then_expr]); + const indent_then_expr = !then_expr_is_block and + !tree.tokensOnSameLine(last_prefix_token, tree.firstToken(then_expr)); + if (indent_then_expr or (then_expr_is_block and ais.isLineOverIndented())) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, last_prefix_token, .newline); + ais.popIndent(); + } else { + try renderToken(ais, tree, last_prefix_token, .space); + } + + if (else_expr != 0) { + if (indent_then_expr) { + ais.pushIndent(); + try renderExpression(gpa, ais, tree, then_expr, .newline); + ais.popIndent(); + } else { + try renderExpression(gpa, ais, tree, then_expr, .space); + } + + var last_else_token = else_token; + + if (maybe_error_token) |error_token| { + try renderToken(ais, tree, else_token, .space); // else + try renderToken(ais, tree, error_token - 1, .none); // | + try renderIdentifier(ais, tree, error_token, .none, .preserve_when_shadowing); // identifier + last_else_token = error_token + 1; // | + } + + const indent_else_expr = indent_then_expr and + !nodeIsBlock(node_tags[else_expr]) and + !nodeIsIfForWhileSwitch(node_tags[else_expr]); + if (indent_else_expr) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, last_else_token, .newline); + ais.popIndent(); + try renderExpressionIndented(gpa, ais, tree, else_expr, space); + } else { + try renderToken(ais, tree, last_else_token, .space); + try renderExpression(gpa, ais, tree, else_expr, space); + } + } else { + if (indent_then_expr) { + try renderExpressionIndented(gpa, ais, tree, then_expr, space); + } else { + try renderExpression(gpa, ais, tree, then_expr, space); + } + } +} + +fn renderFor(gpa: Allocator, ais: *Ais, tree: Ast, for_node: Ast.full.For, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + + if (for_node.label_token) |label| { + try renderIdentifier(ais, tree, label, .none, .eagerly_unquote); // label + try renderToken(ais, tree, label + 1, .space); // : + } + + if (for_node.inline_token) |inline_token| { + try renderToken(ais, tree, inline_token, .space); // inline + } + + try renderToken(ais, tree, for_node.ast.for_token, .space); // if/for/while + + const lparen = for_node.ast.for_token + 1; + try renderParamList(gpa, ais, tree, lparen, for_node.ast.inputs, .space); + + var cur = for_node.payload_token; + const pipe = std.mem.indexOfScalarPos(std.zig.Token.Tag, token_tags, cur, .pipe).?; + if (token_tags[pipe - 1] == .comma) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, cur - 1, .newline); // | + while (true) { + if (token_tags[cur] == .asterisk) { + try renderToken(ais, tree, cur, .none); // * + cur += 1; + } + try renderIdentifier(ais, tree, cur, .none, .preserve_when_shadowing); // identifier + cur += 1; + if (token_tags[cur] == .comma) { + try renderToken(ais, tree, cur, .newline); // , + cur += 1; + } + if (token_tags[cur] == .pipe) { + break; + } + } + ais.popIndent(); + } else { + try renderToken(ais, tree, cur - 1, .none); // | + while (true) { + if (token_tags[cur] == .asterisk) { + try renderToken(ais, tree, cur, .none); // * + cur += 1; + } + try renderIdentifier(ais, tree, cur, .none, .preserve_when_shadowing); // identifier + cur += 1; + if (token_tags[cur] == .comma) { + try renderToken(ais, tree, cur, .space); // , + cur += 1; + } + if (token_tags[cur] == .pipe) { + break; + } + } + } + + try renderThenElse( + gpa, + ais, + tree, + cur, + for_node.ast.then_expr, + for_node.else_token, + null, + for_node.ast.else_expr, + space, + ); +} + +fn renderContainerField( + gpa: Allocator, + ais: *Ais, + tree: Ast, + container: Container, + field_param: Ast.full.ContainerField, + space: Space, +) Error!void { + var field = field_param; + if (container != .tuple) field.convertToNonTupleLike(tree.nodes); + const quote: QuoteBehavior = switch (container) { + .@"enum" => .eagerly_unquote_except_underscore, + .tuple, .other => .eagerly_unquote, + }; + + if (field.comptime_token) |t| { + try renderToken(ais, tree, t, .space); // comptime + } + if (field.ast.type_expr == 0 and field.ast.value_expr == 0) { + if (field.ast.align_expr != 0) { + try renderIdentifier(ais, tree, field.ast.main_token, .space, quote); // name + const lparen_token = tree.firstToken(field.ast.align_expr) - 1; + const align_kw = lparen_token - 1; + const rparen_token = tree.lastToken(field.ast.align_expr) + 1; + try renderToken(ais, tree, align_kw, .none); // align + try renderToken(ais, tree, lparen_token, .none); // ( + try renderExpression(gpa, ais, tree, field.ast.align_expr, .none); // alignment + return renderToken(ais, tree, rparen_token, .space); // ) + } + return renderIdentifierComma(ais, tree, field.ast.main_token, space, quote); // name + } + if (field.ast.type_expr != 0 and field.ast.value_expr == 0) { + if (!field.ast.tuple_like) { + try renderIdentifier(ais, tree, field.ast.main_token, .none, quote); // name + try renderToken(ais, tree, field.ast.main_token + 1, .space); // : + } + + if (field.ast.align_expr != 0) { + try renderExpression(gpa, ais, tree, field.ast.type_expr, .space); // type + const align_token = tree.firstToken(field.ast.align_expr) - 2; + try renderToken(ais, tree, align_token, .none); // align + try renderToken(ais, tree, align_token + 1, .none); // ( + try renderExpression(gpa, ais, tree, field.ast.align_expr, .none); // alignment + const rparen = tree.lastToken(field.ast.align_expr) + 1; + return renderTokenComma(ais, tree, rparen, space); // ) + } else { + return renderExpressionComma(gpa, ais, tree, field.ast.type_expr, space); // type + } + } + if (field.ast.type_expr == 0 and field.ast.value_expr != 0) { + try renderIdentifier(ais, tree, field.ast.main_token, .space, quote); // name + if (field.ast.align_expr != 0) { + const lparen_token = tree.firstToken(field.ast.align_expr) - 1; + const align_kw = lparen_token - 1; + const rparen_token = tree.lastToken(field.ast.align_expr) + 1; + try renderToken(ais, tree, align_kw, .none); // align + try renderToken(ais, tree, lparen_token, .none); // ( + try renderExpression(gpa, ais, tree, field.ast.align_expr, .none); // alignment + try renderToken(ais, tree, rparen_token, .space); // ) + } + try renderToken(ais, tree, field.ast.main_token + 1, .space); // = + return renderExpressionComma(gpa, ais, tree, field.ast.value_expr, space); // value + } + if (!field.ast.tuple_like) { + try renderIdentifier(ais, tree, field.ast.main_token, .none, quote); // name + try renderToken(ais, tree, field.ast.main_token + 1, .space); // : + } + try renderExpression(gpa, ais, tree, field.ast.type_expr, .space); // type + + if (field.ast.align_expr != 0) { + const lparen_token = tree.firstToken(field.ast.align_expr) - 1; + const align_kw = lparen_token - 1; + const rparen_token = tree.lastToken(field.ast.align_expr) + 1; + try renderToken(ais, tree, align_kw, .none); // align + try renderToken(ais, tree, lparen_token, .none); // ( + try renderExpression(gpa, ais, tree, field.ast.align_expr, .none); // alignment + try renderToken(ais, tree, rparen_token, .space); // ) + } + const eq_token = tree.firstToken(field.ast.value_expr) - 1; + const eq_space: Space = if (tree.tokensOnSameLine(eq_token, eq_token + 1)) .space else .newline; + { + ais.pushIndent(); + try renderToken(ais, tree, eq_token, eq_space); // = + ais.popIndent(); + } + + if (eq_space == .space) + return renderExpressionComma(gpa, ais, tree, field.ast.value_expr, space); // value + + const token_tags = tree.tokens.items(.tag); + const maybe_comma = tree.lastToken(field.ast.value_expr) + 1; + + if (token_tags[maybe_comma] == .comma) { + ais.pushIndent(); + try renderExpression(gpa, ais, tree, field.ast.value_expr, .none); // value + ais.popIndent(); + try renderToken(ais, tree, maybe_comma, .newline); + } else { + ais.pushIndent(); + try renderExpression(gpa, ais, tree, field.ast.value_expr, space); // value + ais.popIndent(); + } +} + +fn renderBuiltinCall( + gpa: Allocator, + ais: *Ais, + tree: Ast, + builtin_token: Ast.TokenIndex, + params: []const Ast.Node.Index, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + + // TODO remove before release of 0.12.0 + const slice = tree.tokenSlice(builtin_token); + const rewrite_two_param_cast = params.len == 2 and for ([_][]const u8{ + "@bitCast", + "@errorCast", + "@floatCast", + "@intCast", + "@ptrCast", + "@intFromFloat", + "@floatToInt", + "@enumFromInt", + "@intToEnum", + "@floatFromInt", + "@intToFloat", + "@ptrFromInt", + "@intToPtr", + "@truncate", + }) |name| { + if (mem.eql(u8, slice, name)) break true; + } else false; + + if (rewrite_two_param_cast) { + const after_last_param_token = tree.lastToken(params[1]) + 1; + if (token_tags[after_last_param_token] != .comma) { + // Render all on one line, no trailing comma. + try ais.writer().writeAll("@as"); + try renderToken(ais, tree, builtin_token + 1, .none); // ( + try renderExpression(gpa, ais, tree, params[0], .comma_space); + } else { + // Render one param per line. + try ais.writer().writeAll("@as"); + ais.pushIndent(); + try renderToken(ais, tree, builtin_token + 1, .newline); // ( + try renderExpression(gpa, ais, tree, params[0], .comma); + } + } + // Corresponding logic below builtin name rewrite below + + // TODO remove before release of 0.11.0 + if (mem.eql(u8, slice, "@maximum")) { + try ais.writer().writeAll("@max"); + } else if (mem.eql(u8, slice, "@minimum")) { + try ais.writer().writeAll("@min"); + } + // TODO remove before release of 0.12.0 + else if (mem.eql(u8, slice, "@boolToInt")) { + try ais.writer().writeAll("@intFromBool"); + } else if (mem.eql(u8, slice, "@enumToInt")) { + try ais.writer().writeAll("@intFromEnum"); + } else if (mem.eql(u8, slice, "@errorToInt")) { + try ais.writer().writeAll("@intFromError"); + } else if (mem.eql(u8, slice, "@floatToInt")) { + try ais.writer().writeAll("@intFromFloat"); + } else if (mem.eql(u8, slice, "@intToEnum")) { + try ais.writer().writeAll("@enumFromInt"); + } else if (mem.eql(u8, slice, "@intToError")) { + try ais.writer().writeAll("@errorFromInt"); + } else if (mem.eql(u8, slice, "@intToFloat")) { + try ais.writer().writeAll("@floatFromInt"); + } else if (mem.eql(u8, slice, "@intToPtr")) { + try ais.writer().writeAll("@ptrFromInt"); + } else if (mem.eql(u8, slice, "@ptrToInt")) { + try ais.writer().writeAll("@intFromPtr"); + } else if (mem.eql(u8, slice, "@fabs")) { + try ais.writer().writeAll("@abs"); + } else if (mem.eql(u8, slice, "@errSetCast")) { + try ais.writer().writeAll("@errorCast"); + } else { + try renderToken(ais, tree, builtin_token, .none); // @name + } + + if (rewrite_two_param_cast) { + // Matches with corresponding logic above builtin name rewrite + const after_last_param_token = tree.lastToken(params[1]) + 1; + try ais.writer().writeAll("("); + try renderExpression(gpa, ais, tree, params[1], .none); + try ais.writer().writeAll(")"); + if (token_tags[after_last_param_token] != .comma) { + // Render all on one line, no trailing comma. + return renderToken(ais, tree, after_last_param_token, space); // ) + } else { + // Render one param per line. + ais.popIndent(); + try renderToken(ais, tree, after_last_param_token, .newline); // , + return renderToken(ais, tree, after_last_param_token + 1, space); // ) + } + } + + if (params.len == 0) { + try renderToken(ais, tree, builtin_token + 1, .none); // ( + return renderToken(ais, tree, builtin_token + 2, space); // ) + } + + const last_param = params[params.len - 1]; + const after_last_param_token = tree.lastToken(last_param) + 1; + + if (token_tags[after_last_param_token] != .comma) { + // Render all on one line, no trailing comma. + try renderToken(ais, tree, builtin_token + 1, .none); // ( + + for (params, 0..) |param_node, i| { + const first_param_token = tree.firstToken(param_node); + if (token_tags[first_param_token] == .multiline_string_literal_line or + hasSameLineComment(tree, first_param_token - 1)) + { + ais.pushIndentOneShot(); + } + try renderExpression(gpa, ais, tree, param_node, .none); + + if (i + 1 < params.len) { + const comma_token = tree.lastToken(param_node) + 1; + try renderToken(ais, tree, comma_token, .space); // , + } + } + return renderToken(ais, tree, after_last_param_token, space); // ) + } else { + // Render one param per line. + ais.pushIndent(); + try renderToken(ais, tree, builtin_token + 1, Space.newline); // ( + + for (params) |param_node| { + try renderExpression(gpa, ais, tree, param_node, .comma); + } + ais.popIndent(); + + return renderToken(ais, tree, after_last_param_token + 1, space); // ) + } +} + +fn renderFnProto(gpa: Allocator, ais: *Ais, tree: Ast, fn_proto: Ast.full.FnProto, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + const token_starts = tree.tokens.items(.start); + + const after_fn_token = fn_proto.ast.fn_token + 1; + const lparen = if (token_tags[after_fn_token] == .identifier) blk: { + try renderToken(ais, tree, fn_proto.ast.fn_token, .space); // fn + try renderIdentifier(ais, tree, after_fn_token, .none, .preserve_when_shadowing); // name + break :blk after_fn_token + 1; + } else blk: { + try renderToken(ais, tree, fn_proto.ast.fn_token, .space); // fn + break :blk fn_proto.ast.fn_token + 1; + }; + assert(token_tags[lparen] == .l_paren); + + const maybe_bang = tree.firstToken(fn_proto.ast.return_type) - 1; + const rparen = blk: { + // These may appear in any order, so we have to check the token_starts array + // to find out which is first. + var rparen = if (token_tags[maybe_bang] == .bang) maybe_bang - 1 else maybe_bang; + var smallest_start = token_starts[maybe_bang]; + if (fn_proto.ast.align_expr != 0) { + const tok = tree.firstToken(fn_proto.ast.align_expr) - 3; + const start = token_starts[tok]; + if (start < smallest_start) { + rparen = tok; + smallest_start = start; + } + } + if (fn_proto.ast.addrspace_expr != 0) { + const tok = tree.firstToken(fn_proto.ast.addrspace_expr) - 3; + const start = token_starts[tok]; + if (start < smallest_start) { + rparen = tok; + smallest_start = start; + } + } + if (fn_proto.ast.section_expr != 0) { + const tok = tree.firstToken(fn_proto.ast.section_expr) - 3; + const start = token_starts[tok]; + if (start < smallest_start) { + rparen = tok; + smallest_start = start; + } + } + if (fn_proto.ast.callconv_expr != 0) { + const tok = tree.firstToken(fn_proto.ast.callconv_expr) - 3; + const start = token_starts[tok]; + if (start < smallest_start) { + rparen = tok; + smallest_start = start; + } + } + break :blk rparen; + }; + assert(token_tags[rparen] == .r_paren); + + // The params list is a sparse set that does *not* include anytype or ... parameters. + + const trailing_comma = token_tags[rparen - 1] == .comma; + if (!trailing_comma and !hasComment(tree, lparen, rparen)) { + // Render all on one line, no trailing comma. + try renderToken(ais, tree, lparen, .none); // ( + + var param_i: usize = 0; + var last_param_token = lparen; + while (true) { + last_param_token += 1; + switch (token_tags[last_param_token]) { + .doc_comment => { + try renderToken(ais, tree, last_param_token, .newline); + continue; + }, + .ellipsis3 => { + try renderToken(ais, tree, last_param_token, .none); // ... + break; + }, + .keyword_noalias, .keyword_comptime => { + try renderToken(ais, tree, last_param_token, .space); + last_param_token += 1; + }, + .identifier => {}, + .keyword_anytype => { + try renderToken(ais, tree, last_param_token, .none); // anytype + continue; + }, + .r_paren => break, + .comma => { + try renderToken(ais, tree, last_param_token, .space); // , + continue; + }, + else => {}, // Parameter type without a name. + } + if (token_tags[last_param_token] == .identifier and + token_tags[last_param_token + 1] == .colon) + { + try renderIdentifier(ais, tree, last_param_token, .none, .preserve_when_shadowing); // name + last_param_token += 1; + try renderToken(ais, tree, last_param_token, .space); // : + last_param_token += 1; + } + if (token_tags[last_param_token] == .keyword_anytype) { + try renderToken(ais, tree, last_param_token, .none); // anytype + continue; + } + const param = fn_proto.ast.params[param_i]; + param_i += 1; + try renderExpression(gpa, ais, tree, param, .none); + last_param_token = tree.lastToken(param); + } + } else { + // One param per line. + ais.pushIndent(); + try renderToken(ais, tree, lparen, .newline); // ( + + var param_i: usize = 0; + var last_param_token = lparen; + while (true) { + last_param_token += 1; + switch (token_tags[last_param_token]) { + .doc_comment => { + try renderToken(ais, tree, last_param_token, .newline); + continue; + }, + .ellipsis3 => { + try renderToken(ais, tree, last_param_token, .comma); // ... + break; + }, + .keyword_noalias, .keyword_comptime => { + try renderToken(ais, tree, last_param_token, .space); + last_param_token += 1; + }, + .identifier => {}, + .keyword_anytype => { + try renderToken(ais, tree, last_param_token, .comma); // anytype + if (token_tags[last_param_token + 1] == .comma) + last_param_token += 1; + continue; + }, + .r_paren => break, + else => {}, // Parameter type without a name. + } + if (token_tags[last_param_token] == .identifier and + token_tags[last_param_token + 1] == .colon) + { + try renderIdentifier(ais, tree, last_param_token, .none, .preserve_when_shadowing); // name + last_param_token += 1; + try renderToken(ais, tree, last_param_token, .space); // : + last_param_token += 1; + } + if (token_tags[last_param_token] == .keyword_anytype) { + try renderToken(ais, tree, last_param_token, .comma); // anytype + if (token_tags[last_param_token + 1] == .comma) + last_param_token += 1; + continue; + } + const param = fn_proto.ast.params[param_i]; + param_i += 1; + try renderExpression(gpa, ais, tree, param, .comma); + last_param_token = tree.lastToken(param); + if (token_tags[last_param_token + 1] == .comma) last_param_token += 1; + } + ais.popIndent(); + } + + try renderToken(ais, tree, rparen, .space); // ) + + if (fn_proto.ast.align_expr != 0) { + const align_lparen = tree.firstToken(fn_proto.ast.align_expr) - 1; + const align_rparen = tree.lastToken(fn_proto.ast.align_expr) + 1; + + try renderToken(ais, tree, align_lparen - 1, .none); // align + try renderToken(ais, tree, align_lparen, .none); // ( + try renderExpression(gpa, ais, tree, fn_proto.ast.align_expr, .none); + try renderToken(ais, tree, align_rparen, .space); // ) + } + + if (fn_proto.ast.addrspace_expr != 0) { + const align_lparen = tree.firstToken(fn_proto.ast.addrspace_expr) - 1; + const align_rparen = tree.lastToken(fn_proto.ast.addrspace_expr) + 1; + + try renderToken(ais, tree, align_lparen - 1, .none); // addrspace + try renderToken(ais, tree, align_lparen, .none); // ( + try renderExpression(gpa, ais, tree, fn_proto.ast.addrspace_expr, .none); + try renderToken(ais, tree, align_rparen, .space); // ) + } + + if (fn_proto.ast.section_expr != 0) { + const section_lparen = tree.firstToken(fn_proto.ast.section_expr) - 1; + const section_rparen = tree.lastToken(fn_proto.ast.section_expr) + 1; + + try renderToken(ais, tree, section_lparen - 1, .none); // section + try renderToken(ais, tree, section_lparen, .none); // ( + try renderExpression(gpa, ais, tree, fn_proto.ast.section_expr, .none); + try renderToken(ais, tree, section_rparen, .space); // ) + } + + const is_callconv_inline = mem.eql(u8, "Inline", tree.tokenSlice(tree.nodes.items(.main_token)[fn_proto.ast.callconv_expr])); + const is_declaration = fn_proto.name_token != null; + if (fn_proto.ast.callconv_expr != 0 and !(is_declaration and is_callconv_inline)) { + const callconv_lparen = tree.firstToken(fn_proto.ast.callconv_expr) - 1; + const callconv_rparen = tree.lastToken(fn_proto.ast.callconv_expr) + 1; + + try renderToken(ais, tree, callconv_lparen - 1, .none); // callconv + try renderToken(ais, tree, callconv_lparen, .none); // ( + try renderExpression(gpa, ais, tree, fn_proto.ast.callconv_expr, .none); + try renderToken(ais, tree, callconv_rparen, .space); // ) + } + + if (token_tags[maybe_bang] == .bang) { + try renderToken(ais, tree, maybe_bang, .none); // ! + } + return renderExpression(gpa, ais, tree, fn_proto.ast.return_type, space); +} + +fn renderSwitchCase( + gpa: Allocator, + ais: *Ais, + tree: Ast, + switch_case: Ast.full.SwitchCase, + space: Space, +) Error!void { + const node_tags = tree.nodes.items(.tag); + const token_tags = tree.tokens.items(.tag); + const trailing_comma = token_tags[switch_case.ast.arrow_token - 1] == .comma; + const has_comment_before_arrow = blk: { + if (switch_case.ast.values.len == 0) break :blk false; + break :blk hasComment(tree, tree.firstToken(switch_case.ast.values[0]), switch_case.ast.arrow_token); + }; + + // render inline keyword + if (switch_case.inline_token) |some| { + try renderToken(ais, tree, some, .space); + } + + // Render everything before the arrow + if (switch_case.ast.values.len == 0) { + try renderToken(ais, tree, switch_case.ast.arrow_token - 1, .space); // else keyword + } else if (switch_case.ast.values.len == 1 and !has_comment_before_arrow) { + // render on one line and drop the trailing comma if any + try renderExpression(gpa, ais, tree, switch_case.ast.values[0], .space); + } else if (trailing_comma or has_comment_before_arrow) { + // Render each value on a new line + try renderExpressions(gpa, ais, tree, switch_case.ast.values, .comma); + } else { + // Render on one line + for (switch_case.ast.values) |value_expr| { + try renderExpression(gpa, ais, tree, value_expr, .comma_space); + } + } + + // Render the arrow and everything after it + const pre_target_space = if (node_tags[switch_case.ast.target_expr] == .multiline_string_literal) + // Newline gets inserted when rendering the target expr. + Space.none + else + Space.space; + const after_arrow_space: Space = if (switch_case.payload_token == null) pre_target_space else .space; + try renderToken(ais, tree, switch_case.ast.arrow_token, after_arrow_space); // => + + if (switch_case.payload_token) |payload_token| { + try renderToken(ais, tree, payload_token - 1, .none); // pipe + const ident = payload_token + @intFromBool(token_tags[payload_token] == .asterisk); + if (token_tags[payload_token] == .asterisk) { + try renderToken(ais, tree, payload_token, .none); // asterisk + } + try renderIdentifier(ais, tree, ident, .none, .preserve_when_shadowing); // identifier + if (token_tags[ident + 1] == .comma) { + try renderToken(ais, tree, ident + 1, .space); // , + try renderIdentifier(ais, tree, ident + 2, .none, .preserve_when_shadowing); // identifier + try renderToken(ais, tree, ident + 3, pre_target_space); // pipe + } else { + try renderToken(ais, tree, ident + 1, pre_target_space); // pipe + } + } + + try renderExpression(gpa, ais, tree, switch_case.ast.target_expr, space); +} + +fn renderBlock( + gpa: Allocator, + ais: *Ais, + tree: Ast, + block_node: Ast.Node.Index, + statements: []const Ast.Node.Index, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + const node_tags = tree.nodes.items(.tag); + const lbrace = tree.nodes.items(.main_token)[block_node]; + + if (token_tags[lbrace - 1] == .colon and + token_tags[lbrace - 2] == .identifier) + { + try renderIdentifier(ais, tree, lbrace - 2, .none, .eagerly_unquote); // identifier + try renderToken(ais, tree, lbrace - 1, .space); // : + } + + ais.pushIndentNextLine(); + if (statements.len == 0) { + try renderToken(ais, tree, lbrace, .none); + } else { + try renderToken(ais, tree, lbrace, .newline); + for (statements, 0..) |stmt, i| { + if (i != 0) try renderExtraNewline(ais, tree, stmt); + switch (node_tags[stmt]) { + .global_var_decl, + .local_var_decl, + .simple_var_decl, + .aligned_var_decl, + => try renderVarDecl(gpa, ais, tree, tree.fullVarDecl(stmt).?, false, .semicolon), + else => try renderExpression(gpa, ais, tree, stmt, .semicolon), + } + } + } + ais.popIndent(); + + try renderToken(ais, tree, tree.lastToken(block_node), space); // rbrace +} + +fn renderStructInit( + gpa: Allocator, + ais: *Ais, + tree: Ast, + struct_node: Ast.Node.Index, + struct_init: Ast.full.StructInit, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + if (struct_init.ast.type_expr == 0) { + try renderToken(ais, tree, struct_init.ast.lbrace - 1, .none); // . + } else { + try renderExpression(gpa, ais, tree, struct_init.ast.type_expr, .none); // T + } + if (struct_init.ast.fields.len == 0) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, struct_init.ast.lbrace, .none); // lbrace + ais.popIndent(); + return renderToken(ais, tree, struct_init.ast.lbrace + 1, space); // rbrace + } + + const rbrace = tree.lastToken(struct_node); + const trailing_comma = token_tags[rbrace - 1] == .comma; + if (trailing_comma or hasComment(tree, struct_init.ast.lbrace, rbrace)) { + // Render one field init per line. + ais.pushIndentNextLine(); + try renderToken(ais, tree, struct_init.ast.lbrace, .newline); + + try renderToken(ais, tree, struct_init.ast.lbrace + 1, .none); // . + try renderIdentifier(ais, tree, struct_init.ast.lbrace + 2, .space, .eagerly_unquote); // name + // Don't output a space after the = if expression is a multiline string, + // since then it will start on the next line. + const nodes = tree.nodes.items(.tag); + const expr = nodes[struct_init.ast.fields[0]]; + var space_after_equal: Space = if (expr == .multiline_string_literal) .none else .space; + try renderToken(ais, tree, struct_init.ast.lbrace + 3, space_after_equal); // = + try renderExpression(gpa, ais, tree, struct_init.ast.fields[0], .comma); + + for (struct_init.ast.fields[1..]) |field_init| { + const init_token = tree.firstToken(field_init); + try renderExtraNewlineToken(ais, tree, init_token - 3); + try renderToken(ais, tree, init_token - 3, .none); // . + try renderIdentifier(ais, tree, init_token - 2, .space, .eagerly_unquote); // name + space_after_equal = if (nodes[field_init] == .multiline_string_literal) .none else .space; + try renderToken(ais, tree, init_token - 1, space_after_equal); // = + try renderExpression(gpa, ais, tree, field_init, .comma); + } + + ais.popIndent(); + } else { + // Render all on one line, no trailing comma. + try renderToken(ais, tree, struct_init.ast.lbrace, .space); + + for (struct_init.ast.fields) |field_init| { + const init_token = tree.firstToken(field_init); + try renderToken(ais, tree, init_token - 3, .none); // . + try renderIdentifier(ais, tree, init_token - 2, .space, .eagerly_unquote); // name + try renderToken(ais, tree, init_token - 1, .space); // = + try renderExpression(gpa, ais, tree, field_init, .comma_space); + } + } + + return renderToken(ais, tree, rbrace, space); +} + +fn renderArrayInit( + gpa: Allocator, + ais: *Ais, + tree: Ast, + array_init: Ast.full.ArrayInit, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + + if (array_init.ast.type_expr == 0) { + try renderToken(ais, tree, array_init.ast.lbrace - 1, .none); // . + } else { + try renderExpression(gpa, ais, tree, array_init.ast.type_expr, .none); // T + } + + if (array_init.ast.elements.len == 0) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, array_init.ast.lbrace, .none); // lbrace + ais.popIndent(); + return renderToken(ais, tree, array_init.ast.lbrace + 1, space); // rbrace + } + + const last_elem = array_init.ast.elements[array_init.ast.elements.len - 1]; + const last_elem_token = tree.lastToken(last_elem); + const trailing_comma = token_tags[last_elem_token + 1] == .comma; + const rbrace = if (trailing_comma) last_elem_token + 2 else last_elem_token + 1; + assert(token_tags[rbrace] == .r_brace); + + if (array_init.ast.elements.len == 1) { + const only_elem = array_init.ast.elements[0]; + const first_token = tree.firstToken(only_elem); + if (token_tags[first_token] != .multiline_string_literal_line and + !anythingBetween(tree, last_elem_token, rbrace)) + { + try renderToken(ais, tree, array_init.ast.lbrace, .none); + try renderExpression(gpa, ais, tree, only_elem, .none); + return renderToken(ais, tree, rbrace, space); + } + } + + const contains_comment = hasComment(tree, array_init.ast.lbrace, rbrace); + const contains_multiline_string = hasMultilineString(tree, array_init.ast.lbrace, rbrace); + + if (!trailing_comma and !contains_comment and !contains_multiline_string) { + // Render all on one line, no trailing comma. + if (array_init.ast.elements.len == 1) { + // If there is only one element, we don't use spaces + try renderToken(ais, tree, array_init.ast.lbrace, .none); + try renderExpression(gpa, ais, tree, array_init.ast.elements[0], .none); + } else { + try renderToken(ais, tree, array_init.ast.lbrace, .space); + for (array_init.ast.elements) |elem| { + try renderExpression(gpa, ais, tree, elem, .comma_space); + } + } + return renderToken(ais, tree, last_elem_token + 1, space); // rbrace + } + + ais.pushIndentNextLine(); + try renderToken(ais, tree, array_init.ast.lbrace, .newline); + + var expr_index: usize = 0; + while (true) { + const row_size = rowSize(tree, array_init.ast.elements[expr_index..], rbrace); + const row_exprs = array_init.ast.elements[expr_index..]; + // A place to store the width of each expression and its column's maximum + const widths = try gpa.alloc(usize, row_exprs.len + row_size); + defer gpa.free(widths); + @memset(widths, 0); + + const expr_newlines = try gpa.alloc(bool, row_exprs.len); + defer gpa.free(expr_newlines); + @memset(expr_newlines, false); + + const expr_widths = widths[0..row_exprs.len]; + const column_widths = widths[row_exprs.len..]; + + // Find next row with trailing comment (if any) to end the current section. + const section_end = sec_end: { + var this_line_first_expr: usize = 0; + var this_line_size = rowSize(tree, row_exprs, rbrace); + for (row_exprs, 0..) |expr, i| { + // Ignore comment on first line of this section. + if (i == 0) continue; + const expr_last_token = tree.lastToken(expr); + if (tree.tokensOnSameLine(tree.firstToken(row_exprs[0]), expr_last_token)) + continue; + // Track start of line containing comment. + if (!tree.tokensOnSameLine(tree.firstToken(row_exprs[this_line_first_expr]), expr_last_token)) { + this_line_first_expr = i; + this_line_size = rowSize(tree, row_exprs[this_line_first_expr..], rbrace); + } + + const maybe_comma = expr_last_token + 1; + if (token_tags[maybe_comma] == .comma) { + if (hasSameLineComment(tree, maybe_comma)) + break :sec_end i - this_line_size + 1; + } + } + break :sec_end row_exprs.len; + }; + expr_index += section_end; + + const section_exprs = row_exprs[0..section_end]; + + var sub_expr_buffer = std.ArrayList(u8).init(gpa); + defer sub_expr_buffer.deinit(); + + const sub_expr_buffer_starts = try gpa.alloc(usize, section_exprs.len + 1); + defer gpa.free(sub_expr_buffer_starts); + + var auto_indenting_stream = Ais{ + .indent_delta = indent_delta, + .underlying_writer = sub_expr_buffer.writer(), + }; + + // Calculate size of columns in current section + var column_counter: usize = 0; + var single_line = true; + var contains_newline = false; + for (section_exprs, 0..) |expr, i| { + const start = sub_expr_buffer.items.len; + sub_expr_buffer_starts[i] = start; + + if (i + 1 < section_exprs.len) { + try renderExpression(gpa, &auto_indenting_stream, tree, expr, .none); + const width = sub_expr_buffer.items.len - start; + const this_contains_newline = mem.indexOfScalar(u8, sub_expr_buffer.items[start..], '\n') != null; + contains_newline = contains_newline or this_contains_newline; + expr_widths[i] = width; + expr_newlines[i] = this_contains_newline; + + if (!this_contains_newline) { + const column = column_counter % row_size; + column_widths[column] = @max(column_widths[column], width); + + const expr_last_token = tree.lastToken(expr) + 1; + const next_expr = section_exprs[i + 1]; + column_counter += 1; + if (!tree.tokensOnSameLine(expr_last_token, tree.firstToken(next_expr))) single_line = false; + } else { + single_line = false; + column_counter = 0; + } + } else { + try renderExpression(gpa, &auto_indenting_stream, tree, expr, .comma); + const width = sub_expr_buffer.items.len - start - 2; + const this_contains_newline = mem.indexOfScalar(u8, sub_expr_buffer.items[start .. sub_expr_buffer.items.len - 1], '\n') != null; + contains_newline = contains_newline or this_contains_newline; + expr_widths[i] = width; + expr_newlines[i] = contains_newline; + + if (!contains_newline) { + const column = column_counter % row_size; + column_widths[column] = @max(column_widths[column], width); + } + } + } + sub_expr_buffer_starts[section_exprs.len] = sub_expr_buffer.items.len; + + // Render exprs in current section. + column_counter = 0; + for (section_exprs, 0..) |expr, i| { + const start = sub_expr_buffer_starts[i]; + const end = sub_expr_buffer_starts[i + 1]; + const expr_text = sub_expr_buffer.items[start..end]; + if (!expr_newlines[i]) { + try ais.writer().writeAll(expr_text); + } else { + var by_line = std.mem.splitScalar(u8, expr_text, '\n'); + var last_line_was_empty = false; + try ais.writer().writeAll(by_line.first()); + while (by_line.next()) |line| { + if (std.mem.startsWith(u8, line, "//") and last_line_was_empty) { + try ais.insertNewline(); + } else { + try ais.maybeInsertNewline(); + } + last_line_was_empty = (line.len == 0); + try ais.writer().writeAll(line); + } + } + + if (i + 1 < section_exprs.len) { + const next_expr = section_exprs[i + 1]; + const comma = tree.lastToken(expr) + 1; + + if (column_counter != row_size - 1) { + if (!expr_newlines[i] and !expr_newlines[i + 1]) { + // Neither the current or next expression is multiline + try renderToken(ais, tree, comma, .space); // , + assert(column_widths[column_counter % row_size] >= expr_widths[i]); + const padding = column_widths[column_counter % row_size] - expr_widths[i]; + try ais.writer().writeByteNTimes(' ', padding); + + column_counter += 1; + continue; + } + } + + if (single_line and row_size != 1) { + try renderToken(ais, tree, comma, .space); // , + continue; + } + + column_counter = 0; + try renderToken(ais, tree, comma, .newline); // , + try renderExtraNewline(ais, tree, next_expr); + } + } + + if (expr_index == array_init.ast.elements.len) + break; + } + + ais.popIndent(); + return renderToken(ais, tree, rbrace, space); // rbrace +} + +fn renderContainerDecl( + gpa: Allocator, + ais: *Ais, + tree: Ast, + container_decl_node: Ast.Node.Index, + container_decl: Ast.full.ContainerDecl, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + + if (container_decl.layout_token) |layout_token| { + try renderToken(ais, tree, layout_token, .space); + } + + const container: Container = switch (token_tags[container_decl.ast.main_token]) { + .keyword_enum => .@"enum", + .keyword_struct => for (container_decl.ast.members) |member| { + if (tree.fullContainerField(member)) |field| if (!field.ast.tuple_like) break .other; + } else .tuple, + else => .other, + }; + + var lbrace: Ast.TokenIndex = undefined; + if (container_decl.ast.enum_token) |enum_token| { + try renderToken(ais, tree, container_decl.ast.main_token, .none); // union + try renderToken(ais, tree, enum_token - 1, .none); // lparen + try renderToken(ais, tree, enum_token, .none); // enum + if (container_decl.ast.arg != 0) { + try renderToken(ais, tree, enum_token + 1, .none); // lparen + try renderExpression(gpa, ais, tree, container_decl.ast.arg, .none); + const rparen = tree.lastToken(container_decl.ast.arg) + 1; + try renderToken(ais, tree, rparen, .none); // rparen + try renderToken(ais, tree, rparen + 1, .space); // rparen + lbrace = rparen + 2; + } else { + try renderToken(ais, tree, enum_token + 1, .space); // rparen + lbrace = enum_token + 2; + } + } else if (container_decl.ast.arg != 0) { + try renderToken(ais, tree, container_decl.ast.main_token, .none); // union + try renderToken(ais, tree, container_decl.ast.main_token + 1, .none); // lparen + try renderExpression(gpa, ais, tree, container_decl.ast.arg, .none); + const rparen = tree.lastToken(container_decl.ast.arg) + 1; + try renderToken(ais, tree, rparen, .space); // rparen + lbrace = rparen + 1; + } else { + try renderToken(ais, tree, container_decl.ast.main_token, .space); // union + lbrace = container_decl.ast.main_token + 1; + } + + const rbrace = tree.lastToken(container_decl_node); + if (container_decl.ast.members.len == 0) { + ais.pushIndentNextLine(); + if (token_tags[lbrace + 1] == .container_doc_comment) { + try renderToken(ais, tree, lbrace, .newline); // lbrace + try renderContainerDocComments(ais, tree, lbrace + 1); + } else { + try renderToken(ais, tree, lbrace, .none); // lbrace + } + ais.popIndent(); + return renderToken(ais, tree, rbrace, space); // rbrace + } + + const src_has_trailing_comma = token_tags[rbrace - 1] == .comma; + if (!src_has_trailing_comma) one_line: { + // We print all the members in-line unless one of the following conditions are true: + + // 1. The container has comments or multiline strings. + if (hasComment(tree, lbrace, rbrace) or hasMultilineString(tree, lbrace, rbrace)) { + break :one_line; + } + + // 2. The container has a container comment. + if (token_tags[lbrace + 1] == .container_doc_comment) break :one_line; + + // 3. A member of the container has a doc comment. + for (token_tags[lbrace + 1 .. rbrace - 1]) |tag| { + if (tag == .doc_comment) break :one_line; + } + + // 4. The container has non-field members. + for (container_decl.ast.members) |member| { + if (tree.fullContainerField(member) == null) break :one_line; + } + + // Print all the declarations on the same line. + try renderToken(ais, tree, lbrace, .space); // lbrace + for (container_decl.ast.members) |member| { + try renderMember(gpa, ais, tree, container, member, .space); + } + return renderToken(ais, tree, rbrace, space); // rbrace + } + + // One member per line. + ais.pushIndentNextLine(); + try renderToken(ais, tree, lbrace, .newline); // lbrace + if (token_tags[lbrace + 1] == .container_doc_comment) { + try renderContainerDocComments(ais, tree, lbrace + 1); + } + for (container_decl.ast.members, 0..) |member, i| { + if (i != 0) try renderExtraNewline(ais, tree, member); + switch (tree.nodes.items(.tag)[member]) { + // For container fields, ensure a trailing comma is added if necessary. + .container_field_init, + .container_field_align, + .container_field, + => try renderMember(gpa, ais, tree, container, member, .comma), + + else => try renderMember(gpa, ais, tree, container, member, .newline), + } + } + ais.popIndent(); + + return renderToken(ais, tree, rbrace, space); // rbrace +} + +fn renderAsm( + gpa: Allocator, + ais: *Ais, + tree: Ast, + asm_node: Ast.full.Asm, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + + try renderToken(ais, tree, asm_node.ast.asm_token, .space); // asm + + if (asm_node.volatile_token) |volatile_token| { + try renderToken(ais, tree, volatile_token, .space); // volatile + try renderToken(ais, tree, volatile_token + 1, .none); // lparen + } else { + try renderToken(ais, tree, asm_node.ast.asm_token + 1, .none); // lparen + } + + if (asm_node.ast.items.len == 0) { + ais.pushIndent(); + if (asm_node.first_clobber) |first_clobber| { + // asm ("foo" ::: "a", "b") + // asm ("foo" ::: "a", "b",) + try renderExpression(gpa, ais, tree, asm_node.ast.template, .space); + // Render the three colons. + try renderToken(ais, tree, first_clobber - 3, .none); + try renderToken(ais, tree, first_clobber - 2, .none); + try renderToken(ais, tree, first_clobber - 1, .space); + + var tok_i = first_clobber; + while (true) : (tok_i += 1) { + try renderToken(ais, tree, tok_i, .none); + tok_i += 1; + switch (token_tags[tok_i]) { + .r_paren => { + ais.popIndent(); + return renderToken(ais, tree, tok_i, space); + }, + .comma => { + if (token_tags[tok_i + 1] == .r_paren) { + ais.popIndent(); + return renderToken(ais, tree, tok_i + 1, space); + } else { + try renderToken(ais, tree, tok_i, .space); + } + }, + else => unreachable, + } + } + } else { + // asm ("foo") + try renderExpression(gpa, ais, tree, asm_node.ast.template, .none); + ais.popIndent(); + return renderToken(ais, tree, asm_node.ast.rparen, space); // rparen + } + } + + ais.pushIndent(); + try renderExpression(gpa, ais, tree, asm_node.ast.template, .newline); + ais.setIndentDelta(asm_indent_delta); + const colon1 = tree.lastToken(asm_node.ast.template) + 1; + + const colon2 = if (asm_node.outputs.len == 0) colon2: { + try renderToken(ais, tree, colon1, .newline); // : + break :colon2 colon1 + 1; + } else colon2: { + try renderToken(ais, tree, colon1, .space); // : + + ais.pushIndent(); + for (asm_node.outputs, 0..) |asm_output, i| { + if (i + 1 < asm_node.outputs.len) { + const next_asm_output = asm_node.outputs[i + 1]; + try renderAsmOutput(gpa, ais, tree, asm_output, .none); + + const comma = tree.firstToken(next_asm_output) - 1; + try renderToken(ais, tree, comma, .newline); // , + try renderExtraNewlineToken(ais, tree, tree.firstToken(next_asm_output)); + } else if (asm_node.inputs.len == 0 and asm_node.first_clobber == null) { + try renderAsmOutput(gpa, ais, tree, asm_output, .comma); + ais.popIndent(); + ais.setIndentDelta(indent_delta); + ais.popIndent(); + return renderToken(ais, tree, asm_node.ast.rparen, space); // rparen + } else { + try renderAsmOutput(gpa, ais, tree, asm_output, .comma); + const comma_or_colon = tree.lastToken(asm_output) + 1; + ais.popIndent(); + break :colon2 switch (token_tags[comma_or_colon]) { + .comma => comma_or_colon + 1, + else => comma_or_colon, + }; + } + } else unreachable; + }; + + const colon3 = if (asm_node.inputs.len == 0) colon3: { + try renderToken(ais, tree, colon2, .newline); // : + break :colon3 colon2 + 1; + } else colon3: { + try renderToken(ais, tree, colon2, .space); // : + ais.pushIndent(); + for (asm_node.inputs, 0..) |asm_input, i| { + if (i + 1 < asm_node.inputs.len) { + const next_asm_input = asm_node.inputs[i + 1]; + try renderAsmInput(gpa, ais, tree, asm_input, .none); + + const first_token = tree.firstToken(next_asm_input); + try renderToken(ais, tree, first_token - 1, .newline); // , + try renderExtraNewlineToken(ais, tree, first_token); + } else if (asm_node.first_clobber == null) { + try renderAsmInput(gpa, ais, tree, asm_input, .comma); + ais.popIndent(); + ais.setIndentDelta(indent_delta); + ais.popIndent(); + return renderToken(ais, tree, asm_node.ast.rparen, space); // rparen + } else { + try renderAsmInput(gpa, ais, tree, asm_input, .comma); + const comma_or_colon = tree.lastToken(asm_input) + 1; + ais.popIndent(); + break :colon3 switch (token_tags[comma_or_colon]) { + .comma => comma_or_colon + 1, + else => comma_or_colon, + }; + } + } + unreachable; + }; + + try renderToken(ais, tree, colon3, .space); // : + const first_clobber = asm_node.first_clobber.?; + var tok_i = first_clobber; + while (true) { + switch (token_tags[tok_i + 1]) { + .r_paren => { + ais.setIndentDelta(indent_delta); + ais.popIndent(); + try renderToken(ais, tree, tok_i, .newline); + return renderToken(ais, tree, tok_i + 1, space); + }, + .comma => { + switch (token_tags[tok_i + 2]) { + .r_paren => { + ais.setIndentDelta(indent_delta); + ais.popIndent(); + try renderToken(ais, tree, tok_i, .newline); + return renderToken(ais, tree, tok_i + 2, space); + }, + else => { + try renderToken(ais, tree, tok_i, .none); + try renderToken(ais, tree, tok_i + 1, .space); + tok_i += 2; + }, + } + }, + else => unreachable, + } + } +} + +fn renderCall( + gpa: Allocator, + ais: *Ais, + tree: Ast, + call: Ast.full.Call, + space: Space, +) Error!void { + if (call.async_token) |async_token| { + try renderToken(ais, tree, async_token, .space); + } + try renderExpression(gpa, ais, tree, call.ast.fn_expr, .none); + try renderParamList(gpa, ais, tree, call.ast.lparen, call.ast.params, space); +} + +fn renderParamList( + gpa: Allocator, + ais: *Ais, + tree: Ast, + lparen: Ast.TokenIndex, + params: []const Ast.Node.Index, + space: Space, +) Error!void { + const token_tags = tree.tokens.items(.tag); + + if (params.len == 0) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, lparen, .none); + ais.popIndent(); + return renderToken(ais, tree, lparen + 1, space); // ) + } + + const last_param = params[params.len - 1]; + const after_last_param_tok = tree.lastToken(last_param) + 1; + if (token_tags[after_last_param_tok] == .comma) { + ais.pushIndentNextLine(); + try renderToken(ais, tree, lparen, .newline); // ( + for (params, 0..) |param_node, i| { + if (i + 1 < params.len) { + try renderExpression(gpa, ais, tree, param_node, .none); + + // Unindent the comma for multiline string literals. + const is_multiline_string = + token_tags[tree.firstToken(param_node)] == .multiline_string_literal_line; + if (is_multiline_string) ais.popIndent(); + + const comma = tree.lastToken(param_node) + 1; + try renderToken(ais, tree, comma, .newline); // , + + if (is_multiline_string) ais.pushIndent(); + + try renderExtraNewline(ais, tree, params[i + 1]); + } else { + try renderExpression(gpa, ais, tree, param_node, .comma); + } + } + ais.popIndent(); + return renderToken(ais, tree, after_last_param_tok + 1, space); // ) + } + + try renderToken(ais, tree, lparen, .none); // ( + + for (params, 0..) |param_node, i| { + const first_param_token = tree.firstToken(param_node); + if (token_tags[first_param_token] == .multiline_string_literal_line or + hasSameLineComment(tree, first_param_token - 1)) + { + ais.pushIndentOneShot(); + } + try renderExpression(gpa, ais, tree, param_node, .none); + + if (i + 1 < params.len) { + const comma = tree.lastToken(param_node) + 1; + const next_multiline_string = + token_tags[tree.firstToken(params[i + 1])] == .multiline_string_literal_line; + const comma_space: Space = if (next_multiline_string) .none else .space; + try renderToken(ais, tree, comma, comma_space); + } + } + + return renderToken(ais, tree, after_last_param_tok, space); // ) +} + +/// Renders the given expression indented, popping the indent before rendering +/// any following line comments +fn renderExpressionIndented(gpa: Allocator, ais: *Ais, tree: Ast, node: Ast.Node.Index, space: Space) Error!void { + const token_starts = tree.tokens.items(.start); + const token_tags = tree.tokens.items(.tag); + + ais.pushIndent(); + + var last_token = tree.lastToken(node); + const punctuation = switch (space) { + .none, .space, .newline, .skip => false, + .comma => true, + .comma_space => token_tags[last_token + 1] == .comma, + .semicolon => token_tags[last_token + 1] == .semicolon, + }; + + try renderExpression(gpa, ais, tree, node, if (punctuation) .none else .skip); + + switch (space) { + .none, .space, .newline, .skip => {}, + .comma => { + if (token_tags[last_token + 1] == .comma) { + try renderToken(ais, tree, last_token + 1, .skip); + last_token += 1; + } else { + try ais.writer().writeByte(','); + } + }, + .comma_space => if (token_tags[last_token + 1] == .comma) { + try renderToken(ais, tree, last_token + 1, .skip); + last_token += 1; + }, + .semicolon => if (token_tags[last_token + 1] == .semicolon) { + try renderToken(ais, tree, last_token + 1, .skip); + last_token += 1; + }, + } + + ais.popIndent(); + + if (space == .skip) return; + + const comment_start = token_starts[last_token] + tokenSliceForRender(tree, last_token).len; + const comment = try renderComments(ais, tree, comment_start, token_starts[last_token + 1]); + + if (!comment) switch (space) { + .none => {}, + .space, + .comma_space, + => try ais.writer().writeByte(' '), + .newline, + .comma, + .semicolon, + => try ais.insertNewline(), + .skip => unreachable, + }; +} + +/// Render an expression, and the comma that follows it, if it is present in the source. +/// If a comma is present, and `space` is `Space.comma`, render only a single comma. +fn renderExpressionComma(gpa: Allocator, ais: *Ais, tree: Ast, node: Ast.Node.Index, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + const maybe_comma = tree.lastToken(node) + 1; + if (token_tags[maybe_comma] == .comma and space != .comma) { + try renderExpression(gpa, ais, tree, node, .none); + return renderToken(ais, tree, maybe_comma, space); + } else { + return renderExpression(gpa, ais, tree, node, space); + } +} + +/// Render a token, and the comma that follows it, if it is present in the source. +/// If a comma is present, and `space` is `Space.comma`, render only a single comma. +fn renderTokenComma(ais: *Ais, tree: Ast, token: Ast.TokenIndex, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + const maybe_comma = token + 1; + if (token_tags[maybe_comma] == .comma and space != .comma) { + try renderToken(ais, tree, token, .none); + return renderToken(ais, tree, maybe_comma, space); + } else { + return renderToken(ais, tree, token, space); + } +} + +/// Render an identifier, and the comma that follows it, if it is present in the source. +/// If a comma is present, and `space` is `Space.comma`, render only a single comma. +fn renderIdentifierComma(ais: *Ais, tree: Ast, token: Ast.TokenIndex, space: Space, quote: QuoteBehavior) Error!void { + const token_tags = tree.tokens.items(.tag); + const maybe_comma = token + 1; + if (token_tags[maybe_comma] == .comma and space != .comma) { + try renderIdentifier(ais, tree, token, .none, quote); + return renderToken(ais, tree, maybe_comma, space); + } else { + return renderIdentifier(ais, tree, token, space, quote); + } +} + +const Space = enum { + /// Output the token lexeme only. + none, + /// Output the token lexeme followed by a single space. + space, + /// Output the token lexeme followed by a newline. + newline, + /// If the next token is a comma, render it as well. If not, insert one. + /// In either case, a newline will be inserted afterwards. + comma, + /// Additionally consume the next token if it is a comma. + /// In either case, a space will be inserted afterwards. + comma_space, + /// Additionally consume the next token if it is a semicolon. + /// In either case, a newline will be inserted afterwards. + semicolon, + /// Skip rendering whitespace and comments. If this is used, the caller + /// *must* handle whitespace and comments manually. + skip, +}; + +fn renderToken(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Space) Error!void { + const lexeme = tokenSliceForRender(tree, token_index); + try ais.writer().writeAll(lexeme); + try renderSpace(ais, tree, token_index, lexeme.len, space); +} + +fn renderSpace(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, lexeme_len: usize, space: Space) Error!void { + const token_tags = tree.tokens.items(.tag); + const token_starts = tree.tokens.items(.start); + + const token_start = token_starts[token_index]; + + if (space == .skip) return; + + if (space == .comma and token_tags[token_index + 1] != .comma) { + try ais.writer().writeByte(','); + } + + const comment = try renderComments(ais, tree, token_start + lexeme_len, token_starts[token_index + 1]); + switch (space) { + .none => {}, + .space => if (!comment) try ais.writer().writeByte(' '), + .newline => if (!comment) try ais.insertNewline(), + + .comma => if (token_tags[token_index + 1] == .comma) { + try renderToken(ais, tree, token_index + 1, .newline); + } else if (!comment) { + try ais.insertNewline(); + }, + + .comma_space => if (token_tags[token_index + 1] == .comma) { + try renderToken(ais, tree, token_index + 1, .space); + } else if (!comment) { + try ais.writer().writeByte(' '); + }, + + .semicolon => if (token_tags[token_index + 1] == .semicolon) { + try renderToken(ais, tree, token_index + 1, .newline); + } else if (!comment) { + try ais.insertNewline(); + }, + + .skip => unreachable, + } +} + +const QuoteBehavior = enum { + preserve_when_shadowing, + eagerly_unquote, + eagerly_unquote_except_underscore, +}; + +fn renderIdentifier(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Space, quote: QuoteBehavior) Error!void { + const token_tags = tree.tokens.items(.tag); + assert(token_tags[token_index] == .identifier); + const lexeme = tokenSliceForRender(tree, token_index); + if (lexeme[0] != '@') { + return renderToken(ais, tree, token_index, space); + } + + assert(lexeme.len >= 3); + assert(lexeme[0] == '@'); + assert(lexeme[1] == '\"'); + assert(lexeme[lexeme.len - 1] == '\"'); + const contents = lexeme[2 .. lexeme.len - 1]; // inside the @"" quotation + + // Empty name can't be unquoted. + if (contents.len == 0) { + return renderQuotedIdentifier(ais, tree, token_index, space, false); + } + + // Special case for _ which would incorrectly be rejected by isValidId below. + if (contents.len == 1 and contents[0] == '_') switch (quote) { + .eagerly_unquote => return renderQuotedIdentifier(ais, tree, token_index, space, true), + .eagerly_unquote_except_underscore, + .preserve_when_shadowing, + => return renderQuotedIdentifier(ais, tree, token_index, space, false), + }; + + // Scan the entire name for characters that would (after un-escaping) be illegal in a symbol, + // i.e. contents don't match: [A-Za-z_][A-Za-z0-9_]* + var contents_i: usize = 0; + while (contents_i < contents.len) { + switch (contents[contents_i]) { + '0'...'9' => if (contents_i == 0) return renderQuotedIdentifier(ais, tree, token_index, space, false), + 'A'...'Z', 'a'...'z', '_' => {}, + '\\' => { + var esc_offset = contents_i; + const res = std.zig.string_literal.parseEscapeSequence(contents, &esc_offset); + switch (res) { + .success => |char| switch (char) { + '0'...'9' => if (contents_i == 0) return renderQuotedIdentifier(ais, tree, token_index, space, false), + 'A'...'Z', 'a'...'z', '_' => {}, + else => return renderQuotedIdentifier(ais, tree, token_index, space, false), + }, + .failure => return renderQuotedIdentifier(ais, tree, token_index, space, false), + } + contents_i += esc_offset; + continue; + }, + else => return renderQuotedIdentifier(ais, tree, token_index, space, false), + } + contents_i += 1; + } + + // Read enough of the name (while un-escaping) to determine if it's a keyword or primitive. + // If it's too long to fit in this buffer, we know it's neither and quoting is unnecessary. + // If we read the whole thing, we have to do further checks. + const longest_keyword_or_primitive_len = comptime blk: { + var longest = 0; + for (primitives.names.kvs) |kv| { + if (kv.key.len > longest) longest = kv.key.len; + } + for (std.zig.Token.keywords.kvs) |kv| { + if (kv.key.len > longest) longest = kv.key.len; + } + break :blk longest; + }; + var buf: [longest_keyword_or_primitive_len]u8 = undefined; + + contents_i = 0; + var buf_i: usize = 0; + while (contents_i < contents.len and buf_i < longest_keyword_or_primitive_len) { + if (contents[contents_i] == '\\') { + const res = std.zig.string_literal.parseEscapeSequence(contents, &contents_i).success; + buf[buf_i] = @as(u8, @intCast(res)); + buf_i += 1; + } else { + buf[buf_i] = contents[contents_i]; + contents_i += 1; + buf_i += 1; + } + } + + // We read the whole thing, so it could be a keyword or primitive. + if (contents_i == contents.len) { + if (!std.zig.isValidId(buf[0..buf_i])) { + return renderQuotedIdentifier(ais, tree, token_index, space, false); + } + if (primitives.isPrimitive(buf[0..buf_i])) switch (quote) { + .eagerly_unquote, + .eagerly_unquote_except_underscore, + => return renderQuotedIdentifier(ais, tree, token_index, space, true), + .preserve_when_shadowing => return renderQuotedIdentifier(ais, tree, token_index, space, false), + }; + } + + try renderQuotedIdentifier(ais, tree, token_index, space, true); +} + +// Renders a @"" quoted identifier, normalizing escapes. +// Unnecessary escapes are un-escaped, and \u escapes are normalized to \x when they fit. +// If unquote is true, the @"" is removed and the result is a bare symbol whose validity is asserted. +fn renderQuotedIdentifier(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex, space: Space, comptime unquote: bool) !void { + const token_tags = tree.tokens.items(.tag); + assert(token_tags[token_index] == .identifier); + const lexeme = tokenSliceForRender(tree, token_index); + assert(lexeme.len >= 3 and lexeme[0] == '@'); + + if (!unquote) try ais.writer().writeAll("@\""); + const contents = lexeme[2 .. lexeme.len - 1]; + try renderIdentifierContents(ais.writer(), contents); + if (!unquote) try ais.writer().writeByte('\"'); + + try renderSpace(ais, tree, token_index, lexeme.len, space); +} + +fn renderIdentifierContents(writer: anytype, bytes: []const u8) !void { + var pos: usize = 0; + while (pos < bytes.len) { + const byte = bytes[pos]; + switch (byte) { + '\\' => { + const old_pos = pos; + const res = std.zig.string_literal.parseEscapeSequence(bytes, &pos); + const escape_sequence = bytes[old_pos..pos]; + switch (res) { + .success => |codepoint| { + if (codepoint <= 0x7f) { + const buf = [1]u8{@as(u8, @intCast(codepoint))}; + try std.fmt.format(writer, "{}", .{std.zig.fmtEscapes(&buf)}); + } else { + try writer.writeAll(escape_sequence); + } + }, + .failure => { + try writer.writeAll(escape_sequence); + }, + } + }, + 0x00...('\\' - 1), ('\\' + 1)...0x7f => { + const buf = [1]u8{byte}; + try std.fmt.format(writer, "{}", .{std.zig.fmtEscapes(&buf)}); + pos += 1; + }, + 0x80...0xff => { + try writer.writeByte(byte); + pos += 1; + }, + } + } +} + +/// Returns true if there exists a line comment between any of the tokens from +/// `start_token` to `end_token`. This is used to determine if e.g. a +/// fn_proto should be wrapped and have a trailing comma inserted even if +/// there is none in the source. +fn hasComment(tree: Ast, start_token: Ast.TokenIndex, end_token: Ast.TokenIndex) bool { + const token_starts = tree.tokens.items(.start); + + var i = start_token; + while (i < end_token) : (i += 1) { + const start = token_starts[i] + tree.tokenSlice(i).len; + const end = token_starts[i + 1]; + if (mem.indexOf(u8, tree.source[start..end], "//") != null) return true; + } + + return false; +} + +/// Returns true if there exists a multiline string literal between the start +/// of token `start_token` and the start of token `end_token`. +fn hasMultilineString(tree: Ast, start_token: Ast.TokenIndex, end_token: Ast.TokenIndex) bool { + const token_tags = tree.tokens.items(.tag); + + for (token_tags[start_token..end_token]) |tag| { + switch (tag) { + .multiline_string_literal_line => return true, + else => continue, + } + } + + return false; +} + +/// Assumes that start is the first byte past the previous token and +/// that end is the last byte before the next token. +fn renderComments(ais: *Ais, tree: Ast, start: usize, end: usize) Error!bool { + var index: usize = start; + while (mem.indexOf(u8, tree.source[index..end], "//")) |offset| { + const comment_start = index + offset; + + // If there is no newline, the comment ends with EOF + const newline_index = mem.indexOfScalar(u8, tree.source[comment_start..end], '\n'); + const newline = if (newline_index) |i| comment_start + i else null; + + const untrimmed_comment = tree.source[comment_start .. newline orelse tree.source.len]; + const trimmed_comment = mem.trimRight(u8, untrimmed_comment, &std.ascii.whitespace); + + // Don't leave any whitespace at the start of the file + if (index != 0) { + if (index == start and mem.containsAtLeast(u8, tree.source[index..comment_start], 2, "\n")) { + // Leave up to one empty line before the first comment + try ais.insertNewline(); + try ais.insertNewline(); + } else if (mem.indexOfScalar(u8, tree.source[index..comment_start], '\n') != null) { + // Respect the newline directly before the comment. + // Note: This allows an empty line between comments + try ais.insertNewline(); + } else if (index == start) { + // Otherwise if the first comment is on the same line as + // the token before it, prefix it with a single space. + try ais.writer().writeByte(' '); + } + } + + index = 1 + (newline orelse end - 1); + + const comment_content = mem.trimLeft(u8, trimmed_comment["//".len..], &std.ascii.whitespace); + if (ais.disabled_offset != null and mem.eql(u8, comment_content, "zig fmt: on")) { + // Write the source for which formatting was disabled directly + // to the underlying writer, fixing up invalid whitespace. + const disabled_source = tree.source[ais.disabled_offset.?..comment_start]; + try writeFixingWhitespace(ais.underlying_writer, disabled_source); + // Write with the canonical single space. + try ais.underlying_writer.writeAll("// zig fmt: on\n"); + ais.disabled_offset = null; + } else if (ais.disabled_offset == null and mem.eql(u8, comment_content, "zig fmt: off")) { + // Write with the canonical single space. + try ais.writer().writeAll("// zig fmt: off\n"); + ais.disabled_offset = index; + } else { + // Write the comment minus trailing whitespace. + try ais.writer().print("{s}\n", .{trimmed_comment}); + } + } + + if (index != start and mem.containsAtLeast(u8, tree.source[index - 1 .. end], 2, "\n")) { + // Don't leave any whitespace at the end of the file + if (end != tree.source.len) { + try ais.insertNewline(); + } + } + + return index != start; +} + +fn renderExtraNewline(ais: *Ais, tree: Ast, node: Ast.Node.Index) Error!void { + return renderExtraNewlineToken(ais, tree, tree.firstToken(node)); +} + +/// Check if there is an empty line immediately before the given token. If so, render it. +fn renderExtraNewlineToken(ais: *Ais, tree: Ast, token_index: Ast.TokenIndex) Error!void { + const token_starts = tree.tokens.items(.start); + const token_start = token_starts[token_index]; + if (token_start == 0) return; + const prev_token_end = if (token_index == 0) + 0 + else + token_starts[token_index - 1] + tokenSliceForRender(tree, token_index - 1).len; + + // If there is a comment present, it will handle the empty line + if (mem.indexOf(u8, tree.source[prev_token_end..token_start], "//") != null) return; + + // Iterate backwards to the end of the previous token, stopping if a + // non-whitespace character is encountered or two newlines have been found. + var i = token_start - 1; + var newlines: u2 = 0; + while (std.ascii.isWhitespace(tree.source[i])) : (i -= 1) { + if (tree.source[i] == '\n') newlines += 1; + if (newlines == 2) return ais.insertNewline(); + if (i == prev_token_end) break; + } +} + +/// end_token is the token one past the last doc comment token. This function +/// searches backwards from there. +fn renderDocComments(ais: *Ais, tree: Ast, end_token: Ast.TokenIndex) Error!void { + // Search backwards for the first doc comment. + const token_tags = tree.tokens.items(.tag); + if (end_token == 0) return; + var tok = end_token - 1; + while (token_tags[tok] == .doc_comment) { + if (tok == 0) break; + tok -= 1; + } else { + tok += 1; + } + const first_tok = tok; + if (first_tok == end_token) return; + + if (first_tok != 0) { + const prev_token_tag = token_tags[first_tok - 1]; + + // Prevent accidental use of `renderDocComments` for a function argument doc comment + assert(prev_token_tag != .l_paren); + + if (prev_token_tag != .l_brace) { + try renderExtraNewlineToken(ais, tree, first_tok); + } + } + + while (token_tags[tok] == .doc_comment) : (tok += 1) { + try renderToken(ais, tree, tok, .newline); + } +} + +/// start_token is first container doc comment token. +fn renderContainerDocComments(ais: *Ais, tree: Ast, start_token: Ast.TokenIndex) Error!void { + const token_tags = tree.tokens.items(.tag); + var tok = start_token; + while (token_tags[tok] == .container_doc_comment) : (tok += 1) { + try renderToken(ais, tree, tok, .newline); + } + // Render extra newline if there is one between final container doc comment and + // the next token. If the next token is a doc comment, that code path + // will have its own logic to insert a newline. + if (token_tags[tok] != .doc_comment) { + try renderExtraNewlineToken(ais, tree, tok); + } +} + +fn tokenSliceForRender(tree: Ast, token_index: Ast.TokenIndex) []const u8 { + var ret = tree.tokenSlice(token_index); + switch (tree.tokens.items(.tag)[token_index]) { + .multiline_string_literal_line => { + if (ret[ret.len - 1] == '\n') ret.len -= 1; + }, + .container_doc_comment, .doc_comment => { + ret = mem.trimRight(u8, ret, &std.ascii.whitespace); + }, + else => {}, + } + return ret; +} + +fn hasSameLineComment(tree: Ast, token_index: Ast.TokenIndex) bool { + const token_starts = tree.tokens.items(.start); + const between_source = tree.source[token_starts[token_index]..token_starts[token_index + 1]]; + for (between_source) |byte| switch (byte) { + '\n' => return false, + '/' => return true, + else => continue, + }; + return false; +} + +/// Returns `true` if and only if there are any tokens or line comments between +/// start_token and end_token. +fn anythingBetween(tree: Ast, start_token: Ast.TokenIndex, end_token: Ast.TokenIndex) bool { + if (start_token + 1 != end_token) return true; + const token_starts = tree.tokens.items(.start); + const between_source = tree.source[token_starts[start_token]..token_starts[start_token + 1]]; + for (between_source) |byte| switch (byte) { + '/' => return true, + else => continue, + }; + return false; +} + +fn writeFixingWhitespace(writer: std.ArrayList(u8).Writer, slice: []const u8) Error!void { + for (slice) |byte| switch (byte) { + '\t' => try writer.writeAll(" " ** 4), + '\r' => {}, + else => try writer.writeByte(byte), + }; +} + +fn nodeIsBlock(tag: Ast.Node.Tag) bool { + return switch (tag) { + .block, + .block_semicolon, + .block_two, + .block_two_semicolon, + => true, + else => false, + }; +} + +fn nodeIsIfForWhileSwitch(tag: Ast.Node.Tag) bool { + return switch (tag) { + .@"if", + .if_simple, + .@"for", + .for_simple, + .@"while", + .while_simple, + .while_cont, + .@"switch", + .switch_comma, + => true, + else => false, + }; +} + +fn nodeCausesSliceOpSpace(tag: Ast.Node.Tag) bool { + return switch (tag) { + .@"catch", + .add, + .add_wrap, + .array_cat, + .array_mult, + .assign, + .assign_bit_and, + .assign_bit_or, + .assign_shl, + .assign_shr, + .assign_bit_xor, + .assign_div, + .assign_sub, + .assign_sub_wrap, + .assign_mod, + .assign_add, + .assign_add_wrap, + .assign_mul, + .assign_mul_wrap, + .bang_equal, + .bit_and, + .bit_or, + .shl, + .shr, + .bit_xor, + .bool_and, + .bool_or, + .div, + .equal_equal, + .error_union, + .greater_or_equal, + .greater_than, + .less_or_equal, + .less_than, + .merge_error_sets, + .mod, + .mul, + .mul_wrap, + .sub, + .sub_wrap, + .@"orelse", + => true, + + else => false, + }; +} + +// Returns the number of nodes in `exprs` that are on the same line as `rtoken`. +fn rowSize(tree: Ast, exprs: []const Ast.Node.Index, rtoken: Ast.TokenIndex) usize { + const token_tags = tree.tokens.items(.tag); + + const first_token = tree.firstToken(exprs[0]); + if (tree.tokensOnSameLine(first_token, rtoken)) { + const maybe_comma = rtoken - 1; + if (token_tags[maybe_comma] == .comma) + return 1; + return exprs.len; // no newlines + } + + var count: usize = 1; + for (exprs, 0..) |expr, i| { + if (i + 1 < exprs.len) { + const expr_last_token = tree.lastToken(expr) + 1; + if (!tree.tokensOnSameLine(expr_last_token, tree.firstToken(exprs[i + 1]))) return count; + count += 1; + } else { + return count; + } + } + unreachable; +} + +/// Automatically inserts indentation of written data by keeping +/// track of the current indentation level +fn AutoIndentingStream(comptime UnderlyingWriter: type) type { + return struct { + const Self = @This(); + pub const WriteError = UnderlyingWriter.Error; + pub const Writer = std.io.Writer(*Self, WriteError, write); + + underlying_writer: UnderlyingWriter, + + /// Offset into the source at which formatting has been disabled with + /// a `zig fmt: off` comment. + /// + /// If non-null, the AutoIndentingStream will not write any bytes + /// to the underlying writer. It will however continue to track the + /// indentation level. + disabled_offset: ?usize = null, + + indent_count: usize = 0, + indent_delta: usize, + current_line_empty: bool = true, + /// automatically popped when applied + indent_one_shot_count: usize = 0, + /// the most recently applied indent + applied_indent: usize = 0, + /// not used until the next line + indent_next_line: usize = 0, + + pub fn writer(self: *Self) Writer { + return .{ .context = self }; + } + + pub fn write(self: *Self, bytes: []const u8) WriteError!usize { + if (bytes.len == 0) + return @as(usize, 0); + + try self.applyIndent(); + return self.writeNoIndent(bytes); + } + + // Change the indent delta without changing the final indentation level + pub fn setIndentDelta(self: *Self, new_indent_delta: usize) void { + if (self.indent_delta == new_indent_delta) { + return; + } else if (self.indent_delta > new_indent_delta) { + assert(self.indent_delta % new_indent_delta == 0); + self.indent_count = self.indent_count * (self.indent_delta / new_indent_delta); + } else { + // assert that the current indentation (in spaces) in a multiple of the new delta + assert((self.indent_count * self.indent_delta) % new_indent_delta == 0); + self.indent_count = self.indent_count / (new_indent_delta / self.indent_delta); + } + self.indent_delta = new_indent_delta; + } + + fn writeNoIndent(self: *Self, bytes: []const u8) WriteError!usize { + if (bytes.len == 0) + return @as(usize, 0); + + if (self.disabled_offset == null) try self.underlying_writer.writeAll(bytes); + if (bytes[bytes.len - 1] == '\n') + self.resetLine(); + return bytes.len; + } + + pub fn insertNewline(self: *Self) WriteError!void { + _ = try self.writeNoIndent("\n"); + } + + fn resetLine(self: *Self) void { + self.current_line_empty = true; + self.indent_next_line = 0; + } + + /// Insert a newline unless the current line is blank + pub fn maybeInsertNewline(self: *Self) WriteError!void { + if (!self.current_line_empty) + try self.insertNewline(); + } + + /// Push default indentation + /// Doesn't actually write any indentation. + /// Just primes the stream to be able to write the correct indentation if it needs to. + pub fn pushIndent(self: *Self) void { + self.indent_count += 1; + } + + /// Push an indent that is automatically popped after being applied + pub fn pushIndentOneShot(self: *Self) void { + self.indent_one_shot_count += 1; + self.pushIndent(); + } + + /// Turns all one-shot indents into regular indents + /// Returns number of indents that must now be manually popped + pub fn lockOneShotIndent(self: *Self) usize { + var locked_count = self.indent_one_shot_count; + self.indent_one_shot_count = 0; + return locked_count; + } + + /// Push an indent that should not take effect until the next line + pub fn pushIndentNextLine(self: *Self) void { + self.indent_next_line += 1; + self.pushIndent(); + } + + pub fn popIndent(self: *Self) void { + assert(self.indent_count != 0); + self.indent_count -= 1; + + if (self.indent_next_line > 0) + self.indent_next_line -= 1; + } + + /// Writes ' ' bytes if the current line is empty + fn applyIndent(self: *Self) WriteError!void { + const current_indent = self.currentIndent(); + if (self.current_line_empty and current_indent > 0) { + if (self.disabled_offset == null) { + try self.underlying_writer.writeByteNTimes(' ', current_indent); + } + self.applied_indent = current_indent; + } + + self.indent_count -= self.indent_one_shot_count; + self.indent_one_shot_count = 0; + self.current_line_empty = false; + } + + /// Checks to see if the most recent indentation exceeds the currently pushed indents + pub fn isLineOverIndented(self: *Self) bool { + if (self.current_line_empty) return false; + return self.applied_indent > self.currentIndent(); + } + + fn currentIndent(self: *Self) usize { + var indent_current: usize = 0; + if (self.indent_count > 0) { + const indent_count = self.indent_count - self.indent_next_line; + indent_current = indent_count * self.indent_delta; + } + return indent_current; + } + }; +} diff --git a/src/stage2/tokenizer.zig b/src/stage2/tokenizer.zig new file mode 100644 index 0000000000..88d3cbe4e8 --- /dev/null +++ b/src/stage2/tokenizer.zig @@ -0,0 +1,1930 @@ +const std = @import("std"); + +pub const Token = struct { + tag: Tag, + loc: Loc, + + pub const Loc = struct { + start: usize, + end: usize, + }; + + pub const keywords = std.ComptimeStringMap(Tag, .{ + .{ "addrspace", .keyword_addrspace }, + .{ "align", .keyword_align }, + .{ "allowzero", .keyword_allowzero }, + .{ "and", .keyword_and }, + .{ "anyframe", .keyword_anyframe }, + .{ "anytype", .keyword_anytype }, + .{ "asm", .keyword_asm }, + .{ "async", .keyword_async }, + .{ "await", .keyword_await }, + .{ "break", .keyword_break }, + .{ "callconv", .keyword_callconv }, + .{ "catch", .keyword_catch }, + .{ "comptime", .keyword_comptime }, + .{ "const", .keyword_const }, + .{ "continue", .keyword_continue }, + .{ "defer", .keyword_defer }, + .{ "else", .keyword_else }, + .{ "enum", .keyword_enum }, + .{ "errdefer", .keyword_errdefer }, + .{ "error", .keyword_error }, + .{ "export", .keyword_export }, + .{ "extern", .keyword_extern }, + .{ "fn", .keyword_fn }, + .{ "for", .keyword_for }, + .{ "if", .keyword_if }, + .{ "inline", .keyword_inline }, + .{ "noalias", .keyword_noalias }, + .{ "noinline", .keyword_noinline }, + .{ "nosuspend", .keyword_nosuspend }, + .{ "opaque", .keyword_opaque }, + .{ "or", .keyword_or }, + .{ "orelse", .keyword_orelse }, + .{ "packed", .keyword_packed }, + .{ "pub", .keyword_pub }, + .{ "resume", .keyword_resume }, + .{ "return", .keyword_return }, + .{ "linksection", .keyword_linksection }, + .{ "struct", .keyword_struct }, + .{ "suspend", .keyword_suspend }, + .{ "switch", .keyword_switch }, + .{ "test", .keyword_test }, + .{ "threadlocal", .keyword_threadlocal }, + .{ "try", .keyword_try }, + .{ "union", .keyword_union }, + .{ "unreachable", .keyword_unreachable }, + .{ "usingnamespace", .keyword_usingnamespace }, + .{ "var", .keyword_var }, + .{ "volatile", .keyword_volatile }, + .{ "while", .keyword_while }, + }); + + pub fn getKeyword(bytes: []const u8) ?Tag { + return keywords.get(bytes); + } + + pub const Tag = enum { + invalid, + invalid_periodasterisks, + identifier, + string_literal, + multiline_string_literal_line, + char_literal, + eof, + builtin, + bang, + pipe, + pipe_pipe, + pipe_equal, + equal, + equal_equal, + equal_angle_bracket_right, + bang_equal, + l_paren, + r_paren, + semicolon, + percent, + percent_equal, + l_brace, + r_brace, + l_bracket, + r_bracket, + period, + period_asterisk, + ellipsis2, + ellipsis3, + caret, + caret_equal, + plus, + plus_plus, + plus_equal, + plus_percent, + plus_percent_equal, + plus_pipe, + plus_pipe_equal, + minus, + minus_equal, + minus_percent, + minus_percent_equal, + minus_pipe, + minus_pipe_equal, + asterisk, + asterisk_equal, + asterisk_asterisk, + asterisk_percent, + asterisk_percent_equal, + asterisk_pipe, + asterisk_pipe_equal, + arrow, + colon, + slash, + slash_equal, + comma, + ampersand, + ampersand_equal, + question_mark, + angle_bracket_left, + angle_bracket_left_equal, + angle_bracket_angle_bracket_left, + angle_bracket_angle_bracket_left_equal, + angle_bracket_angle_bracket_left_pipe, + angle_bracket_angle_bracket_left_pipe_equal, + angle_bracket_right, + angle_bracket_right_equal, + angle_bracket_angle_bracket_right, + angle_bracket_angle_bracket_right_equal, + tilde, + number_literal, + doc_comment, + container_doc_comment, + keyword_addrspace, + keyword_align, + keyword_allowzero, + keyword_and, + keyword_anyframe, + keyword_anytype, + keyword_asm, + keyword_async, + keyword_await, + keyword_break, + keyword_callconv, + keyword_catch, + keyword_comptime, + keyword_const, + keyword_continue, + keyword_defer, + keyword_else, + keyword_enum, + keyword_errdefer, + keyword_error, + keyword_export, + keyword_extern, + keyword_fn, + keyword_for, + keyword_if, + keyword_inline, + keyword_noalias, + keyword_noinline, + keyword_nosuspend, + keyword_opaque, + keyword_or, + keyword_orelse, + keyword_packed, + keyword_pub, + keyword_resume, + keyword_return, + keyword_linksection, + keyword_struct, + keyword_suspend, + keyword_switch, + keyword_test, + keyword_threadlocal, + keyword_try, + keyword_union, + keyword_unreachable, + keyword_usingnamespace, + keyword_var, + keyword_volatile, + keyword_while, + + pub fn lexeme(tag: Tag) ?[]const u8 { + return switch (tag) { + .invalid, + .identifier, + .string_literal, + .multiline_string_literal_line, + .char_literal, + .eof, + .builtin, + .number_literal, + .doc_comment, + .container_doc_comment, + => null, + + .invalid_periodasterisks => ".**", + .bang => "!", + .pipe => "|", + .pipe_pipe => "||", + .pipe_equal => "|=", + .equal => "=", + .equal_equal => "==", + .equal_angle_bracket_right => "=>", + .bang_equal => "!=", + .l_paren => "(", + .r_paren => ")", + .semicolon => ";", + .percent => "%", + .percent_equal => "%=", + .l_brace => "{", + .r_brace => "}", + .l_bracket => "[", + .r_bracket => "]", + .period => ".", + .period_asterisk => ".*", + .ellipsis2 => "..", + .ellipsis3 => "...", + .caret => "^", + .caret_equal => "^=", + .plus => "+", + .plus_plus => "++", + .plus_equal => "+=", + .plus_percent => "+%", + .plus_percent_equal => "+%=", + .plus_pipe => "+|", + .plus_pipe_equal => "+|=", + .minus => "-", + .minus_equal => "-=", + .minus_percent => "-%", + .minus_percent_equal => "-%=", + .minus_pipe => "-|", + .minus_pipe_equal => "-|=", + .asterisk => "*", + .asterisk_equal => "*=", + .asterisk_asterisk => "**", + .asterisk_percent => "*%", + .asterisk_percent_equal => "*%=", + .asterisk_pipe => "*|", + .asterisk_pipe_equal => "*|=", + .arrow => "->", + .colon => ":", + .slash => "/", + .slash_equal => "/=", + .comma => ",", + .ampersand => "&", + .ampersand_equal => "&=", + .question_mark => "?", + .angle_bracket_left => "<", + .angle_bracket_left_equal => "<=", + .angle_bracket_angle_bracket_left => "<<", + .angle_bracket_angle_bracket_left_equal => "<<=", + .angle_bracket_angle_bracket_left_pipe => "<<|", + .angle_bracket_angle_bracket_left_pipe_equal => "<<|=", + .angle_bracket_right => ">", + .angle_bracket_right_equal => ">=", + .angle_bracket_angle_bracket_right => ">>", + .angle_bracket_angle_bracket_right_equal => ">>=", + .tilde => "~", + .keyword_addrspace => "addrspace", + .keyword_align => "align", + .keyword_allowzero => "allowzero", + .keyword_and => "and", + .keyword_anyframe => "anyframe", + .keyword_anytype => "anytype", + .keyword_asm => "asm", + .keyword_async => "async", + .keyword_await => "await", + .keyword_break => "break", + .keyword_callconv => "callconv", + .keyword_catch => "catch", + .keyword_comptime => "comptime", + .keyword_const => "const", + .keyword_continue => "continue", + .keyword_defer => "defer", + .keyword_else => "else", + .keyword_enum => "enum", + .keyword_errdefer => "errdefer", + .keyword_error => "error", + .keyword_export => "export", + .keyword_extern => "extern", + .keyword_fn => "fn", + .keyword_for => "for", + .keyword_if => "if", + .keyword_inline => "inline", + .keyword_noalias => "noalias", + .keyword_noinline => "noinline", + .keyword_nosuspend => "nosuspend", + .keyword_opaque => "opaque", + .keyword_or => "or", + .keyword_orelse => "orelse", + .keyword_packed => "packed", + .keyword_pub => "pub", + .keyword_resume => "resume", + .keyword_return => "return", + .keyword_linksection => "linksection", + .keyword_struct => "struct", + .keyword_suspend => "suspend", + .keyword_switch => "switch", + .keyword_test => "test", + .keyword_threadlocal => "threadlocal", + .keyword_try => "try", + .keyword_union => "union", + .keyword_unreachable => "unreachable", + .keyword_usingnamespace => "usingnamespace", + .keyword_var => "var", + .keyword_volatile => "volatile", + .keyword_while => "while", + }; + } + + pub fn symbol(tag: Tag) []const u8 { + return tag.lexeme() orelse switch (tag) { + .invalid => "invalid bytes", + .identifier => "an identifier", + .string_literal, .multiline_string_literal_line => "a string literal", + .char_literal => "a character literal", + .eof => "EOF", + .builtin => "a builtin function", + .number_literal => "a number literal", + .doc_comment, .container_doc_comment => "a document comment", + else => unreachable, + }; + } + }; +}; + +pub const Tokenizer = struct { + buffer: [:0]const u8, + index: usize, + pending_invalid_token: ?Token, + + /// For debugging purposes + pub fn dump(self: *Tokenizer, token: *const Token) void { + std.debug.print("{s} \"{s}\"\n", .{ @tagName(token.tag), self.buffer[token.loc.start..token.loc.end] }); + } + + pub fn init(buffer: [:0]const u8) Tokenizer { + // Skip the UTF-8 BOM if present + const src_start: usize = if (std.mem.startsWith(u8, buffer, "\xEF\xBB\xBF")) 3 else 0; + return Tokenizer{ + .buffer = buffer, + .index = src_start, + .pending_invalid_token = null, + }; + } + + const State = enum { + start, + identifier, + builtin, + string_literal, + string_literal_backslash, + multiline_string_literal_line, + char_literal, + char_literal_backslash, + char_literal_hex_escape, + char_literal_unicode_escape_saw_u, + char_literal_unicode_escape, + char_literal_unicode_invalid, + char_literal_unicode, + char_literal_end, + backslash, + equal, + bang, + pipe, + minus, + minus_percent, + minus_pipe, + asterisk, + asterisk_percent, + asterisk_pipe, + slash, + line_comment_start, + line_comment, + doc_comment_start, + doc_comment, + int, + int_exponent, + int_period, + float, + float_exponent, + ampersand, + caret, + percent, + plus, + plus_percent, + plus_pipe, + angle_bracket_left, + angle_bracket_angle_bracket_left, + angle_bracket_angle_bracket_left_pipe, + angle_bracket_right, + angle_bracket_angle_bracket_right, + period, + period_2, + period_asterisk, + saw_at_sign, + }; + + /// This is a workaround to the fact that the tokenizer can queue up + /// 'pending_invalid_token's when parsing literals, which means that we need + /// to scan from the start of the current line to find a matching tag - just + /// in case it was an invalid character generated during literal + /// tokenization. Ideally this processing of this would be pushed to the AST + /// parser or another later stage, both to give more useful error messages + /// with that extra context and in order to be able to remove this + /// workaround. + pub fn findTagAtCurrentIndex(self: *Tokenizer, tag: Token.Tag) Token { + if (tag == .invalid) { + const target_index = self.index; + var starting_index = target_index; + while (starting_index > 0) { + if (self.buffer[starting_index] == '\n') { + break; + } + starting_index -= 1; + } + + self.index = starting_index; + while (self.index <= target_index or self.pending_invalid_token != null) { + const result = self.next(); + if (result.loc.start == target_index and result.tag == tag) { + return result; + } + } + unreachable; + } else { + return self.next(); + } + } + + pub fn next(self: *Tokenizer) Token { + if (self.pending_invalid_token) |token| { + self.pending_invalid_token = null; + return token; + } + var state: State = .start; + var result = Token{ + .tag = .eof, + .loc = .{ + .start = self.index, + .end = undefined, + }, + }; + var seen_escape_digits: usize = undefined; + var remaining_code_units: usize = undefined; + while (true) : (self.index += 1) { + const c = self.buffer[self.index]; + switch (state) { + .start => switch (c) { + 0 => { + if (self.index != self.buffer.len) { + result.tag = .invalid; + result.loc.start = self.index; + self.index += 1; + result.loc.end = self.index; + return result; + } + break; + }, + ' ', '\n', '\t', '\r' => { + result.loc.start = self.index + 1; + }, + '"' => { + state = .string_literal; + result.tag = .string_literal; + }, + '\'' => { + state = .char_literal; + }, + 'a'...'z', 'A'...'Z', '_' => { + state = .identifier; + result.tag = .identifier; + }, + '@' => { + state = .saw_at_sign; + }, + '=' => { + state = .equal; + }, + '!' => { + state = .bang; + }, + '|' => { + state = .pipe; + }, + '(' => { + result.tag = .l_paren; + self.index += 1; + break; + }, + ')' => { + result.tag = .r_paren; + self.index += 1; + break; + }, + '[' => { + result.tag = .l_bracket; + self.index += 1; + break; + }, + ']' => { + result.tag = .r_bracket; + self.index += 1; + break; + }, + ';' => { + result.tag = .semicolon; + self.index += 1; + break; + }, + ',' => { + result.tag = .comma; + self.index += 1; + break; + }, + '?' => { + result.tag = .question_mark; + self.index += 1; + break; + }, + ':' => { + result.tag = .colon; + self.index += 1; + break; + }, + '%' => { + state = .percent; + }, + '*' => { + state = .asterisk; + }, + '+' => { + state = .plus; + }, + '<' => { + state = .angle_bracket_left; + }, + '>' => { + state = .angle_bracket_right; + }, + '^' => { + state = .caret; + }, + '\\' => { + state = .backslash; + result.tag = .multiline_string_literal_line; + }, + '{' => { + result.tag = .l_brace; + self.index += 1; + break; + }, + '}' => { + result.tag = .r_brace; + self.index += 1; + break; + }, + '~' => { + result.tag = .tilde; + self.index += 1; + break; + }, + '.' => { + state = .period; + }, + '-' => { + state = .minus; + }, + '/' => { + state = .slash; + }, + '&' => { + state = .ampersand; + }, + '0'...'9' => { + state = .int; + result.tag = .number_literal; + }, + else => { + result.tag = .invalid; + result.loc.end = self.index; + self.index += 1; + return result; + }, + }, + + .saw_at_sign => switch (c) { + '"' => { + result.tag = .identifier; + state = .string_literal; + }, + 'a'...'z', 'A'...'Z', '_' => { + state = .builtin; + result.tag = .builtin; + }, + else => { + result.tag = .invalid; + break; + }, + }, + + .ampersand => switch (c) { + '=' => { + result.tag = .ampersand_equal; + self.index += 1; + break; + }, + else => { + result.tag = .ampersand; + break; + }, + }, + + .asterisk => switch (c) { + '=' => { + result.tag = .asterisk_equal; + self.index += 1; + break; + }, + '*' => { + result.tag = .asterisk_asterisk; + self.index += 1; + break; + }, + '%' => { + state = .asterisk_percent; + }, + '|' => { + state = .asterisk_pipe; + }, + else => { + result.tag = .asterisk; + break; + }, + }, + + .asterisk_percent => switch (c) { + '=' => { + result.tag = .asterisk_percent_equal; + self.index += 1; + break; + }, + else => { + result.tag = .asterisk_percent; + break; + }, + }, + + .asterisk_pipe => switch (c) { + '=' => { + result.tag = .asterisk_pipe_equal; + self.index += 1; + break; + }, + else => { + result.tag = .asterisk_pipe; + break; + }, + }, + + .percent => switch (c) { + '=' => { + result.tag = .percent_equal; + self.index += 1; + break; + }, + else => { + result.tag = .percent; + break; + }, + }, + + .plus => switch (c) { + '=' => { + result.tag = .plus_equal; + self.index += 1; + break; + }, + '+' => { + result.tag = .plus_plus; + self.index += 1; + break; + }, + '%' => { + state = .plus_percent; + }, + '|' => { + state = .plus_pipe; + }, + else => { + result.tag = .plus; + break; + }, + }, + + .plus_percent => switch (c) { + '=' => { + result.tag = .plus_percent_equal; + self.index += 1; + break; + }, + else => { + result.tag = .plus_percent; + break; + }, + }, + + .plus_pipe => switch (c) { + '=' => { + result.tag = .plus_pipe_equal; + self.index += 1; + break; + }, + else => { + result.tag = .plus_pipe; + break; + }, + }, + + .caret => switch (c) { + '=' => { + result.tag = .caret_equal; + self.index += 1; + break; + }, + else => { + result.tag = .caret; + break; + }, + }, + + .identifier => switch (c) { + 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, + else => { + if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| { + result.tag = tag; + } + break; + }, + }, + .builtin => switch (c) { + 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, + else => break, + }, + .backslash => switch (c) { + '\\' => { + state = .multiline_string_literal_line; + }, + else => { + result.tag = .invalid; + break; + }, + }, + .string_literal => switch (c) { + '\\' => { + state = .string_literal_backslash; + }, + '"' => { + self.index += 1; + break; + }, + 0 => { + if (self.index == self.buffer.len) { + result.tag = .invalid; + break; + } else { + self.checkLiteralCharacter(); + } + }, + '\n' => { + result.tag = .invalid; + break; + }, + else => self.checkLiteralCharacter(), + }, + + .string_literal_backslash => switch (c) { + 0, '\n' => { + result.tag = .invalid; + break; + }, + else => { + state = .string_literal; + }, + }, + + .char_literal => switch (c) { + 0 => { + result.tag = .invalid; + break; + }, + '\\' => { + state = .char_literal_backslash; + }, + '\'', 0x80...0xbf, 0xf8...0xff => { + result.tag = .invalid; + break; + }, + 0xc0...0xdf => { // 110xxxxx + remaining_code_units = 1; + state = .char_literal_unicode; + }, + 0xe0...0xef => { // 1110xxxx + remaining_code_units = 2; + state = .char_literal_unicode; + }, + 0xf0...0xf7 => { // 11110xxx + remaining_code_units = 3; + state = .char_literal_unicode; + }, + '\n' => { + result.tag = .invalid; + break; + }, + else => { + state = .char_literal_end; + }, + }, + + .char_literal_backslash => switch (c) { + 0, '\n' => { + result.tag = .invalid; + break; + }, + 'x' => { + state = .char_literal_hex_escape; + seen_escape_digits = 0; + }, + 'u' => { + state = .char_literal_unicode_escape_saw_u; + }, + else => { + state = .char_literal_end; + }, + }, + + .char_literal_hex_escape => switch (c) { + '0'...'9', 'a'...'f', 'A'...'F' => { + seen_escape_digits += 1; + if (seen_escape_digits == 2) { + state = .char_literal_end; + } + }, + else => { + result.tag = .invalid; + break; + }, + }, + + .char_literal_unicode_escape_saw_u => switch (c) { + 0 => { + result.tag = .invalid; + break; + }, + '{' => { + state = .char_literal_unicode_escape; + }, + else => { + result.tag = .invalid; + state = .char_literal_unicode_invalid; + }, + }, + + .char_literal_unicode_escape => switch (c) { + 0 => { + result.tag = .invalid; + break; + }, + '0'...'9', 'a'...'f', 'A'...'F' => {}, + '}' => { + state = .char_literal_end; // too many/few digits handled later + }, + else => { + result.tag = .invalid; + state = .char_literal_unicode_invalid; + }, + }, + + .char_literal_unicode_invalid => switch (c) { + // Keep consuming characters until an obvious stopping point. + // This consolidates e.g. `u{0ab1Q}` into a single invalid token + // instead of creating the tokens `u{0ab1`, `Q`, `}` + '0'...'9', 'a'...'z', 'A'...'Z', '}' => {}, + else => break, + }, + + .char_literal_end => switch (c) { + '\'' => { + result.tag = .char_literal; + self.index += 1; + break; + }, + else => { + result.tag = .invalid; + break; + }, + }, + + .char_literal_unicode => switch (c) { + 0x80...0xbf => { + remaining_code_units -= 1; + if (remaining_code_units == 0) { + state = .char_literal_end; + } + }, + else => { + result.tag = .invalid; + break; + }, + }, + + .multiline_string_literal_line => switch (c) { + 0 => break, + '\n' => { + self.index += 1; + break; + }, + '\t' => {}, + else => self.checkLiteralCharacter(), + }, + + .bang => switch (c) { + '=' => { + result.tag = .bang_equal; + self.index += 1; + break; + }, + else => { + result.tag = .bang; + break; + }, + }, + + .pipe => switch (c) { + '=' => { + result.tag = .pipe_equal; + self.index += 1; + break; + }, + '|' => { + result.tag = .pipe_pipe; + self.index += 1; + break; + }, + else => { + result.tag = .pipe; + break; + }, + }, + + .equal => switch (c) { + '=' => { + result.tag = .equal_equal; + self.index += 1; + break; + }, + '>' => { + result.tag = .equal_angle_bracket_right; + self.index += 1; + break; + }, + else => { + result.tag = .equal; + break; + }, + }, + + .minus => switch (c) { + '>' => { + result.tag = .arrow; + self.index += 1; + break; + }, + '=' => { + result.tag = .minus_equal; + self.index += 1; + break; + }, + '%' => { + state = .minus_percent; + }, + '|' => { + state = .minus_pipe; + }, + else => { + result.tag = .minus; + break; + }, + }, + + .minus_percent => switch (c) { + '=' => { + result.tag = .minus_percent_equal; + self.index += 1; + break; + }, + else => { + result.tag = .minus_percent; + break; + }, + }, + .minus_pipe => switch (c) { + '=' => { + result.tag = .minus_pipe_equal; + self.index += 1; + break; + }, + else => { + result.tag = .minus_pipe; + break; + }, + }, + + .angle_bracket_left => switch (c) { + '<' => { + state = .angle_bracket_angle_bracket_left; + }, + '=' => { + result.tag = .angle_bracket_left_equal; + self.index += 1; + break; + }, + else => { + result.tag = .angle_bracket_left; + break; + }, + }, + + .angle_bracket_angle_bracket_left => switch (c) { + '=' => { + result.tag = .angle_bracket_angle_bracket_left_equal; + self.index += 1; + break; + }, + '|' => { + state = .angle_bracket_angle_bracket_left_pipe; + }, + else => { + result.tag = .angle_bracket_angle_bracket_left; + break; + }, + }, + + .angle_bracket_angle_bracket_left_pipe => switch (c) { + '=' => { + result.tag = .angle_bracket_angle_bracket_left_pipe_equal; + self.index += 1; + break; + }, + else => { + result.tag = .angle_bracket_angle_bracket_left_pipe; + break; + }, + }, + + .angle_bracket_right => switch (c) { + '>' => { + state = .angle_bracket_angle_bracket_right; + }, + '=' => { + result.tag = .angle_bracket_right_equal; + self.index += 1; + break; + }, + else => { + result.tag = .angle_bracket_right; + break; + }, + }, + + .angle_bracket_angle_bracket_right => switch (c) { + '=' => { + result.tag = .angle_bracket_angle_bracket_right_equal; + self.index += 1; + break; + }, + else => { + result.tag = .angle_bracket_angle_bracket_right; + break; + }, + }, + + .period => switch (c) { + '.' => { + state = .period_2; + }, + '*' => { + state = .period_asterisk; + }, + else => { + result.tag = .period; + break; + }, + }, + + .period_2 => switch (c) { + '.' => { + result.tag = .ellipsis3; + self.index += 1; + break; + }, + else => { + result.tag = .ellipsis2; + break; + }, + }, + + .period_asterisk => switch (c) { + '*' => { + result.tag = .invalid_periodasterisks; + break; + }, + else => { + result.tag = .period_asterisk; + break; + }, + }, + + .slash => switch (c) { + '/' => { + state = .line_comment_start; + }, + '=' => { + result.tag = .slash_equal; + self.index += 1; + break; + }, + else => { + result.tag = .slash; + break; + }, + }, + .line_comment_start => switch (c) { + 0 => { + if (self.index != self.buffer.len) { + result.tag = .invalid; + self.index += 1; + } + break; + }, + '/' => { + state = .doc_comment_start; + }, + '!' => { + result.tag = .container_doc_comment; + state = .doc_comment; + }, + '\n' => { + state = .start; + result.loc.start = self.index + 1; + }, + '\t' => state = .line_comment, + else => { + state = .line_comment; + self.checkLiteralCharacter(); + }, + }, + .doc_comment_start => switch (c) { + '/' => { + state = .line_comment; + }, + 0, '\n' => { + result.tag = .doc_comment; + break; + }, + '\t' => { + state = .doc_comment; + result.tag = .doc_comment; + }, + else => { + state = .doc_comment; + result.tag = .doc_comment; + self.checkLiteralCharacter(); + }, + }, + .line_comment => switch (c) { + 0 => { + if (self.index != self.buffer.len) { + result.tag = .invalid; + self.index += 1; + } + break; + }, + '\n' => { + state = .start; + result.loc.start = self.index + 1; + }, + '\t' => {}, + else => self.checkLiteralCharacter(), + }, + .doc_comment => switch (c) { + 0, '\n' => break, + '\t' => {}, + else => self.checkLiteralCharacter(), + }, + .int => switch (c) { + '.' => state = .int_period, + '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {}, + 'e', 'E', 'p', 'P' => state = .int_exponent, + else => break, + }, + .int_exponent => switch (c) { + '-', '+' => { + state = .float; + }, + else => { + self.index -= 1; + state = .int; + }, + }, + .int_period => switch (c) { + '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => { + state = .float; + }, + 'e', 'E', 'p', 'P' => state = .float_exponent, + else => { + self.index -= 1; + break; + }, + }, + .float => switch (c) { + '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {}, + 'e', 'E', 'p', 'P' => state = .float_exponent, + else => break, + }, + .float_exponent => switch (c) { + '-', '+' => state = .float, + else => { + self.index -= 1; + state = .float; + }, + }, + } + } + + if (result.tag == .eof) { + if (self.pending_invalid_token) |token| { + self.pending_invalid_token = null; + return token; + } + result.loc.start = self.index; + } + + result.loc.end = self.index; + return result; + } + + fn checkLiteralCharacter(self: *Tokenizer) void { + if (self.pending_invalid_token != null) return; + const invalid_length = self.getInvalidCharacterLength(); + if (invalid_length == 0) return; + self.pending_invalid_token = .{ + .tag = .invalid, + .loc = .{ + .start = self.index, + .end = self.index + invalid_length, + }, + }; + } + + fn getInvalidCharacterLength(self: *Tokenizer) u3 { + const c0 = self.buffer[self.index]; + if (std.ascii.isASCII(c0)) { + if (c0 == '\r') { + if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') { + // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise + // they constitute an illegal byte! + return 0; + } else { + return 1; + } + } else if (std.ascii.isControl(c0)) { + // ascii control codes are never allowed + // (note that \n was checked before we got here) + return 1; + } + // looks fine to me. + return 0; + } else { + // check utf8-encoded character. + const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1; + if (self.index + length > self.buffer.len) { + return @as(u3, @intCast(self.buffer.len - self.index)); + } + const bytes = self.buffer[self.index .. self.index + length]; + switch (length) { + 2 => { + const value = std.unicode.utf8Decode2(bytes) catch return length; + if (value == 0x85) return length; // U+0085 (NEL) + }, + 3 => { + const value = std.unicode.utf8Decode3(bytes) catch return length; + if (value == 0x2028) return length; // U+2028 (LS) + if (value == 0x2029) return length; // U+2029 (PS) + }, + 4 => { + _ = std.unicode.utf8Decode4(bytes) catch return length; + }, + else => unreachable, + } + self.index += length - 1; + return 0; + } + } +}; + +test "keywords" { + try testTokenize("test const else", &.{ .keyword_test, .keyword_const, .keyword_else }); +} + +test "line comment followed by top-level comptime" { + try testTokenize( + \\// line comment + \\comptime {} + \\ + , &.{ + .keyword_comptime, + .l_brace, + .r_brace, + }); +} + +test "unknown length pointer and then c pointer" { + try testTokenize( + \\[*]u8 + \\[*c]u8 + , &.{ + .l_bracket, + .asterisk, + .r_bracket, + .identifier, + .l_bracket, + .asterisk, + .identifier, + .r_bracket, + .identifier, + }); +} + +test "code point literal with hex escape" { + try testTokenize( + \\'\x1b' + , &.{.char_literal}); + try testTokenize( + \\'\x1' + , &.{ .invalid, .invalid }); +} + +test "newline in char literal" { + try testTokenize( + \\' + \\' + , &.{ .invalid, .invalid }); +} + +test "newline in string literal" { + try testTokenize( + \\" + \\" + , &.{ .invalid, .invalid }); +} + +test "code point literal with unicode escapes" { + // Valid unicode escapes + try testTokenize( + \\'\u{3}' + , &.{.char_literal}); + try testTokenize( + \\'\u{01}' + , &.{.char_literal}); + try testTokenize( + \\'\u{2a}' + , &.{.char_literal}); + try testTokenize( + \\'\u{3f9}' + , &.{.char_literal}); + try testTokenize( + \\'\u{6E09aBc1523}' + , &.{.char_literal}); + try testTokenize( + \\"\u{440}" + , &.{.string_literal}); + + // Invalid unicode escapes + try testTokenize( + \\'\u' + , &.{.invalid}); + try testTokenize( + \\'\u{{' + , &.{ .invalid, .invalid }); + try testTokenize( + \\'\u{}' + , &.{.char_literal}); + try testTokenize( + \\'\u{s}' + , &.{ .invalid, .invalid }); + try testTokenize( + \\'\u{2z}' + , &.{ .invalid, .invalid }); + try testTokenize( + \\'\u{4a' + , &.{.invalid}); + + // Test old-style unicode literals + try testTokenize( + \\'\u0333' + , &.{ .invalid, .invalid }); + try testTokenize( + \\'\U0333' + , &.{ .invalid, .number_literal, .invalid }); +} + +test "code point literal with unicode code point" { + try testTokenize( + \\'💩' + , &.{.char_literal}); +} + +test "float literal e exponent" { + try testTokenize("a = 4.94065645841246544177e-324;\n", &.{ + .identifier, + .equal, + .number_literal, + .semicolon, + }); +} + +test "float literal p exponent" { + try testTokenize("a = 0x1.a827999fcef32p+1022;\n", &.{ + .identifier, + .equal, + .number_literal, + .semicolon, + }); +} + +test "chars" { + try testTokenize("'c'", &.{.char_literal}); +} + +test "invalid token characters" { + try testTokenize("#", &.{.invalid}); + try testTokenize("`", &.{.invalid}); + try testTokenize("'c", &.{.invalid}); + try testTokenize("'", &.{.invalid}); + try testTokenize("''", &.{ .invalid, .invalid }); +} + +test "invalid literal/comment characters" { + try testTokenize("\"\x00\"", &.{ + .string_literal, + .invalid, + }); + try testTokenize("//\x00", &.{ + .invalid, + }); + try testTokenize("//\x1f", &.{ + .invalid, + }); + try testTokenize("//\x7f", &.{ + .invalid, + }); +} + +test "utf8" { + try testTokenize("//\xc2\x80", &.{}); + try testTokenize("//\xf4\x8f\xbf\xbf", &.{}); +} + +test "invalid utf8" { + try testTokenize("//\x80", &.{ + .invalid, + }); + try testTokenize("//\xbf", &.{ + .invalid, + }); + try testTokenize("//\xf8", &.{ + .invalid, + }); + try testTokenize("//\xff", &.{ + .invalid, + }); + try testTokenize("//\xc2\xc0", &.{ + .invalid, + }); + try testTokenize("//\xe0", &.{ + .invalid, + }); + try testTokenize("//\xf0", &.{ + .invalid, + }); + try testTokenize("//\xf0\x90\x80\xc0", &.{ + .invalid, + }); +} + +test "illegal unicode codepoints" { + // unicode newline characters.U+0085, U+2028, U+2029 + try testTokenize("//\xc2\x84", &.{}); + try testTokenize("//\xc2\x85", &.{ + .invalid, + }); + try testTokenize("//\xc2\x86", &.{}); + try testTokenize("//\xe2\x80\xa7", &.{}); + try testTokenize("//\xe2\x80\xa8", &.{ + .invalid, + }); + try testTokenize("//\xe2\x80\xa9", &.{ + .invalid, + }); + try testTokenize("//\xe2\x80\xaa", &.{}); +} + +test "string identifier and builtin fns" { + try testTokenize( + \\const @"if" = @import("std"); + , &.{ + .keyword_const, + .identifier, + .equal, + .builtin, + .l_paren, + .string_literal, + .r_paren, + .semicolon, + }); +} + +test "multiline string literal with literal tab" { + try testTokenize( + \\\\foo bar + , &.{ + .multiline_string_literal_line, + }); +} + +test "comments with literal tab" { + try testTokenize( + \\//foo bar + \\//!foo bar + \\///foo bar + \\// foo + \\/// foo + \\/// /foo + , &.{ + .container_doc_comment, + .doc_comment, + .doc_comment, + .doc_comment, + }); +} + +test "pipe and then invalid" { + try testTokenize("||=", &.{ + .pipe_pipe, + .equal, + }); +} + +test "line comment and doc comment" { + try testTokenize("//", &.{}); + try testTokenize("// a / b", &.{}); + try testTokenize("// /", &.{}); + try testTokenize("/// a", &.{.doc_comment}); + try testTokenize("///", &.{.doc_comment}); + try testTokenize("////", &.{}); + try testTokenize("//!", &.{.container_doc_comment}); + try testTokenize("//!!", &.{.container_doc_comment}); +} + +test "line comment followed by identifier" { + try testTokenize( + \\ Unexpected, + \\ // another + \\ Another, + , &.{ + .identifier, + .comma, + .identifier, + .comma, + }); +} + +test "UTF-8 BOM is recognized and skipped" { + try testTokenize("\xEF\xBB\xBFa;\n", &.{ + .identifier, + .semicolon, + }); +} + +test "correctly parse pointer assignment" { + try testTokenize("b.*=3;\n", &.{ + .identifier, + .period_asterisk, + .equal, + .number_literal, + .semicolon, + }); +} + +test "correctly parse pointer dereference followed by asterisk" { + try testTokenize("\"b\".* ** 10", &.{ + .string_literal, + .period_asterisk, + .asterisk_asterisk, + .number_literal, + }); + + try testTokenize("(\"b\".*)** 10", &.{ + .l_paren, + .string_literal, + .period_asterisk, + .r_paren, + .asterisk_asterisk, + .number_literal, + }); + + try testTokenize("\"b\".*** 10", &.{ + .string_literal, + .invalid_periodasterisks, + .asterisk_asterisk, + .number_literal, + }); +} + +test "range literals" { + try testTokenize("0...9", &.{ .number_literal, .ellipsis3, .number_literal }); + try testTokenize("'0'...'9'", &.{ .char_literal, .ellipsis3, .char_literal }); + try testTokenize("0x00...0x09", &.{ .number_literal, .ellipsis3, .number_literal }); + try testTokenize("0b00...0b11", &.{ .number_literal, .ellipsis3, .number_literal }); + try testTokenize("0o00...0o11", &.{ .number_literal, .ellipsis3, .number_literal }); +} + +test "number literals decimal" { + try testTokenize("0", &.{.number_literal}); + try testTokenize("1", &.{.number_literal}); + try testTokenize("2", &.{.number_literal}); + try testTokenize("3", &.{.number_literal}); + try testTokenize("4", &.{.number_literal}); + try testTokenize("5", &.{.number_literal}); + try testTokenize("6", &.{.number_literal}); + try testTokenize("7", &.{.number_literal}); + try testTokenize("8", &.{.number_literal}); + try testTokenize("9", &.{.number_literal}); + try testTokenize("1..", &.{ .number_literal, .ellipsis2 }); + try testTokenize("0a", &.{.number_literal}); + try testTokenize("9b", &.{.number_literal}); + try testTokenize("1z", &.{.number_literal}); + try testTokenize("1z_1", &.{.number_literal}); + try testTokenize("9z3", &.{.number_literal}); + + try testTokenize("0_0", &.{.number_literal}); + try testTokenize("0001", &.{.number_literal}); + try testTokenize("01234567890", &.{.number_literal}); + try testTokenize("012_345_6789_0", &.{.number_literal}); + try testTokenize("0_1_2_3_4_5_6_7_8_9_0", &.{.number_literal}); + + try testTokenize("00_", &.{.number_literal}); + try testTokenize("0_0_", &.{.number_literal}); + try testTokenize("0__0", &.{.number_literal}); + try testTokenize("0_0f", &.{.number_literal}); + try testTokenize("0_0_f", &.{.number_literal}); + try testTokenize("0_0_f_00", &.{.number_literal}); + try testTokenize("1_,", &.{ .number_literal, .comma }); + + try testTokenize("0.0", &.{.number_literal}); + try testTokenize("1.0", &.{.number_literal}); + try testTokenize("10.0", &.{.number_literal}); + try testTokenize("0e0", &.{.number_literal}); + try testTokenize("1e0", &.{.number_literal}); + try testTokenize("1e100", &.{.number_literal}); + try testTokenize("1.0e100", &.{.number_literal}); + try testTokenize("1.0e+100", &.{.number_literal}); + try testTokenize("1.0e-100", &.{.number_literal}); + try testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &.{.number_literal}); + + try testTokenize("1.", &.{ .number_literal, .period }); + try testTokenize("1e", &.{.number_literal}); + try testTokenize("1.e100", &.{.number_literal}); + try testTokenize("1.0e1f0", &.{.number_literal}); + try testTokenize("1.0p100", &.{.number_literal}); + try testTokenize("1.0p-100", &.{.number_literal}); + try testTokenize("1.0p1f0", &.{.number_literal}); + try testTokenize("1.0_,", &.{ .number_literal, .comma }); + try testTokenize("1_.0", &.{.number_literal}); + try testTokenize("1._", &.{.number_literal}); + try testTokenize("1.a", &.{.number_literal}); + try testTokenize("1.z", &.{.number_literal}); + try testTokenize("1._0", &.{.number_literal}); + try testTokenize("1.+", &.{ .number_literal, .period, .plus }); + try testTokenize("1._+", &.{ .number_literal, .plus }); + try testTokenize("1._e", &.{.number_literal}); + try testTokenize("1.0e", &.{.number_literal}); + try testTokenize("1.0e,", &.{ .number_literal, .comma }); + try testTokenize("1.0e_", &.{.number_literal}); + try testTokenize("1.0e+_", &.{.number_literal}); + try testTokenize("1.0e-_", &.{.number_literal}); + try testTokenize("1.0e0_+", &.{ .number_literal, .plus }); +} + +test "number literals binary" { + try testTokenize("0b0", &.{.number_literal}); + try testTokenize("0b1", &.{.number_literal}); + try testTokenize("0b2", &.{.number_literal}); + try testTokenize("0b3", &.{.number_literal}); + try testTokenize("0b4", &.{.number_literal}); + try testTokenize("0b5", &.{.number_literal}); + try testTokenize("0b6", &.{.number_literal}); + try testTokenize("0b7", &.{.number_literal}); + try testTokenize("0b8", &.{.number_literal}); + try testTokenize("0b9", &.{.number_literal}); + try testTokenize("0ba", &.{.number_literal}); + try testTokenize("0bb", &.{.number_literal}); + try testTokenize("0bc", &.{.number_literal}); + try testTokenize("0bd", &.{.number_literal}); + try testTokenize("0be", &.{.number_literal}); + try testTokenize("0bf", &.{.number_literal}); + try testTokenize("0bz", &.{.number_literal}); + + try testTokenize("0b0000_0000", &.{.number_literal}); + try testTokenize("0b1111_1111", &.{.number_literal}); + try testTokenize("0b10_10_10_10", &.{.number_literal}); + try testTokenize("0b0_1_0_1_0_1_0_1", &.{.number_literal}); + try testTokenize("0b1.", &.{ .number_literal, .period }); + try testTokenize("0b1.0", &.{.number_literal}); + + try testTokenize("0B0", &.{.number_literal}); + try testTokenize("0b_", &.{.number_literal}); + try testTokenize("0b_0", &.{.number_literal}); + try testTokenize("0b1_", &.{.number_literal}); + try testTokenize("0b0__1", &.{.number_literal}); + try testTokenize("0b0_1_", &.{.number_literal}); + try testTokenize("0b1e", &.{.number_literal}); + try testTokenize("0b1p", &.{.number_literal}); + try testTokenize("0b1e0", &.{.number_literal}); + try testTokenize("0b1p0", &.{.number_literal}); + try testTokenize("0b1_,", &.{ .number_literal, .comma }); +} + +test "number literals octal" { + try testTokenize("0o0", &.{.number_literal}); + try testTokenize("0o1", &.{.number_literal}); + try testTokenize("0o2", &.{.number_literal}); + try testTokenize("0o3", &.{.number_literal}); + try testTokenize("0o4", &.{.number_literal}); + try testTokenize("0o5", &.{.number_literal}); + try testTokenize("0o6", &.{.number_literal}); + try testTokenize("0o7", &.{.number_literal}); + try testTokenize("0o8", &.{.number_literal}); + try testTokenize("0o9", &.{.number_literal}); + try testTokenize("0oa", &.{.number_literal}); + try testTokenize("0ob", &.{.number_literal}); + try testTokenize("0oc", &.{.number_literal}); + try testTokenize("0od", &.{.number_literal}); + try testTokenize("0oe", &.{.number_literal}); + try testTokenize("0of", &.{.number_literal}); + try testTokenize("0oz", &.{.number_literal}); + + try testTokenize("0o01234567", &.{.number_literal}); + try testTokenize("0o0123_4567", &.{.number_literal}); + try testTokenize("0o01_23_45_67", &.{.number_literal}); + try testTokenize("0o0_1_2_3_4_5_6_7", &.{.number_literal}); + try testTokenize("0o7.", &.{ .number_literal, .period }); + try testTokenize("0o7.0", &.{.number_literal}); + + try testTokenize("0O0", &.{.number_literal}); + try testTokenize("0o_", &.{.number_literal}); + try testTokenize("0o_0", &.{.number_literal}); + try testTokenize("0o1_", &.{.number_literal}); + try testTokenize("0o0__1", &.{.number_literal}); + try testTokenize("0o0_1_", &.{.number_literal}); + try testTokenize("0o1e", &.{.number_literal}); + try testTokenize("0o1p", &.{.number_literal}); + try testTokenize("0o1e0", &.{.number_literal}); + try testTokenize("0o1p0", &.{.number_literal}); + try testTokenize("0o_,", &.{ .number_literal, .comma }); +} + +test "number literals hexadecimal" { + try testTokenize("0x0", &.{.number_literal}); + try testTokenize("0x1", &.{.number_literal}); + try testTokenize("0x2", &.{.number_literal}); + try testTokenize("0x3", &.{.number_literal}); + try testTokenize("0x4", &.{.number_literal}); + try testTokenize("0x5", &.{.number_literal}); + try testTokenize("0x6", &.{.number_literal}); + try testTokenize("0x7", &.{.number_literal}); + try testTokenize("0x8", &.{.number_literal}); + try testTokenize("0x9", &.{.number_literal}); + try testTokenize("0xa", &.{.number_literal}); + try testTokenize("0xb", &.{.number_literal}); + try testTokenize("0xc", &.{.number_literal}); + try testTokenize("0xd", &.{.number_literal}); + try testTokenize("0xe", &.{.number_literal}); + try testTokenize("0xf", &.{.number_literal}); + try testTokenize("0xA", &.{.number_literal}); + try testTokenize("0xB", &.{.number_literal}); + try testTokenize("0xC", &.{.number_literal}); + try testTokenize("0xD", &.{.number_literal}); + try testTokenize("0xE", &.{.number_literal}); + try testTokenize("0xF", &.{.number_literal}); + try testTokenize("0x0z", &.{.number_literal}); + try testTokenize("0xz", &.{.number_literal}); + + try testTokenize("0x0123456789ABCDEF", &.{.number_literal}); + try testTokenize("0x0123_4567_89AB_CDEF", &.{.number_literal}); + try testTokenize("0x01_23_45_67_89AB_CDE_F", &.{.number_literal}); + try testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &.{.number_literal}); + + try testTokenize("0X0", &.{.number_literal}); + try testTokenize("0x_", &.{.number_literal}); + try testTokenize("0x_1", &.{.number_literal}); + try testTokenize("0x1_", &.{.number_literal}); + try testTokenize("0x0__1", &.{.number_literal}); + try testTokenize("0x0_1_", &.{.number_literal}); + try testTokenize("0x_,", &.{ .number_literal, .comma }); + + try testTokenize("0x1.0", &.{.number_literal}); + try testTokenize("0xF.0", &.{.number_literal}); + try testTokenize("0xF.F", &.{.number_literal}); + try testTokenize("0xF.Fp0", &.{.number_literal}); + try testTokenize("0xF.FP0", &.{.number_literal}); + try testTokenize("0x1p0", &.{.number_literal}); + try testTokenize("0xfp0", &.{.number_literal}); + try testTokenize("0x1.0+0xF.0", &.{ .number_literal, .plus, .number_literal }); + + try testTokenize("0x1.", &.{ .number_literal, .period }); + try testTokenize("0xF.", &.{ .number_literal, .period }); + try testTokenize("0x1.+0xF.", &.{ .number_literal, .period, .plus, .number_literal, .period }); + try testTokenize("0xff.p10", &.{.number_literal}); + + try testTokenize("0x0123456.789ABCDEF", &.{.number_literal}); + try testTokenize("0x0_123_456.789_ABC_DEF", &.{.number_literal}); + try testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &.{.number_literal}); + try testTokenize("0x0p0", &.{.number_literal}); + try testTokenize("0x0.0p0", &.{.number_literal}); + try testTokenize("0xff.ffp10", &.{.number_literal}); + try testTokenize("0xff.ffP10", &.{.number_literal}); + try testTokenize("0xffp10", &.{.number_literal}); + try testTokenize("0xff_ff.ff_ffp1_0_0_0", &.{.number_literal}); + try testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &.{.number_literal}); + try testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &.{.number_literal}); + + try testTokenize("0x1e", &.{.number_literal}); + try testTokenize("0x1e0", &.{.number_literal}); + try testTokenize("0x1p", &.{.number_literal}); + try testTokenize("0xfp0z1", &.{.number_literal}); + try testTokenize("0xff.ffpff", &.{.number_literal}); + try testTokenize("0x0.p", &.{.number_literal}); + try testTokenize("0x0.z", &.{.number_literal}); + try testTokenize("0x0._", &.{.number_literal}); + try testTokenize("0x0_.0", &.{.number_literal}); + try testTokenize("0x0_.0.0", &.{ .number_literal, .period, .number_literal }); + try testTokenize("0x0._0", &.{.number_literal}); + try testTokenize("0x0.0_", &.{.number_literal}); + try testTokenize("0x0_p0", &.{.number_literal}); + try testTokenize("0x0_.p0", &.{.number_literal}); + try testTokenize("0x0._p0", &.{.number_literal}); + try testTokenize("0x0.0_p0", &.{.number_literal}); + try testTokenize("0x0._0p0", &.{.number_literal}); + try testTokenize("0x0.0p_0", &.{.number_literal}); + try testTokenize("0x0.0p+_0", &.{.number_literal}); + try testTokenize("0x0.0p-_0", &.{.number_literal}); + try testTokenize("0x0.0p0_", &.{.number_literal}); +} + +test "multi line string literal with only 1 backslash" { + try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon }); +} + +test "invalid builtin identifiers" { + try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren }); + try testTokenize("@0()", &.{ .invalid, .number_literal, .l_paren, .r_paren }); +} + +test "invalid token with unfinished escape right before eof" { + try testTokenize("\"\\", &.{.invalid}); + try testTokenize("'\\", &.{.invalid}); + try testTokenize("'\\u", &.{.invalid}); +} + +test "saturating operators" { + try testTokenize("<<", &.{.angle_bracket_angle_bracket_left}); + try testTokenize("<<|", &.{.angle_bracket_angle_bracket_left_pipe}); + try testTokenize("<<|=", &.{.angle_bracket_angle_bracket_left_pipe_equal}); + + try testTokenize("*", &.{.asterisk}); + try testTokenize("*|", &.{.asterisk_pipe}); + try testTokenize("*|=", &.{.asterisk_pipe_equal}); + + try testTokenize("+", &.{.plus}); + try testTokenize("+|", &.{.plus_pipe}); + try testTokenize("+|=", &.{.plus_pipe_equal}); + + try testTokenize("-", &.{.minus}); + try testTokenize("-|", &.{.minus_pipe}); + try testTokenize("-|=", &.{.minus_pipe_equal}); +} + +test "null byte before eof" { + try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal }); + try testTokenize("//\x00", &.{.invalid}); + try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid }); + try testTokenize("\x00", &.{.invalid}); + try testTokenize("// NUL\x00\n", &.{.invalid}); + try testTokenize("///\x00\n", &.{ .doc_comment, .invalid }); + try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid }); +} + +fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { + var tokenizer = Tokenizer.init(source); + for (expected_token_tags) |expected_token_tag| { + const token = tokenizer.next(); + try std.testing.expectEqual(expected_token_tag, token.tag); + } + const last_token = tokenizer.next(); + try std.testing.expectEqual(Token.Tag.eof, last_token.tag); + try std.testing.expectEqual(source.len, last_token.loc.start); + try std.testing.expectEqual(source.len, last_token.loc.end); +}