refactor: Keep interpreted token streams separate from TokenStream

This aims to avoid rust-lang/rust-analyzer#18211 (comment) in most cases

src/commands/control_flow_commands.rs

@@ -225,12 +225,10 @@ impl ControlFlowCommandDefinition for ForCommand {
 
         let mut iteration_counter = interpreter.start_iteration_counter(&self.in_token);
 
-        for token in stream.into_token_stream() {
+        for token in stream.into_item_vec() {
             iteration_counter.increment_and_check()?;
-            self.parse_place.handle_parse_from_stream(
-                InterpretedStream::raw(token.into_token_stream()),
-                interpreter,
-            )?;
+            self.parse_place
+                .handle_parse_from_stream(token.into(), interpreter)?;
             match self
                 .loop_code
                 .clone()

src/commands/core_commands.rs

@@ -99,7 +99,7 @@ impl StreamCommandDefinition for RawCommand {
         _interpreter: &mut Interpreter,
         output: &mut InterpretedStream,
     ) -> Result<()> {
-        output.extend_raw_token_iter(self.token_stream);
+        output.extend_raw_tokens(self.token_stream);
         Ok(())
     }
 }
@@ -255,9 +255,8 @@ impl NoOutputCommandDefinition for ErrorCommand {
                 // transparent groups (as of Jan 2025), so gets it right without this flattening:
                 // https://github.com/rust-lang/rust-analyzer/issues/18211
 
-                let error_span_stream = error_span_stream
-                    .into_token_stream()
-                    .flatten_transparent_groups();
+                let error_span_stream =
+                    error_span_stream.into_token_stream_removing_any_transparent_groups();
                 if error_span_stream.is_empty() {
                     Span::call_site().span_range()
                 } else {

src/commands/expression_commands.rs

@@ -161,7 +161,7 @@ impl StreamCommandDefinition for RangeCommand {
 }
 
 fn output_range(iter: impl Iterator<Item = i128>, span: Span, output: &mut InterpretedStream) {
-    output.extend_raw_token_iter(iter.map(|value| {
+    output.extend_raw_tokens(iter.map(|value| {
         let literal = Literal::i128_unsuffixed(value).with_span(span);
         TokenTree::Literal(literal)
             // We wrap it in a singleton group to ensure that negative

src/commands/token_commands.rs

@@ -46,8 +46,7 @@ impl ValueCommandDefinition for LengthCommand {
     fn execute(self: Box<Self>, interpreter: &mut Interpreter) -> Result<TokenTree> {
         let output_span = self.arguments.span_range().span();
         let interpreted = self.arguments.interpret_to_new_stream(interpreter)?;
-        let stream_length = interpreted.into_token_stream().into_iter().count();
-        let length_literal = Literal::usize_unsuffixed(stream_length).with_span(output_span);
+        let length_literal = Literal::usize_unsuffixed(interpreted.len()).with_span(output_span);
         Ok(length_literal.into())
     }
 }
@@ -121,7 +120,7 @@ impl StreamCommandDefinition for IntersperseCommand {
             .inputs
             .items
             .interpret_to_new_stream(interpreter)?
-            .into_token_stream();
+            .into_item_vec();
         let add_trailing = match self.inputs.add_trailing {
             Some(add_trailing) => add_trailing.interpret(interpreter)?.value(),
             None => false,
@@ -140,7 +139,7 @@ impl StreamCommandDefinition for IntersperseCommand {
         let mut items = items.into_iter().peekable();
         let mut this_item = items.next().unwrap(); // Safe to unwrap as non-empty
         loop {
-            output.push_raw_token_tree(this_item);
+            output.push_segment_item(this_item);
             let next_item = items.next();
             match next_item {
                 Some(next_item) => {

src/expressions/expression_stream.rs

@@ -219,17 +219,16 @@ impl ExpressionBuilder {
         appender: impl FnOnce(&mut InterpretedStream) -> Result<()>,
         span: Span,
     ) -> Result<()> {
-        // Currently using Expr::Parse, it ignores transparent groups, which is
-        // a little too permissive.
-        // Instead, we use parentheses to ensure that the group has to be a valid
-        // expression itself, without being flattened
+        // Currently using Expr::Parse, it ignores transparent groups, which is a little too permissive.
+        // Instead, we use parentheses to ensure that the group has to be a valid expression itself, without being flattened.
+        // This also works around the SAFETY issue in syn_parse below
         self.interpreted_stream
             .push_grouped(appender, Delimiter::Parenthesis, span)
     }
 
     pub(crate) fn extend_with_evaluation_output(&mut self, value: EvaluationOutput) {
         self.interpreted_stream
-            .extend_raw_tokens(value.into_token_tree());
+            .extend_with_raw_tokens_from(value.into_token_tree());
     }
 
     pub(crate) fn push_expression_group(
@@ -254,7 +253,11 @@ impl ExpressionBuilder {
         // Because of the kind of expressions we're parsing (i.e. no {} allowed),
         // we can get by with parsing it as `Expr::parse` rather than with
         // `Expr::parse_without_eager_brace` or `Expr::parse_with_earlier_boundary_rule`.
-        let expression = Expr::parse.parse2(self.interpreted_stream.into_token_stream())?;
+        let expression = unsafe {
+            // RUST-ANALYZER SAFETY: We wrap commands and variables in `()` instead of none-delimited groups in expressions,
+            // so it doesn't matter that we can drop none-delimited groups
+            self.interpreted_stream.syn_parse(Expr::parse)?
+        };
 
         EvaluationTree::build_from(&expression)?.evaluate()
     }

src/interpretation/command_stream_input.rs

@@ -62,64 +62,53 @@ impl Interpret for CommandStreamInput {
                     | CommandOutputKind::Ident => {
                         command.err("The command does not output a stream")
                     }
-                    CommandOutputKind::FlattenedStream => {
-                        let span = command.span();
-                        let tokens = parse_as_stream_input(
-                            command.interpret_to_new_stream(interpreter)?,
-                            || {
-                                span.error("Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.")
-                            },
-                        )?;
-                        output.extend_raw_tokens(tokens);
-                        Ok(())
-                    }
+                    CommandOutputKind::FlattenedStream => parse_as_stream_input(
+                        command,
+                        interpreter,
+                        || {
+                            "Expected output of flattened command to contain a single [ ... ] or transparent group. Perhaps you want to remove the .., to use the command output as-is.".to_string()
+                        },
+                        output,
+                    ),
                     CommandOutputKind::GroupedStream => {
                         unsafe {
                             // SAFETY: The kind change GroupedStream <=> FlattenedStream is valid
                             command.set_output_kind(CommandOutputKind::FlattenedStream);
                         }
                         command.interpret_into(interpreter, output)
                     }
-                    CommandOutputKind::ControlFlowCodeStream => {
-                        let span = command.span();
-                        let tokens = parse_as_stream_input(
-                            command.interpret_to_new_stream(interpreter)?,
-                            || {
-                                span.error("Expected output of control flow command to contain a single [ ... ] or transparent group.")
-                            },
-                        )?;
-                        output.extend_raw_tokens(tokens);
-                        Ok(())
-                    }
+                    CommandOutputKind::ControlFlowCodeStream => parse_as_stream_input(
+                        command,
+                        interpreter,
+                        || {
+                            "Expected output of control flow command to contain a single [ ... ] or transparent group.".to_string()
+                        },
+                        output,
+                    ),
                 }
             }
-            CommandStreamInput::FlattenedVariable(variable) => {
-                let tokens = parse_as_stream_input(
-                    variable.interpret_to_new_stream(interpreter)?,
-                    || {
-                        variable.error(format!(
+            CommandStreamInput::FlattenedVariable(variable) => parse_as_stream_input(
+                &variable,
+                interpreter,
+                || {
+                    format!(
                         "Expected variable to contain a single [ ... ] or transparent group. Perhaps you want to use {} instead, to use the content of the variable as the stream.",
                         variable.display_grouped_variable_token(),
-                    ))
-                    },
-                )?;
-                output.extend_raw_tokens(tokens);
-                Ok(())
-            }
+                    )
+                },
+                output,
+            ),
             CommandStreamInput::GroupedVariable(variable) => {
                 variable.substitute_ungrouped_contents_into(interpreter, output)
             }
-            CommandStreamInput::Code(code) => {
-                let span = code.span();
-                let tokens = parse_as_stream_input(
-                    code.interpret_to_new_stream(interpreter)?,
-                    || {
-                        span.error("Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string())
-                    },
-                )?;
-                output.extend_raw_tokens(tokens);
-                Ok(())
-            }
+            CommandStreamInput::Code(code) => parse_as_stream_input(
+                code,
+                interpreter,
+                || {
+                    "Expected the { ... } block to output a single [ ... ] group or transparent group. You may wish to replace the outer `{ ... }` block with a `[ ... ]` block, which outputs all its contents as a stream.".to_string()
+                },
+                output,
+            ),
             CommandStreamInput::ExplicitStream(group) => {
                 group.into_content().interpret_into(interpreter, output)
             }
@@ -128,24 +117,18 @@ impl Interpret for CommandStreamInput {
 }
 
 fn parse_as_stream_input(
-    interpreted: InterpretedStream,
-    on_error: impl FnOnce() -> Error,
-) -> Result<TokenStream> {
-    fn get_group(interpreted: InterpretedStream) -> Option<Group> {
-        let mut token_iter = interpreted.into_token_stream().into_iter();
-        let group = match token_iter.next()? {
-            TokenTree::Group(group)
-                if matches!(group.delimiter(), Delimiter::Bracket | Delimiter::None) =>
-            {
-                Some(group)
-            }
-            _ => return None,
-        };
-        if token_iter.next().is_some() {
-            return None;
-        }
-        group
-    }
-    let group = get_group(interpreted).ok_or_else(on_error)?;
-    Ok(group.stream())
+    input: impl Interpret + HasSpanRange,
+    interpreter: &mut Interpreter,
+    error_message: impl FnOnce() -> String,
+    output: &mut InterpretedStream,
+) -> Result<()> {
+    let span = input.span_range();
+    input
+        .interpret_to_new_stream(interpreter)?
+        .unwrap_singleton_group(
+            |delimiter| matches!(delimiter, Delimiter::Bracket | Delimiter::None),
+            || span.error(error_message()),
+        )?
+        .append_into(output);
+    Ok(())
 }

src/interpretation/command_value_input.rs

@@ -60,13 +60,17 @@ impl<T: InterpretValue<InterpretedValue = I>, I: Parse> InterpretValue for Comma
             CommandValueInput::Code(code) => code.interpret_to_new_stream(interpreter)?,
             CommandValueInput::Value(value) => return value.interpret(interpreter),
         };
-        match interpreted_stream.syn_parse(I::parse) {
-            Ok(value) => Ok(value),
-            Err(err) => Err(err.concat(&format!(
-                "\nOccurred whilst parsing the {} to a {}.",
-                descriptor,
-                std::any::type_name::<I>()
-            ))),
+        unsafe {
+            // RUST-ANALYZER SAFETY: We only use I with simple parse functions so far which don't care about
+            // none-delimited groups
+            match interpreted_stream.syn_parse(I::parse) {
+                Ok(value) => Ok(value),
+                Err(err) => Err(err.concat(&format!(
+                    "\nOccurred whilst parsing the {} to a {}.",
+                    descriptor,
+                    std::any::type_name::<I>()
+                ))),
+            }
         }
     }
 }

src/interpretation/interpret_traits.rs

@@ -26,7 +26,7 @@ impl<T: InterpretValue<InterpretedValue = I> + HasSpanRange, I: ToTokens> Interp
         interpreter: &mut Interpreter,
         output: &mut InterpretedStream,
     ) -> Result<()> {
-        output.extend_raw_tokens(self.interpret(interpreter)?);
+        output.extend_with_raw_tokens_from(self.interpret(interpreter)?);
         Ok(())
     }
 }

src/interpretation/interpretation_stream.rs

@@ -141,7 +141,7 @@ impl Express for RawGroup {
     ) -> Result<()> {
         expression_stream.push_grouped(
             |inner| {
-                inner.extend_raw_token_iter(self.content);
+                inner.extend_raw_tokens(self.content);
                 Ok(())
             },
             self.source_delim_span.join(),