Initial work towards next-gen collectionItem

packages/cursorless-engine/src/processTargets/ModifierStageFactoryImpl.ts

@@ -21,7 +21,6 @@ import {
   ExcludeInteriorStage,
   InteriorOnlyStage,
 } from "./modifiers/InteriorStage";
-import { ItemStage } from "./modifiers/ItemStage";
 import { LeadingStage, TrailingStage } from "./modifiers/LeadingTrailingStages";
 import { OrdinalScopeStage } from "./modifiers/OrdinalScopeStage";
 import { EndOfStage, StartOfStage } from "./modifiers/PositionStage";
@@ -131,8 +130,6 @@ export class ModifierStageFactoryImpl implements ModifierStageFactory {
     switch (modifier.scopeType.type) {
       case "notebookCell":
         return new NotebookCellStage(modifier);
-      case "collectionItem":
-        return new ItemStage(this.languageDefinitions, this, modifier);
       default:
         // Default to containing syntax scope using tree sitter
         return new LegacyContainingSyntaxScopeStage(

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/CollectionItemScopeHandler/CollectionItemScopeHandler.ts

@@ -0,0 +1,96 @@
+import type { Direction, Position, TextEditor } from "@cursorless/common";
+import { Range, type ScopeType } from "@cursorless/common";
+import type { LanguageDefinitions } from "../../../../languages/LanguageDefinitions";
+import { ScopeTypeTarget } from "../../../targets";
+import { BaseScopeHandler } from "../BaseScopeHandler";
+import { compareTargetScopes } from "../compareTargetScopes";
+import type { TargetScope } from "../scope.types";
+import type { ScopeIteratorRequirements } from "../scopeHandler.types";
+import { getDelimiterOccurrences } from "../SurroundingPairScopeHandler/getDelimiterOccurrences";
+import { getIndividualDelimiters } from "../SurroundingPairScopeHandler/getIndividualDelimiters";
+import type {
+  CollectionItemOccurrence,
+  IndividualDelimiter,
+  IndividualSeparator,
+} from "../SurroundingPairScopeHandler/types";
+import { getCollectionItemOccurrences } from "./getCollectionItemOccurrences";
+
+export class CollectionItemScopeHandler extends BaseScopeHandler {
+  public scopeType: ScopeType = { type: "collectionItem" };
+
+  public readonly iterationScopeType: ScopeType = {
+    type: "oneOf",
+    scopeTypes: [
+      { type: "line" },
+      {
+        type: "surroundingPairInterior",
+        delimiter: "any",
+      },
+    ],
+  };
+  protected isHierarchical = true;
+
+  constructor(
+    private languageDefinitions: LanguageDefinitions,
+    private languageId: string,
+  ) {
+    super();
+  }
+
+  *generateScopeCandidates(
+    editor: TextEditor,
+    position: Position,
+    direction: Direction,
+    _hints: ScopeIteratorRequirements,
+  ): Iterable<TargetScope> {
+    const delimiterOccurrences = getDelimiterOccurrences(
+      this.languageDefinitions.get(this.languageId),
+      editor.document,
+      this.getIndividualDelimiters(),
+    );
+
+    const surroundingPairs = getCollectionItemOccurrences(delimiterOccurrences);
+
+    yield* surroundingPairs
+      .map((pair) => createTargetScope(editor, pair))
+      .sort((a, b) => compareTargetScopes(direction, position, a, b));
+  }
+
+  private getIndividualDelimiters(): (
+    | IndividualDelimiter
+    | IndividualSeparator
+  )[] {
+    return [
+      ...getIndividualDelimiters("collectionBoundary", this.languageId),
+      { side: "separator", text: "," },
+    ];
+  }
+}
+
+function createTargetScope(
+  editor: TextEditor,
+  pair: CollectionItemOccurrence,
+): TargetScope {
+  const contentRange = new Range(
+    pair.openingDelimiterRange.start,
+    pair.closingDelimiterRange.end,
+  );
+  return {
+    editor,
+    domain: contentRange,
+    getTargets(isReversed) {
+      return [
+        new ScopeTypeTarget({
+          scopeTypeType: "collectionItem",
+          editor,
+          isReversed,
+          contentRange,
+          insertionDelimiter: ", ",
+          //   leadingDelimiterRange: pair.leadingDelimiterRange,
+          //   trailingDelimiterRange: pair.trailingDelimiterRange,
+          //   removalRange: pair.removalRange,
+        }),
+      ];
+    },
+  };
+}

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/CollectionItemScopeHandler/getCollectionItemOccurrences.ts

@@ -0,0 +1,151 @@
+import type { SimpleSurroundingPairName } from "@cursorless/common";
+import { DefaultMap } from "@cursorless/common";
+import type {
+  CollectionItemOccurrence,
+  DelimiterOccurrence,
+  IndividualDelimiter,
+  IndividualSeparator,
+} from "../SurroundingPairScopeHandler/types";
+
+interface DelimiterEntry {
+  openingDelimiter: DelimiterOccurrence<IndividualDelimiter>;
+  separator?: DelimiterOccurrence<IndividualSeparator>;
+}
+
+/**
+ * Given a list of occurrences of delimiters, returns a list of occurrences of
+ * surrounding pairs by matching opening and closing delimiters.
+ *
+ * @param delimiterOccurrences A list of occurrences of delimiters
+ * @returns A list of occurrences of surrounding pairs
+ */
+export function getCollectionItemOccurrences(
+  delimiterOccurrences: DelimiterOccurrence<
+    IndividualDelimiter | IndividualSeparator
+  >[],
+): CollectionItemOccurrence[] {
+  const result: CollectionItemOccurrence[] = [];
+
+  /**
+   * A map from delimiter names to occurrences of the opening delimiter
+   */
+  const openingDelimiterOccurrences = new DefaultMap<
+    SimpleSurroundingPairName,
+    DelimiterEntry[]
+  >(() => []);
+
+  for (const occurrence of delimiterOccurrences) {
+    const { delimiterInfo, isDisqualified, textFragmentRange, range } =
+      occurrence;
+
+    if (isDisqualified) {
+      continue;
+    }
+
+    if (delimiterInfo.side === "separator") {
+      const openingDelimiters = Array.from(
+        openingDelimiterOccurrences.values(),
+      ).flat();
+
+      /**
+       * A list of opening delimiters that are relevant to the current occurrence.
+       * We exclude delimiters that are not in the same text fragment range as the
+       * current occurrence.
+       */
+      const relevantOpeningDelimiters = openingDelimiters.filter(
+        ({ openingDelimiter }) =>
+          (textFragmentRange == null &&
+            openingDelimiter.textFragmentRange == null) ||
+          (textFragmentRange != null &&
+            openingDelimiter.textFragmentRange != null &&
+            openingDelimiter.textFragmentRange.isRangeEqual(textFragmentRange)),
+      );
+
+      relevantOpeningDelimiters.sort((a, b) =>
+        a.openingDelimiter.range.start.isBefore(b.openingDelimiter.range.start)
+          ? -1
+          : 1,
+      );
+
+      const lastOpeningDelimiter = relevantOpeningDelimiters.at(-1);
+
+      if (lastOpeningDelimiter == null) {
+        // TODO: in this case, we are yielding entries on a line with no delimiters
+        throw new Error("Not implemented");
+      }
+
+      result.push({
+        openingDelimiterRange:
+          lastOpeningDelimiter.separator?.range ??
+          lastOpeningDelimiter.openingDelimiter.range,
+        closingDelimiterRange: range,
+      });
+
+      lastOpeningDelimiter.separator =
+        occurrence as DelimiterOccurrence<IndividualSeparator>;
+
+      continue;
+    }
+
+    const { side, delimiterName, isSingleLine } = delimiterInfo;
+
+    let openingDelimiters = openingDelimiterOccurrences.get(delimiterName);
+
+    if (isSingleLine) {
+      // If single line, remove all opening delimiters that are not on the same line
+      // as occurrence
+      openingDelimiters = openingDelimiters.filter(
+        (openingDelimiter) =>
+          openingDelimiter.openingDelimiter.range.start.line ===
+          range.start.line,
+      );
+      openingDelimiterOccurrences.set(delimiterName, openingDelimiters);
+    }
+
+    /**
+     * A list of opening delimiters that are relevant to the current occurrence.
+     * We exclude delimiters that are not in the same text fragment range as the
+     * current occurrence.
+     */
+    const relevantOpeningDelimiters = openingDelimiters.filter(
+      (openingDelimiter) =>
+        (textFragmentRange == null &&
+          openingDelimiter.openingDelimiter.textFragmentRange == null) ||
+        (textFragmentRange != null &&
+          openingDelimiter.openingDelimiter.textFragmentRange != null &&
+          openingDelimiter.openingDelimiter.textFragmentRange.isRangeEqual(
+            textFragmentRange,
+          )),
+    );
+
+    if (
+      side === "left" ||
+      (side === "unknown" && relevantOpeningDelimiters.length % 2 === 0)
+    ) {
+      openingDelimiters.push({
+        openingDelimiter:
+          occurrence as DelimiterOccurrence<IndividualDelimiter>,
+      });
+    } else {
+      const openingDelimiter = relevantOpeningDelimiters.at(-1);
+
+      if (openingDelimiter == null) {
+        continue;
+      }
+
+      openingDelimiters.splice(
+        openingDelimiters.lastIndexOf(openingDelimiter),
+        1,
+      );
+
+      result.push({
+        openingDelimiterRange:
+          openingDelimiter.separator?.range ??
+          openingDelimiter.openingDelimiter.range,
+        closingDelimiterRange: range,
+      });
+    }
+  }
+
+  return result;
+}

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/ScopeHandlerFactoryImpl.ts

@@ -5,6 +5,7 @@ import {
   BoundedParagraphScopeHandler,
 } from "./BoundedScopeHandler";
 import { CharacterScopeHandler } from "./CharacterScopeHandler";
+import { CollectionItemScopeHandler } from "./CollectionItemScopeHandler/CollectionItemScopeHandler";
 import { DocumentScopeHandler } from "./DocumentScopeHandler";
 import { IdentifierScopeHandler } from "./IdentifierScopeHandler";
 import { LineScopeHandler } from "./LineScopeHandler";
@@ -103,6 +104,11 @@ export class ScopeHandlerFactoryImpl implements ScopeHandlerFactory {
           scopeType,
           languageId,
         );
+      case "collectionItem":
+        return new CollectionItemScopeHandler(
+          this.languageDefinitions,
+          languageId,
+        );
       case "custom":
         return scopeType.scopeHandler;
       case "instance":

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterOccurrences.ts

@@ -1,7 +1,7 @@
 import { matchAll, Range, type TextDocument } from "@cursorless/common";
 import type { LanguageDefinition } from "../../../../languages/LanguageDefinition";
 import { getDelimiterRegex } from "./getDelimiterRegex";
-import type { DelimiterOccurrence, IndividualDelimiter } from "./types";
+import type { DelimiterOccurrence } from "./types";
 
 /**
  * Finds all occurrences of delimiters of a particular kind in a document.
@@ -11,11 +11,11 @@ import type { DelimiterOccurrence, IndividualDelimiter } from "./types";
  * @param individualDelimiters A list of individual delimiters to search for
  * @returns A list of occurrences of the delimiters
  */
-export function getDelimiterOccurrences(
+export function getDelimiterOccurrences<T extends { text: string }>(
   languageDefinition: LanguageDefinition | undefined,
   document: TextDocument,
-  individualDelimiters: IndividualDelimiter[],
-): DelimiterOccurrence[] {
+  individualDelimiters: T[],
+): DelimiterOccurrence<T>[] {
   if (individualDelimiters.length === 0) {
     return [];
   }
@@ -36,7 +36,7 @@ export function getDelimiterOccurrences(
 
   const text = document.getText();
 
-  return matchAll(text, delimiterRegex, (match): DelimiterOccurrence => {
+  return matchAll(text, delimiterRegex, (match): DelimiterOccurrence<T> => {
     const text = match[0];
     const range = new Range(
       document.positionAt(match.index!),

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getDelimiterRegex.ts

@@ -1,5 +1,4 @@
 import { escapeRegExp, uniq } from "lodash-es";
-import type { IndividualDelimiter } from "./types";
 
 /**
  * Given a list of all possible left / right delimiter instances, returns a regex
@@ -8,7 +7,9 @@ import type { IndividualDelimiter } from "./types";
  * @param individualDelimiters A list of all possible left / right delimiter instances
  * @returns A regex which matches any of the individual delimiters
  */
-export function getDelimiterRegex(individualDelimiters: IndividualDelimiter[]) {
+export function getDelimiterRegex<T extends { text: string }>(
+  individualDelimiters: T[],
+) {
   // Create a regex which is a disjunction of all possible left / right
   // delimiter texts
   const individualDelimiterDisjunct = uniq(

packages/cursorless-engine/src/processTargets/modifiers/scopeHandlers/SurroundingPairScopeHandler/getSurroundingPairOccurrences.ts

@@ -1,6 +1,10 @@
 import type { SimpleSurroundingPairName } from "@cursorless/common";
 import { DefaultMap } from "@cursorless/common";
-import type { DelimiterOccurrence, SurroundingPairOccurrence } from "./types";
+import type {
+  DelimiterOccurrence,
+  IndividualDelimiter,
+  SurroundingPairOccurrence,
+} from "./types";
 
 /**
  * Given a list of occurrences of delimiters, returns a list of occurrences of
@@ -10,7 +14,7 @@ import type { DelimiterOccurrence, SurroundingPairOccurrence } from "./types";
  * @returns A list of occurrences of surrounding pairs
  */
 export function getSurroundingPairOccurrences(
-  delimiterOccurrences: DelimiterOccurrence[],
+  delimiterOccurrences: DelimiterOccurrence<IndividualDelimiter>[],
 ): SurroundingPairOccurrence[] {
   const result: SurroundingPairOccurrence[] = [];
 
@@ -19,7 +23,7 @@ export function getSurroundingPairOccurrences(
    */
   const openingDelimiterOccurrences = new DefaultMap<
     SimpleSurroundingPairName,
-    DelimiterOccurrence[]
+    DelimiterOccurrence<IndividualDelimiter>[]
   >(() => []);
 
   for (const occurrence of delimiterOccurrences) {