feat: new syntax for template strings (#1311)

### Summary of Changes

- Template strings now use backticks as delimiters. Previously, they
used double quotes like normal strings.
- Template expressions are now enclosed in single curly braces.
Previously, they were enclosed in double curly braces.

---------

Co-authored-by: megalinter-bot <[email protected]>

docs/pipeline-language/expressions/literals.md

@@ -28,7 +28,9 @@ String literals describe text. Their syntax is simply text enclosed by double qu
 | `\0`            | Null character                                                      |
 | `\'`            | Single quote                                                        |
 | `\"`            | Double quote                                                        |
+| `` \` ``        | Backtick                                                            |
 | `\{`            | Opening curly brace (used for [template strings][template-strings]) |
+| `\}`            | Closing curly brace (used for [template strings][template-strings]) |
 | `\\`            | Backslash                                                           |
 | `\uXXXX`        | Unicode character, where `XXXX` is its hexadecimal code             |
 

docs/pipeline-language/expressions/template-strings.md

@@ -3,10 +3,10 @@
 [String literals][string-literals] can only be used to denote a fixed string. Sometimes, however, parts of the string have to be computed and then interpolated into the remaining text. This is done with template strings. Here is an example:
 
 ```sds
-"1 + 2 = {{ 1 + 2 }}"
+`1 + 2 = { 1 + 2 }`
 ```
 
-The syntax for template strings is similar to [string literals][string-literals]: They are also delimited by double quotes, the text can contain escape sequences, and raw newlines can be inserted. The additional syntax are _template expressions_, which are any expression enclosed by `#!sds {{` and `#!sds }}`. There must be no space between the curly braces.
+Template strings are also delimited by backticks, the text can contain escape sequences, and raw newlines can be inserted. The additional syntax are _template expressions_, which are any expression enclosed by `#!sds {` and `#!sds }`.
 
 These template expressions are evaluated, converted to a string and inserted into the template string at their position. The template string in the example above is, hence, equivalent to the [string literal][string-literals] `#!sds "1 + 2 = 3"`.
 

docs/src/lexer/safe_ds_lexer/_safe_ds_lexer.py

@@ -1,4 +1,4 @@
-from pygments.lexer import RegexLexer, words
+from pygments.lexer import RegexLexer, include, words
 from pygments.token import Comment, Keyword, Name, Number, Operator, String, Whitespace
 
 keywords_annotation = ("annotation",)
@@ -88,7 +88,8 @@ class SafeDsLexer(RegexLexer):
         "root": [
             # Literals
             (r"\b([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)\b", Number),
-            (r'"|}}', String, "string"),
+            (r'"', String, "string"),
+            (r"`", String, "template_string"),
             # Keywords
             (
                 words(keywords_annotation, prefix=r"\b", suffix=r"\b"),
@@ -121,6 +122,9 @@ class SafeDsLexer(RegexLexer):
             (r"/\*[\s\S]*?\*/", Comment.Multiline),
             # Whitespace
             (r"\s+", Whitespace),
+            # Block (needed to highlight curly braces in template string expressions)
+            (r"{", Operator, "block"),
+            (r"}", Operator, "#pop"),
         ],
         "annotation": [
             (identifier_regex, Name.Decorator, "#pop"),
@@ -138,7 +142,20 @@ class SafeDsLexer(RegexLexer):
             (identifier_regex, Name.Constant, "#pop"),
         ],
         "string": [
-            (r'([^"{]|\{(?!\{))+', String),
-            (r'\{\{|"', String, "#pop"),
+            (r'(\\"|[^"])+', String),
+            (r'"', String, "#pop"),
+        ],
+        "template_string": [
+            (r"(\\{|\\`|[^`{])+", String),
+            (r"{", String, "template_expression"),
+            (r"`", String, "#pop"),
+        ],
+        "template_expression": [
+            # Order matters
+            (r"}", String, "#pop"),
+            include("root"),
+        ],
+        "block": [
+            include("root"),
         ],
     }

packages/safe-ds-lang/src/language/grammar/safe-ds-token-builder.ts

@@ -0,0 +1,75 @@
+import { DefaultTokenBuilder, GrammarAST, isTokenTypeArray } from 'langium';
+import { TokenType, TokenVocabulary } from 'chevrotain';
+
+// Inspired by https://eclipse-langium.github.io/langium-previews/pr-previews/pr-132/guides/multi-mode-lexing/
+
+// Lexer modes
+const DEFAULT_MODE = 'default';
+const TEMPLATE_STRING_MODE = 'template-string';
+
+// Tokens
+const BLOCK_START = '{';
+const BLOCK_END = '}';
+const TEMPLATE_STRING_START = 'TEMPLATE_STRING_START';
+const TEMPLATE_STRING_INNER = 'TEMPLATE_STRING_INNER';
+const TEMPLATE_STRING_END = 'TEMPLATE_STRING_END';
+
+export class SafeDsTokenBuilder extends DefaultTokenBuilder {
+    override buildTokens(grammar: GrammarAST.Grammar, options?: { caseInsensitive?: boolean }): TokenVocabulary {
+        const tokenTypes = super.buildTokens(grammar, options);
+
+        if (isTokenTypeArray(tokenTypes)) {
+            const defaultModeTokens = tokenTypes.filter(
+                (token) => ![TEMPLATE_STRING_INNER, TEMPLATE_STRING_END].includes(token.name),
+            );
+            const templateStringModeTokens = tokenTypes.filter((token) => ![BLOCK_END].includes(token.name));
+
+            return {
+                modes: {
+                    [DEFAULT_MODE]: defaultModeTokens,
+                    [TEMPLATE_STRING_MODE]: templateStringModeTokens,
+                },
+                defaultMode: DEFAULT_MODE,
+            };
+        } else {
+            /* c8 ignore next 2 */
+            throw new Error('Invalid TokenVocabulary received from DefaultTokenBuilder.');
+        }
+    }
+
+    protected override buildKeywordToken(
+        keyword: GrammarAST.Keyword,
+        terminalTokens: TokenType[],
+        caseInsensitive: boolean,
+    ): TokenType {
+        let tokenType = super.buildKeywordToken(keyword, terminalTokens, caseInsensitive);
+
+        if (tokenType.name === BLOCK_START) {
+            // Enter default mode (for map literals and block lambdas)
+            tokenType.PUSH_MODE = DEFAULT_MODE;
+        } else if (tokenType.name === BLOCK_END) {
+            // Return to previous mode
+            tokenType.POP_MODE = true;
+
+            // BLOCK_END has TEMPLATE_STRING_INNER and TEMPLATE_STRING_END as longer alternatives, which are not valid
+            // in the default mode.
+            delete tokenType.LONGER_ALT;
+        }
+
+        return tokenType;
+    }
+
+    protected override buildTerminalToken(terminal: GrammarAST.TerminalRule): TokenType {
+        let tokenType = super.buildTerminalToken(terminal);
+
+        if (tokenType.name === TEMPLATE_STRING_START) {
+            // Enter template string mode
+            tokenType.PUSH_MODE = TEMPLATE_STRING_MODE;
+        } else if (tokenType.name === TEMPLATE_STRING_END) {
+            // Return to previous mode
+            tokenType.POP_MODE = true;
+        }
+
+        return tokenType;
+    }
+}

packages/safe-ds-lang/src/language/grammar/safe-ds-value-converter.ts

@@ -9,12 +9,14 @@ export class SafeDsValueConverter extends DefaultValueConverter {
                 return ValueConverter.convertBigint(input);
             case 'STRING':
                 return convertString(input, 1, 1);
+            case 'TEMPLATE_STRING_FULL':
+                return convertTemplateStringPart(input);
             case 'TEMPLATE_STRING_START':
-                return convertString(input, 1, 2);
+                return convertTemplateStringPart(input);
             case 'TEMPLATE_STRING_INNER':
-                return convertString(input, 2, 2);
+                return convertTemplateStringPart(input);
             case 'TEMPLATE_STRING_END':
-                return convertString(input, 2, 1);
+                return convertTemplateStringPart(input);
             default:
                 return super.runConverter(rule, input, cstNode);
         }
@@ -39,6 +41,10 @@ const convertString = (input: string, openingDelimiterLength: number, closingDel
     return result;
 };
 
+const convertTemplateStringPart = (input: string): string => {
+    return convertString(input, 1, 1);
+};
+
 /**
  * Handle an escape sequence.
  *
@@ -85,12 +91,11 @@ const replacements = new Map([
     ['\v', '\\v'],
     ['\0', '\\0'],
     ['"', '\\"'],
-    ['{', '\\{'],
     ['\\', '\\\\'],
 ]);
 
 /**
- * Escape a string.
+ * Escape a string. Not applicable to template strings.
  */
 export const escapeString = (input: string): string => {
     let result = '';

packages/safe-ds-lang/src/language/grammar/safe-ds.langium

@@ -873,16 +873,27 @@ interface SdsTemplateString extends SdsExpression {
 }
 
 SdsTemplateString returns SdsTemplateString:
-    expressions+=SdsTemplateStringStart
-    expressions+=SdsExpression?
-    (expressions+=SdsTemplateStringInner expressions+=SdsExpression?)*
-    expressions+=SdsTemplateStringEnd
+    (
+        expressions+=SdsTemplateStringFull
+    ) | (
+        expressions+=SdsTemplateStringStart
+        expressions+=SdsExpression?
+        (expressions+=SdsTemplateStringInner expressions+=SdsExpression?)*
+        expressions+=SdsTemplateStringEnd
+    )
 ;
 
 interface SdsTemplateStringPart extends SdsLiteral {
     value: string
 }
 
+interface SdsTemplateStringFull extends SdsTemplateStringPart {}
+
+SdsTemplateStringFull returns SdsExpression:
+    {SdsTemplateStringFull}
+    value=TEMPLATE_STRING_FULL
+;
+
 interface SdsTemplateStringStart extends SdsTemplateStringPart {}
 
 SdsTemplateStringStart returns SdsExpression:
@@ -1081,20 +1092,20 @@ terminal FLOAT returns number
 terminal fragment DECIMAL_DIGIT: /[0-9]/;
 terminal fragment FLOAT_EXPONENT: ('e' | 'E' )('+' | '-' )? DECIMAL_DIGIT+;
 terminal INT returns bigint: DECIMAL_DIGIT+;
-terminal STRING returns string: STRING_START STRING_TEXT* STRING_END;
-terminal fragment STRING_START: STRING_DELIMITER;
-terminal fragment STRING_END: '{'? STRING_DELIMITER;
-terminal fragment STRING_DELIMITER: '"';
+terminal STRING returns string: '"' STRING_TEXT* '"';
 terminal fragment STRING_TEXT
-    : '{'? ESCAPE_SEQUENCE
-    | /{?[^\\"{]/
+    : ESCAPE_SEQUENCE
+    | /[^\\"]/
 ;
 terminal fragment ESCAPE_SEQUENCE: '\\' .;
-terminal fragment TEMPLATE_EXPRESSION_START: '{{';
-terminal fragment TEMPLATE_EXPRESSION_END: '}}';
-terminal TEMPLATE_STRING_START returns string: STRING_START STRING_TEXT* TEMPLATE_EXPRESSION_START;
-terminal TEMPLATE_STRING_INNER returns string: TEMPLATE_EXPRESSION_END STRING_TEXT* TEMPLATE_EXPRESSION_START;
-terminal TEMPLATE_STRING_END returns string: TEMPLATE_EXPRESSION_END STRING_TEXT* STRING_END;
+terminal TEMPLATE_STRING_FULL returns string: '`' TEMPLATE_STRING_TEXT* '`';
+terminal TEMPLATE_STRING_START returns string: '`' TEMPLATE_STRING_TEXT* '{';
+terminal TEMPLATE_STRING_INNER returns string: '}' TEMPLATE_STRING_TEXT* '{';
+terminal TEMPLATE_STRING_END returns string: '}' TEMPLATE_STRING_TEXT* '`';
+terminal fragment TEMPLATE_STRING_TEXT
+    : ESCAPE_SEQUENCE
+    | /[^\\`{]/
+;
 
 hidden terminal ML_COMMENT: /\/\*[\s\S]*?\*\//;
 hidden terminal SL_COMMENT: /\/\/[^\n\r]*/;

packages/safe-ds-lang/src/language/partialEvaluation/safe-ds-partial-evaluator.ts

@@ -30,9 +30,7 @@ import {
     isSdsSegment,
     isSdsString,
     isSdsTemplateString,
-    isSdsTemplateStringEnd,
-    isSdsTemplateStringInner,
-    isSdsTemplateStringStart,
+    isSdsTemplateStringPart,
     isSdsThis,
     isSdsTypeCast,
     isSdsUnknown,
@@ -214,11 +212,7 @@ export class SafeDsPartialEvaluator {
             return NullConstant;
         } else if (isSdsString(node)) {
             return new StringConstant(node.value);
-        } else if (isSdsTemplateStringStart(node)) {
-            return new StringConstant(node.value);
-        } else if (isSdsTemplateStringInner(node)) {
-            return new StringConstant(node.value);
-        } else if (isSdsTemplateStringEnd(node)) {
+        } else if (isSdsTemplateStringPart(node)) {
             return new StringConstant(node.value);
         } else if (isSdsThis(node) || isSdsUnknown(node)) {
             return UnknownEvaluatedNode;

packages/safe-ds-lang/src/language/safe-ds-module.ts

@@ -59,6 +59,7 @@ import { SafeDsSyntheticProperties } from './helpers/safe-ds-synthetic-propertie
 import { SafeDsLinker } from './scoping/safe-ds-linker.js';
 import { SafeDsCodeActionProvider } from './codeActions/safe-ds-code-action-provider.js';
 import { SafeDsQuickfixProvider } from './codeActions/quickfixes/safe-ds-quickfix-provider.js';
+import { SafeDsTokenBuilder } from './grammar/safe-ds-token-builder.js';
 
 /**
  * Declaration of custom services - add your own service classes here.
@@ -184,6 +185,7 @@ export const SafeDsModule: Module<SafeDsServices, PartialLangiumServices & SafeD
         TypeHierarchyProvider: (services) => new SafeDsTypeHierarchyProvider(services),
     },
     parser: {
+        TokenBuilder: () => new SafeDsTokenBuilder(),
         ValueConverter: () => new SafeDsValueConverter(),
     },
     purity: {

packages/safe-ds-lang/src/language/typing/safe-ds-type-computer.ts

@@ -349,7 +349,8 @@ export class SafeDsTypeComputer {
             return this.computeType(node.type);
         }
 
-        // Partial evaluation (definitely handles SdsBoolean, SdsFloat, SdsInt, SdsNull, and SdsString)
+        // Partial evaluation. This definitely handles SdsBoolean, SdsFloat, SdsInt, SdsNull, SdsString, and
+        // SdsTemplateStringPart.
         const evaluatedNode = this.partialEvaluator.evaluate(node);
         if (evaluatedNode instanceof Constant) {
             return this.factory.createLiteralType(evaluatedNode);