[Batch File] Fix line continuation (#3999)

* [Batch File] Fix line continuation This commit applies the following rule: If `^` appears at the end of a line, the linefeed is stripped and the next character is escaped or uses its special meaning, if any. If the next character is also a linefeed, it is treated literal. REM command seems to be working special by line continuation working only directly after the first token. * [Batch File] Only scope ^ punctuation * [Batch File] Treat first caret after continuation literal A caret (`^`) at the beginning of a continued line is printed literal as it is escaped by the continuing caret after stripping newline character. 1. It does not introduce an escape sequence. 2. It is not scoped `constant.character.escape` as it is just printed as is. 3. It differs from `^^` escape sequences by escaping caret already being scoped `punctuation.separator.continuation.line`. Rule ==== Escaped <LF> - <LF> is stripped - The next character is escaped. If at the end of line buffer, then the next line is read and processed by phases 1 and 1.5 and appended to the current one before escaping the next character. Note: Due to `^` being treated special, only at beginning of line, this construct won't be supported properly under all circumstances when embedding batch syntax into Markdown or any other syntax, with probably indented code blocks. * [Batch File] Exclude prototypes from continuation contexts May be of interest, if syntax is extended by template languages.
sublimehq · Jul 9, 2024 · 5dbbc4d · 5dbbc4d
1 parent 16c47e7
commit 5dbbc4d
Show file tree

Hide file tree

Showing 2 changed files with 224 additions and 15 deletions.
diff --git a/Batch File/Batch File.sublime-syntax b/Batch File/Batch File.sublime-syntax
@@ -1,6 +1,7 @@
 %YAML 1.2
 ---
 # https://www.sublimetext.com/docs/syntax.html
+# https://stackoverflow.com/questions/4094699/how-does-the-windows-command-interpreter-cmd-exe-parse-scripts/4095133#4095133
 name: Batch File
 scope: source.dosbatch
 version: 2
@@ -836,7 +837,20 @@ contexts:
     - include: cmd-arg-help
     - include: unquoted-eol-pop
     - match: (?=\S)
-      set: cmd-rem-comment-body
+      set:
+        - cmd-rem-comment-body
+        - cmd-rem-comment-begin
+
+  cmd-rem-comment-begin:
+    # In REM comments only the first continuation directly following
+    # the first token is taken into account.
+    - meta_include_prototype: false
+    - match: (\^)\n
+      captures:
+        1: punctuation.separator.continuation.line.dosbatch
+      set: line-continuation-body
+    - match: (?={{metachar}})
+      pop: 1
 
   cmd-rem-comment-body:
     - meta_include_prototype: false
@@ -1967,8 +1981,8 @@ contexts:
 ###[ PROTOTYPES ]#############################################################
 
   bol-pop:
-    # Note: empty lines are ignored
-    - match: ^(?!$)
+    - meta_include_prototype: false
+    - match: ^
       pop: 1
 
   else-pop:
@@ -1981,9 +1995,21 @@ contexts:
       pop: 1
 
   line-continuations:
-    - match: \^\n
-      scope: punctuation.separator.continuation.line.dosbatch
-      push: bol-pop
+    - match: (\^)\n
+      captures:
+        1: punctuation.separator.continuation.line.dosbatch
+      push: line-continuation-body
+
+  line-continuation-body:
+    - meta_include_prototype: false
+    # The first linefeed diretly following a continuation is treated literal.
+    # Note: Must consume \n to work.
+    - match: ^\n
+      set: bol-pop
+    # The first caret at a continued line is escaped by the continuation caret
+    # and thus printed literal.
+    - match: ^\^?
+      pop: 1
 
   eoc-pop:
     # Note: An end of command appears in unquoted regions only!

diff --git a/Batch File/tests/syntax_test_batch_file.bat b/Batch File/tests/syntax_test_batch_file.bat
@@ -1,7 +1,7 @@
 :: SYNTAX TEST "Packages/Batch File/Batch File.sublime-syntax"
 
 
-:::: [ Comments ] :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:::: [ REM Comments ] :::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 
    REM/? ignored
 :: ^^^^^^^^^^^^^ meta.command.rem.dosbatch
@@ -47,6 +47,11 @@
    not a comment
 :: ^^^^^^^^^^^^^^ - comment
 
+   REM ^
+   Line^
+   continuation after first token
+:: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.command.rem.dosbatch comment.line.rem.dosbatch
+
 REM
    not a comment
 :: ^^^^^^^^^^^^^^ - comment
@@ -59,11 +64,96 @@ REM This & and | echo "is commented out" ^
 :: <- keyword.declaration.rem.dosbatch
 ::  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ meta.command.rem.dosbatch comment.line.rem.dosbatch
 
+::: Test Case : single token continued at next line
+
+REM Line^
+continuation
+:: <- meta.command.rem.dosbatch comment.line.rem.dosbatch
+:: ^^^^^^^^^^ meta.command.rem.dosbatch comment.line.rem.dosbatch
+
+::: Test Case : single token continued after empty line
+
+REM Line^
+
+continuation
+:: <- meta.command.rem.dosbatch comment.line.rem.dosbatch
+:: ^^^^^^^^^^ meta.command.rem.dosbatch comment.line.rem.dosbatch
+
+::: Test Case : no more continuation after 2nd token
+
+REM Line^
+
+continuation^
+not a comment
+:: <- - comment
+:: ^^^^^^^^^^ - comment
+
+::: Test Case : no continuation after 2nd empty line
+
+REM Line^
+
+
+not a comment
+:: <- - comment
+:: ^^^^^^^ - comment
+
+::: Test Case : continuation beginning with literal caret
+
+REM Line^
+^continuation
+not a comment
+:: <- - comment
+:: ^^^^^^^^^^ - comment
+
+::: Test Case : continuation beginning escaping caret after empty line
+
+REM Line^
+
+^continuation
+not a comment
+:: <- - comment
+:: ^^^^^^^^^^ - comment
+
+::: Test Case : continuation with only litaral caret, no recursive continuation
+
+REM Line^
+^
+not a comment
+:: <- - comment
+:: ^^^^^^^^^^ - comment
+
+::: Test Case : continuation with only escaping caret after empty line, no recursive continuation
+
+REM Line^
+
+^
+not a comment
+:: <- - comment
+:: ^^^^^^^^^^ - comment
+
+::: Test Case : no line continuation after 2nd token #1
+
 REM No line ^
 continuation
 :: <- - comment
 :: ^^^^^^^^^^ - comment
 
+::: Test Case : no line continuation after 2nd token #2
+
+REM No line ^
+^
+:: <- meta.function-call.identifier.dosbatch variable.function.dosbatch punctuation.separator.continuation.line.dosbatch
+
+::: Test Case : no line continuation after 2nd token #3
+
+REM No line ^
+^
+not a comment
+:: <- - comment
+::^^^^^^^^^^^^ - comment
+
+:::: [ Label Comments ] :::::::::::::::::::::::::::::::::::::::::::::::::::::::
+
    ::: Me too!
 :: ^^^ punctuation.definition.comment.dosbatch
 :: ^^^^^^^^^^^^ comment.line.colon.dosbatch
@@ -134,6 +224,16 @@ continuation
    Me too!
 :: ^^^^^^^^ comment.line.colon.dosbatch
 
+   ::: ^
+   ^
+   Me too!
+:: ^^^^^^^^ comment.line.colon.dosbatch
+
+::: ^
+^
+   Not me, though.
+:: ^^^^^^^^^^^^^^^^ - comment
+
    ::^
    Me too!
 :: ^^^^^^^^ comment.line.colon.dosbatch
@@ -148,6 +248,16 @@ continuation
    Me too!
 :: ^^^^^^^^ comment.line.colon.dosbatch
 
+   :: ^
+   ^
+   Me too!
+:: ^^^^^^^^ comment.line.colon.dosbatch
+
+:: ^
+^
+   Not me, though.
+:: ^^^^^^^^^^^^^^^^ - comment
+
    : ^
    Me too!
 :: ^^^^^^^^ comment.line.colon.dosbatch
@@ -162,6 +272,16 @@ continuation
    Me too!
 :: ^^^^^^^^ comment.line.colon.dosbatch
 
+   : ^
+   ^
+   Me too!
+:: ^^^^^^^^ comment.line.colon.dosbatch
+
+: ^
+^
+   Not me, though.
+:: ^^^^^^^^^^^^^^^^ - comment
+
    :> ignored content ( & | )
 :: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ comment.line.colon.dosbatch
 :: ^^ punctuation.definition.comment.dosbatch
@@ -199,7 +319,8 @@ ECHO |:: Not a comment
 
 ECHO : Not a comment ^
 ::   ^^^^^^^^^^^^^^^^^^ - comment
-::                   ^^ punctuation.separator.continuation.line.dosbatch
+::                   ^ punctuation.separator.continuation.line.dosbatch
+::                    ^ - punctuation
 
 ECHO : Not a comment ^
 :: Me not, too
@@ -481,6 +602,22 @@ ECHO : Not a comment ^
 :: ^ punctuation.definition.variable.dosbatch
 :: ^^^^ variable.label.dosbatch - keyword
 
+   CALL ^
+
+   :EOF
+:: ^^^^ meta.function-call.identifier.dosbatch
+::     ^ - meta.function-call
+:: ^ punctuation.definition.variable.dosbatch
+:: ^^^^ variable.label.dosbatch - keyword
+
+   CALL ^
+
+
+   :EOF
+:: ^^^^ - meta.function-call
+:: ^ punctuation.definition.label.dosbatch
+:: ^^^^ entity.name.label.dosbatch
+
    CALL :foo 10 %1
 ::^ - meta.function-call
 :: ^^^^^ meta.function-call.dosbatch
@@ -1334,7 +1471,8 @@ is a #@$虎" strange label
 
    IF^
 :: ^^ - keyword.control.conditional
-::   ^^ punctuation.separator.continuation.line.dosbatch
+::   ^ punctuation.separator.continuation.line.dosbatch
+::    ^ - punctuation
 
    IF ^
    NOT EXIST "C:\file.log"
@@ -1889,7 +2027,8 @@ is a #@$虎" strange label
 :: ^^^^^^ meta.function-call.identifier.dosbatch variable.function.dosbatch
 ::       ^^^^^^^^^^ meta.function-call.arguments.dosbatch
    out^
-::    ^^ punctuation.separator.continuation.line.dosbatch
+::    ^ punctuation.separator.continuation.line.dosbatch
+::     ^ - punctuation
 
    out^
 put arg1 arg2
@@ -2065,7 +2204,8 @@ put arg1 arg2
 
    command arg^
 ::         ^^^^^ meta.function-call.arguments.dosbatch
-::            ^^ punctuation.separator.continuation.line.dosbatch
+::            ^ punctuation.separator.continuation.line.dosbatch
+::             ^ - punctuation
 
    command arg^
    for
@@ -2510,6 +2650,7 @@ put arg1 arg2
 :: ^^^^ meta.function-call.identifier.dosbatch support.function.builtin.dosbatch
 ::     ^^^ meta.function-call.arguments.dosbatch
 ::      ^ punctuation.separator.continuation.line.dosbatch
+::       ^ - punctuation
 
    ECHO ^
    /? ignored
@@ -2652,6 +2793,44 @@ put arg1 arg2
 ::                           ^^^ meta.string.dosbatch string.unquoted.dosbatch
 ::                              ^ - meta.command - meta.string - string
 
+   ECHO line ^
+::      ^^^^^^^ meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+::           ^ punctuation.separator.continuation.line.dosbatch
+::            ^ - punctuation
+
+   ECHO line ^
+   continuation
+:: <- meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+::^^^^^^^^^^^^^ meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+
+   ECHO line ^
+
+   continuation
+:: <- meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+::^^^^^^^^^^^^^ meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+
+   ECHO line ^
+   ^
+:: <- meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+::^^^ meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+:: ^ punctuation.separator.continuation.line.dosbatch
+::  ^ - punctuation
+
+   ECHO line ^
+   ^
+   continuation
+:: <- meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+::^^^^^^^^^^^^^ meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch
+
+ECHO line ^
+^
+:: <- meta.command.echo.dosbatch meta.function-call.arguments.dosbatch meta.string.dosbatch string.unquoted.dosbatch - punctuation
+
+ECHO line ^
+^
+no continuation
+:: <- - meta.command.echo - meta.string - string
+::^^^^^^^^^^^^^ - meta.command.echo
 
 :::: [ ECHO escaped characters ]:::::::::::::::::::::::::::::::::::::::::::::::
 
@@ -3417,7 +3596,8 @@ put arg1 arg2
 ::     ^^^^ variable.other.readwrite.dosbatch
 ::         ^ keyword.operator.assignment.dosbatch
 ::          ^^^^^^ string.unquoted.dosbatch
-::              ^^ punctuation.separator.continuation.line.dosbatch
+::              ^ punctuation.separator.continuation.line.dosbatch
+::               ^ - punctuation
 
    set foo"="bar^
    baz
@@ -3446,7 +3626,8 @@ put arg1 arg2
 ::     ^^^^^ variable.other.readwrite.dosbatch
 ::          ^ keyword.operator.assignment.dosbatch
 ::           ^^^^^ string.unquoted.dosbatch
-::              ^^ punctuation.separator.continuation.line.dosbatch
+::              ^ punctuation.separator.continuation.line.dosbatch
+::               ^ - punctuation
 
    set fo"o"=bar^
    baz
@@ -3474,7 +3655,8 @@ put arg1 arg2
 ::     ^^^^ variable.other.readwrite.dosbatch
 ::          ^ keyword.operator.assignment.dosbatch
 ::           ^^^^^^^ meta.string.dosbatch string.unquoted.dosbatch
-::                ^^ punctuation.separator.continuation.line.dosbatch
+::                ^ punctuation.separator.continuation.line.dosbatch
+::                 ^ - punctuation
 
    set fo"o"="bar"^
    baz
@@ -4556,7 +4738,8 @@ put arg1 arg2
 ::        ^^^^^ variable.other.readwrite.dosbatch
 ::             ^ keyword.operator.assignment.dosbatch
 ::              ^^^^^^^^^ string.unquoted.dosbatch
-::                     ^^ punctuation.separator.continuation.line.dosbatch
+::                     ^ punctuation.separator.continuation.line.dosbatch
+::                      ^ - punctuation
 
    :: even number of quotes in l-value
    :: unquoted value