[Expressions] Add support of comments (elastic#122457)

* Add comments support to the expressions grammar
* Add typings to the interpreter parser
* Add expressions comments highlighting
* Update canvas to preserve original expression formatting
* Update documentation to cover comments

docs/canvas/canvas-expression-lifecycle.asciidoc

@@ -14,6 +14,7 @@ To use demo dataset available in Canvas to produce a table, run the following ex
 
 [source,text]
 ----
+/* Simple demo table */
 filters
 | demodata
 | table
@@ -24,6 +25,7 @@ This expression starts out with the <<filters_fn, filters>> function, which prov
 
 The filtered <<demodata_fn, demo data>> becomes the _context_ of the next function, <<table_fn, table>>, which creates a table visualization from this data set. The <<table_fn, table>> function isn’t strictly required, but by being explicit, you have the option of providing arguments to control things like the font used in the table. The output of the <<table_fn, table>> function becomes the _context_ of the <<render_fn, render>> function. Like the <<table_fn, table>>, the <<render_fn, render>> function isn’t required either, but it allows access to other arguments, such as styling the border of the element or injecting custom CSS.
 
+It is possible to add comments to the expression by starting them with a `//` sequence or by using `/*` and `*/` to enclose multi-line comments.
 
 [[canvas-function-arguments]]
 === Function arguments

docs/developer/plugin-list.asciidoc

@@ -140,6 +140,9 @@ All the arguments to expression functions need to be serializable, as well as in
 Expression functions should try to stay 'pure'. This makes functions easy to reuse and also 
 make it possible to serialize the whole chain as well as output at every step of execution.
 
+It is possible to add comments to expressions by starting them with a `//` sequence
+or by using `/*` and `*/` to enclose multi-line comments.
+
 Expressions power visualizations in Dashboard and Lens, as well as, every
 *element* in Canvas is backed by an expression.
 

packages/kbn-interpreter/grammar/grammar.peggy

@@ -23,7 +23,7 @@ start
   = expression
 
 expression
-  = space? first:function? rest:('|' space? fn:function { return fn; })* {
+  = blank? first:function? rest:('|' blank? fn:function { return fn; })* {
     return addMeta({
       type: 'expression',
       chain: first ? [first].concat(rest) : []
@@ -44,7 +44,7 @@ function "function"
 /* ----- Arguments ----- */
 
 argument_assignment
-  = name:identifier space? '=' space? value:argument {
+  = name:identifier blank? '=' blank? value:argument {
     return { name, value };
   }
   / value:argument {
@@ -58,7 +58,7 @@ argument
   }
 
 arg_list
-  = args:(space arg:argument_assignment { return arg; })* space? {
+  = args:(blank arg:argument_assignment { return arg; })* blank? {
     return args.reduce((accumulator, { name, value }) => ({
       ...accumulator,
       [name]: (accumulator[name] || []).concat(value)
@@ -82,7 +82,7 @@ phrase
 
 unquoted_string_or_number
   // Make sure we're not matching the beginning of a search
-  = string:unquoted+ { // this also matches nulls, booleans, and numbers
+  = !comment string:unquoted+ { // this also matches nulls, booleans, and numbers
     var result = string.join('');
     // Sort of hacky, but PEG doesn't have backtracking so
     // a null/boolean/number rule is hard to read, and performs worse
@@ -93,8 +93,20 @@ unquoted_string_or_number
     return Number(result);
   }
 
+blank
+  = (space / comment)+
+
 space
-  = [\ \t\r\n]+
+  = [\ \t\r\n]
+
+comment
+  = inline_comment / multiline_comment
+
+inline_comment
+  = "//" [^\n]*
+
+multiline_comment
+  = "/*" (!"*/" .)* "*/"?
 
 unquoted
   = "\\" sequence:([\"'(){}<>\[\]$`|=\ \t\n\r] / "\\") { return sequence; }

packages/kbn-interpreter/src/common/index.ts

@@ -6,14 +6,26 @@
  * Side Public License, v 1.
  */
 
-export type { Ast, ExpressionFunctionAST } from './lib/ast';
-export { fromExpression, toExpression, safeElementFromExpression } from './lib/ast';
+export type {
+  Ast,
+  AstArgument,
+  AstFunction,
+  AstNode,
+  AstWithMeta,
+  AstArgumentWithMeta,
+  AstFunctionWithMeta,
+} from './lib/ast';
+export {
+  fromExpression,
+  isAst,
+  isAstWithMeta,
+  toExpression,
+  safeElementFromExpression,
+} from './lib/ast';
 export { Fn } from './lib/fn';
 export { getType } from './lib/get_type';
 export { castProvider } from './lib/cast';
-// @ts-expect-error
-// @internal
-export { parse } from '../../grammar';
+export { parse } from './lib/parse';
 export { getByAlias } from './lib/get_by_alias';
 export { Registry } from './lib/registry';
 export { addRegistries, register, registryFactory } from './registries';

packages/kbn-interpreter/src/common/lib/ast/ast.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export type AstNode = Ast | AstFunction | AstArgument;
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
+export type Ast = {
+  type: 'expression';
+  chain: AstFunction[];
+};
+
+// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
+export type AstFunction = {
+  type: 'function';
+  function: string;
+  arguments: Record<string, AstArgument[]>;
+};
+
+export type AstArgument = string | boolean | number | Ast;
+
+interface WithMeta<T> {
+  start: number;
+  end: number;
+  text: string;
+  node: T;
+}
+
+type Replace<T, R> = Pick<T, Exclude<keyof T, keyof R>> & R;
+
+type WrapAstArgumentWithMeta<T> = T extends Ast ? AstWithMeta : WithMeta<T>;
+export type AstArgumentWithMeta = WrapAstArgumentWithMeta<AstArgument>;
+
+export type AstFunctionWithMeta = WithMeta<
+  Replace<
+    AstFunction,
+    {
+      arguments: {
+        [key: string]: AstArgumentWithMeta[];
+      };
+    }
+  >
+>;
+
+export type AstWithMeta = WithMeta<
+  Replace<
+    Ast,
+    {
+      chain: AstFunctionWithMeta[];
+    }
+  >
+>;
+
+export function isAstWithMeta(value: any): value is AstWithMeta {
+  return typeof value?.node === 'object';
+}
+
+export function isAst(value: any): value is Ast {
+  return typeof value === 'object' && value?.type === 'expression';
+}