Merge pull request #9418 from keymanapp/chore/feature-gestures-a17s18…

…-master-merge

chore(web): feature-gestures master merge (end of A17S18) 🐵

.github/workflows/crowdin.yml

@@ -2,8 +2,8 @@ name: Upload translation sources to Crowdin translate.keyman.com
 
 on:
   schedule:
-    # At 06:00 every two weeks
-    - cron: '0 6 1,15 * *'
+    # At 06:00 every day. https://crontab.cronhub.io/
+    - cron: '0 6 * * *'
 
 jobs:
   upload-sources-to-crowdin:

.github/workflows/deb-packaging.yml

@@ -1,4 +1,5 @@
 name: "Ubuntu packaging"
+run-name: "Ubuntu packaging - ${{ github.event.client_payload.branch }} (branch ${{ github.head_ref }}), by @${{ github.actor }}"
 on:
   repository_dispatch:
     types: ['deb-release-packaging:*', 'deb-pr-packaging:*']
@@ -17,6 +18,8 @@ jobs:
       VERSION: ${{ steps.version_step.outputs.VERSION }}
       PRERELEASE_TAG: ${{ steps.prerelease_tag.outputs.PRERELEASE_TAG }}
       GIT_SHA: ${{ steps.set_status.outputs.GIT_SHA }}
+      GHA_TEST_BUILD: ${{ github.event.client_payload.isTestBuild }}
+      GHA_BRANCH: ${{ github.event.client_payload.branch }}
     steps:
     - name: Checkout
       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0
@@ -102,7 +105,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        dist: [focal, jammy, kinetic, lunar]
+        # Currently not building mantic until ibus version on mantic stabilizied
+        # and we can provide a patched version
+        # dist: [focal, jammy, lunar, mantic]
+        dist: [focal, jammy, lunar]
         arch: [amd64]
 
     runs-on: ubuntu-latest

HISTORY.md

@@ -1,5 +1,60 @@
 # Keyman Version History
 
+## 17.0.153 alpha 2023-08-03
+
+* docs(windows): Update OS requirement to Windows 10 (#9381)
+* fix(web): maintenance of focus when changing keyboard via Toolbar UI (#9397)
+* chore(linux): Remove Kinetic from GHA (#9399)
+* chore(linux): Properly treat test builds with packaging GHA (#9400)
+
+## 17.0.152 alpha 2023-08-02
+
+* fix(developer): more wasm uset fixes (#9382)
+* docs(windows): corrected nmake cmd for certificates (#9376)
+* chore: add run-name to deb-packaging (#9386)
+* chore: try another variable for reporting (#9388)
+* chore(linux): Remove package build on Jenkins for Keyman 17 (#9380)
+* docs(linux): Add build doc for Keyman Web and Android (#9383)
+
+## 17.0.151 alpha 2023-08-01
+
+* feat(developer) marker steps (#9364)
+* feat(common): marker processing (#9365)
+* chore(linux): Don't fail on parallel builds (#9368)
+* fix(developer): fix breakage from emscripten 3.1.44 (#9375)
+* docs(core): Document how to build Core on Linux (#9328)
+
+## 17.0.150 alpha 2023-07-31
+
+* chore(linux): Update debian changelog (#9358)
+* chore(linux): Fix creation of PRs after uploading to Debian (#9360)
+
+## 17.0.149 alpha 2023-07-30
+
+* fix(core): Better range check for Uni_IsValid() (#9346)
+* chore(core): update documentation in transform logic and processor (#9352)
+
+## 17.0.148 alpha 2023-07-27
+
+* feat(core): merge transform/reorder processing w/ u32 (#9293)
+* chore(developer): make unknown vkey a hint, not error (#9344)
+* chore(linux): Update supported Ubuntu versions (#9341)
+
+## 17.0.147 alpha 2023-07-25
+
+* chore(linux): Update debian changelog (#9327)
+
+## 17.0.146 alpha 2023-07-24
+
+* chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314)
+
+## 17.0.145 alpha 2023-07-21
+
+* fix(linux): Fix logging (#9310)
+* fix(windows): open pdf in an external browser (#9295)
+* fix(linux): Fix installation of keyboards with lang tag `mul` (#9027)
+* fix(web): allows registering precached keyboards (#9304)
+
 ## 17.0.144 alpha 2023-07-20
 
 * refactor(linux): Use better way to get username (#9313)

VERSION.md

@@ -1 +1 @@
-17.0.145
+17.0.154

common/web/lm-worker/build-wrap-and-minify.js

@@ -35,6 +35,7 @@ if(MINIFY) {
     sourcesContent: DEBUG,
     minify: true,
     keepNames: true,
+    target: 'es5',
     outfile: `build/lib/worker-main.polyfilled.min.js`
   });
 }

common/web/types/src/kmx/kmx-plus-builder/build-list.ts

@@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_
  * @returns
  */
 export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF {
+  if (!value) { 
+    return 0; // empty list
+  }
   if(!(value instanceof ListItem)) {
     throw new Error('unexpected value '+ value);
   }

common/web/types/src/kmx/kmx-plus-builder/build-vars.ts

@@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants";
 import { KMXPlusData } from "../kmx-plus.js";
 import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js";
 import { BUILDER_SECTION } from "./builder-section.js";
-import { BUILDER_LIST_REF } from "./build-list.js";
+import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js";
 import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js";
 
 
@@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION {
 /**
 * Builder for the 'vars' section
 */
-export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS {
+export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS {
   if(!kmxplus.vars) {
     return null;
   }
@@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e
     size: constants.length_vars +
       (constants.length_vars_item * kmxplus.vars.totalCount()),
     _offset: 0,
-    markers: 0,
+    markers: build_list_index(sect_list, kmxplus.vars.markers),
     varCount: kmxplus.vars.totalCount(),
     varEntries: [
       ...stringVars,

common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts

@@ -99,7 +99,7 @@ export default class KMXPlusBuilder {
     this.sect.name = build_name(this.file.kmxplus, this.sect.strs);
     this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem);
     this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs);
-    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem);
+    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list);
     this.sect.vkey = build_vkey(this.file.kmxplus);
 
     // Finalize the sect (index) section

common/web/types/src/kmx/kmx-plus.ts

@@ -6,6 +6,7 @@ import { isOneChar, toOneChar, unescapeString } from '../util/util.js';
 import { KMXFile } from './kmx.js';
 import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types';
 import { VariableParser } from '../ldml-keyboard/pattern-parser.js';
+import { MarkerParser } from '../ldml-keyboard/pattern-parser.js';
 
 // Implementation of file structures from /core/src/ldml/C7043_ldml.md
 // Writer in kmx-builder.ts
@@ -292,6 +293,9 @@ export class Vars extends Section {
       return v[0];
     }
   }
+  substituteMarkerString(s : string) : string {
+    return MarkerParser.toSentinelString(s, this.markers);
+  }
 };
 
 /**

common/web/types/src/kmx/string-list.ts

@@ -1,3 +1,4 @@
+import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js';
 import { Strs, StrsItem } from './kmx-plus.js';
 
 /**
@@ -22,7 +23,7 @@ export class ListIndex {
  * A string list in memory. This will be replaced with an index
  * into the string table at finalization.
  */
-export class ListItem extends Array<ListIndex> {
+export class ListItem extends Array<ListIndex> implements OrderedStringList {
   /**
    * Construct a new list from an array of strings.
    * Use List. This is meant to be called by the List.allocString*() functions.
@@ -41,6 +42,9 @@ export class ListItem extends Array<ListIndex> {
         this.push(index);
     }
   }
+  getItemOrder(item: string): number {
+    return this.findIndex(({value}) => value.value === item);
+  }
   isEqual(a: ListItem | string[]): boolean {
     if (a.length != this.length) {
       return false;
@@ -68,7 +72,12 @@ export class ListItem extends Array<ListIndex> {
       return 0;
     }
   }
+  /** for debugging, print as single string */
   toString(): string {
-    return this.map(v => v.value.value).join(' ');
+    return this.toStringArray().join(' ');
+  }
+  /** for debugging, map to string array */
+  toStringArray(): string[] {
+    return this.map(v => v.value.value);
   }
 };

common/web/types/src/ldml-keyboard/pattern-parser.ts

@@ -2,6 +2,7 @@
  * Utilities for transform and marker processing
  */
 
+import { constants } from "@keymanapp/ldml-keyboard-constants";
 import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js";
 
 
@@ -21,6 +22,12 @@ function matchArray(str: string, match: RegExp) : string[] {
  */
 const COMMON_ID = /^[0-9A-Za-z_]{1,32}$/;
 
+/** for use with markers, means an ordering can be determined */
+export interface OrderedStringList {
+  /** @returns the ordering of an item (0..), or -1 if not found */
+  getItemOrder(item : string) : number;
+}
+
 /**
  * Class for helping with markers
  */
@@ -40,6 +47,25 @@ export class MarkerParser {
    */
   public static readonly ANY_MARKER_ID = '.';
 
+  /**
+   * Marker sentinel as a string - U+FFFF
+   */
+  public static readonly SENTINEL = String.fromCodePoint(constants.marker_sentinel);
+
+  /**
+   * Matches all markers.
+   */
+  public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL;
+
+  /** Minimum ID (trailing code unit) */
+  public static readonly MIN_MARKER_INDEX = constants.marker_min_index;
+  /** Index meaning 'any marker' == `\m{.}` */
+  public static readonly ANY_MARKER_INDEX = constants.marker_any_index;
+  /** Maximum usable marker index */
+  public static readonly MAX_MARKER_INDEX = constants.marker_max_index;
+  /** Max count of markers */
+  public static readonly MAX_MARKER_COUNT = constants.marker_max_count;
+
   /**
    * Pattern for matching a marker reference, OR the special marker \m{.}
    */
@@ -51,8 +77,40 @@ export class MarkerParser {
    * @returns `[]` or an array of all markers referenced
    */
   public static allReferences(str: string): string[] {
+    if (!str) {
+      return [];
+    }
     return matchArray(str, this.REFERENCE);
   }
+
+  /** @returns string for marker #n */
+  public static markerOutput(n: number): string {
+    if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) {
+      throw RangeError(`Internal Error: marker index out of range ${n}`);
+    }
+    return this.SENTINEL + String.fromCharCode(n);
+  }
+
+  /** @returns all marker strings as sentinel values */
+  public static toSentinelString(s: string, markers?: OrderedStringList) : string {
+    if (!s) return s;
+    return s.replaceAll(this.REFERENCE, (sub, arg) => {
+      if (arg === MarkerParser.ANY_MARKER_ID) {
+        return MarkerParser.SENTINEL_ALL_MARKERS;
+      }
+      if (!markers) {
+        throw RangeError(`Internal Error: Could not find marker \\m{${arg}} (no markers defined)`);
+      }
+      const order = markers.getItemOrder(arg);
+      if (order === -1) {
+        throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`);
+      } else if(order >= MarkerParser.MAX_MARKER_INDEX) {
+        throw RangeError(`Internal Error: marker \\m{${arg}} has out of range index ${order}`);
+      } else {
+        return MarkerParser.markerOutput(order+1);
+      }
+    });
+  }
 }
 
 /**

common/web/types/test/ldml-keyboard/test-pattern-parser.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import { assert } from 'chai';
-import { ElementParser, ElementSegment, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
+import { ElementParser, ElementSegment, ElementType, MarkerParser, OrderedStringList, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
 
 describe('Test of Pattern Parsers', () => {
   describe('should test MarkerParser', () => {
@@ -51,6 +51,57 @@ describe('Test of Pattern Parsers', () => {
         assert.deepEqual(MarkerParser.allReferences(str), [], `expected no markers: ${str}`);
       }
     });
+    it('should be able to emit sentinel values', () => {
+      assert.equal(MarkerParser.markerOutput(295), '\uFFFF\u0127', 'Wrong sentinel value emitted');
+      assert.equal(MarkerParser.markerOutput(MarkerParser.ANY_MARKER_INDEX), MarkerParser.SENTINEL_ALL_MARKERS, 'Wrong sentinel value emitted for ffff');
+      assert.throws(() => MarkerParser.markerOutput(0)); // below MIN
+      assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX
+    });
+    it('should be able to output sentinel strings', () => {
+      // with nothing (no markers)
+      assert.equal(
+        MarkerParser.toSentinelString(`No markers here!`),
+        `No markers here!`
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(`Marker \\m{sorryNoMarkers}`)
+      );
+      // with a custom class
+      class MyMarkers implements OrderedStringList {
+        getItemOrder(item: string): number {
+          const m : any = {
+            'a': 0,
+            'b': 1,
+            'c': 2,
+            'zzz': 0x2FFFFF,
+          };
+          const o = m[item];
+          if (o === undefined) return -1;
+          return o;
+        }
+      };
+      const markers = new MyMarkers();
+      assert.equal(MarkerParser.toSentinelString(
+        `No markers here!`, markers),
+        `No markers here!`
+      );
+      assert.equal(MarkerParser.toSentinelString(
+        `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers),
+        `Give me \uFFFF\u0001 and \uFFFF\u0003, or \uFFFF\uFFFF.`
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(
+          `Want to see something funny? \\m{zzz}`, // out of range
+          markers
+        )
+      );
+      assert.throws(() =>
+        MarkerParser.toSentinelString(
+          `Want to see something sad? \\m{nothing}`, // non existent
+          markers
+        )
+      );
+    });
   });
   describe('should test VariableParser', () => {
     // same test as for markers

core/include/ldml/keyboardprocessor_ldml.h

@@ -93,6 +93,11 @@
 #define LDML_LENGTH_VARS_ITEM 0x10
 #define LDML_LENGTH_VKEY 0xC
 #define LDML_LENGTH_VKEY_ITEM 0x8
+#define LDML_MARKER_ANY_INDEX 0xFFFF
+#define LDML_MARKER_MAX_COUNT 0xFFFD
+#define LDML_MARKER_MAX_INDEX 0xFFFE
+#define LDML_MARKER_MIN_INDEX 0x1
+#define LDML_MARKER_SENTINEL 0xFFFF
 #define LDML_META_SETTINGS_FALLBACK_OMIT 0x1
 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2
 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4

core/include/ldml/keyboardprocessor_ldml.ts

@@ -613,6 +613,19 @@ class Constants {
     }
     return chars.join('');
   }
+
+  // ---- marker stuff ----
+  /** sentinel value indicating a marker follows */
+  readonly marker_sentinel = 0xFFFF;
+  /** minimum usable marker index */
+  readonly marker_min_index = 0x0001;
+  /** index value referring to the 'any' marker match */
+  readonly marker_any_index = 0xFFFF;
+  /** maximum marker index prior to the 'any' value */
+  readonly marker_max_index = this.marker_any_index - 1;
+  /** maximum count of markers (not including 'any') */
+  readonly marker_max_count = this.marker_max_index - this.marker_min_index;
+
 };
 
 export const constants = new Constants();