diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml index a7d1c1df192..27b4d705ba3 100644 --- a/.github/workflows/crowdin.yml +++ b/.github/workflows/crowdin.yml @@ -2,8 +2,8 @@ name: Upload translation sources to Crowdin translate.keyman.com on: schedule: - # At 06:00 every two weeks - - cron: '0 6 1,15 * *' + # At 06:00 every day. https://crontab.cronhub.io/ + - cron: '0 6 * * *' jobs: upload-sources-to-crowdin: diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index ddbdc66dc05..7106ed49b3f 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -1,4 +1,5 @@ name: "Ubuntu packaging" +run-name: "Ubuntu packaging - ${{ github.event.client_payload.branch }} (branch ${{ github.head_ref }}), by @${{ github.actor }}" on: repository_dispatch: types: ['deb-release-packaging:*', 'deb-pr-packaging:*'] @@ -17,6 +18,8 @@ jobs: VERSION: ${{ steps.version_step.outputs.VERSION }} PRERELEASE_TAG: ${{ steps.prerelease_tag.outputs.PRERELEASE_TAG }} GIT_SHA: ${{ steps.set_status.outputs.GIT_SHA }} + GHA_TEST_BUILD: ${{ github.event.client_payload.isTestBuild }} + GHA_BRANCH: ${{ github.event.client_payload.branch }} steps: - name: Checkout uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0 @@ -102,7 +105,10 @@ jobs: strategy: fail-fast: true matrix: - dist: [focal, jammy, kinetic, lunar] + # Currently not building mantic until ibus version on mantic stabilizied + # and we can provide a patched version + # dist: [focal, jammy, lunar, mantic] + dist: [focal, jammy, lunar] arch: [amd64] runs-on: ubuntu-latest diff --git a/HISTORY.md b/HISTORY.md index 335f2ccc796..3da35e3d94c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,60 @@ # Keyman Version History +## 17.0.153 alpha 2023-08-03 + +* docs(windows): Update OS requirement to Windows 10 (#9381) +* fix(web): maintenance of focus when changing keyboard via Toolbar UI (#9397) +* chore(linux): Remove Kinetic from GHA (#9399) +* chore(linux): Properly treat test builds with packaging GHA (#9400) + +## 17.0.152 alpha 2023-08-02 + +* fix(developer): more wasm uset fixes (#9382) +* docs(windows): corrected nmake cmd for certificates (#9376) +* chore: add run-name to deb-packaging (#9386) +* chore: try another variable for reporting (#9388) +* chore(linux): Remove package build on Jenkins for Keyman 17 (#9380) +* docs(linux): Add build doc for Keyman Web and Android (#9383) + +## 17.0.151 alpha 2023-08-01 + +* feat(developer) marker steps (#9364) +* feat(common): marker processing (#9365) +* chore(linux): Don't fail on parallel builds (#9368) +* fix(developer): fix breakage from emscripten 3.1.44 (#9375) +* docs(core): Document how to build Core on Linux (#9328) + +## 17.0.150 alpha 2023-07-31 + +* chore(linux): Update debian changelog (#9358) +* chore(linux): Fix creation of PRs after uploading to Debian (#9360) + +## 17.0.149 alpha 2023-07-30 + +* fix(core): Better range check for Uni_IsValid() (#9346) +* chore(core): update documentation in transform logic and processor (#9352) + +## 17.0.148 alpha 2023-07-27 + +* feat(core): merge transform/reorder processing w/ u32 (#9293) +* chore(developer): make unknown vkey a hint, not error (#9344) +* chore(linux): Update supported Ubuntu versions (#9341) + +## 17.0.147 alpha 2023-07-25 + +* chore(linux): Update debian changelog (#9327) + +## 17.0.146 alpha 2023-07-24 + +* chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314) + +## 17.0.145 alpha 2023-07-21 + +* fix(linux): Fix logging (#9310) +* fix(windows): open pdf in an external browser (#9295) +* fix(linux): Fix installation of keyboards with lang tag `mul` (#9027) +* fix(web): allows registering precached keyboards (#9304) + ## 17.0.144 alpha 2023-07-20 * refactor(linux): Use better way to get username (#9313) diff --git a/VERSION.md b/VERSION.md index 56d15fc7ec3..0f0cc2cceb6 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -17.0.145 \ No newline at end of file +17.0.154 \ No newline at end of file diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js index 009a8f496c0..1572238b7a8 100644 --- a/common/web/lm-worker/build-wrap-and-minify.js +++ b/common/web/lm-worker/build-wrap-and-minify.js @@ -35,6 +35,7 @@ if(MINIFY) { sourcesContent: DEBUG, minify: true, keepNames: true, + target: 'es5', outfile: `build/lib/worker-main.polyfilled.min.js` }); } diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts index e770e7b2783..26659f8eb6a 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts @@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_ * @returns */ export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF { + if (!value) { + return 0; // empty list + } if(!(value instanceof ListItem)) { throw new Error('unexpected value '+ value); } diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts index 5f70689e962..daa0e4a46c2 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts @@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlusData } from "../kmx-plus.js"; import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js"; import { BUILDER_SECTION } from "./builder-section.js"; -import { BUILDER_LIST_REF } from "./build-list.js"; +import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js"; import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js"; @@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION { /** * Builder for the 'vars' section */ -export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS { +export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS { if(!kmxplus.vars) { return null; } @@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e size: constants.length_vars + (constants.length_vars_item * kmxplus.vars.totalCount()), _offset: 0, - markers: 0, + markers: build_list_index(sect_list, kmxplus.vars.markers), varCount: kmxplus.vars.totalCount(), varEntries: [ ...stringVars, diff --git a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts index 082e5495eb3..3cc58c9fe75 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts @@ -99,7 +99,7 @@ export default class KMXPlusBuilder { this.sect.name = build_name(this.file.kmxplus, this.sect.strs); this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem); this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs); - this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem); + this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list); this.sect.vkey = build_vkey(this.file.kmxplus); // Finalize the sect (index) section diff --git a/common/web/types/src/kmx/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus.ts index 9ca0a13ad78..795cebc6965 100644 --- a/common/web/types/src/kmx/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus.ts @@ -6,6 +6,7 @@ import { isOneChar, toOneChar, unescapeString } from '../util/util.js'; import { KMXFile } from './kmx.js'; import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types'; import { VariableParser } from '../ldml-keyboard/pattern-parser.js'; +import { MarkerParser } from '../ldml-keyboard/pattern-parser.js'; // Implementation of file structures from /core/src/ldml/C7043_ldml.md // Writer in kmx-builder.ts @@ -292,6 +293,9 @@ export class Vars extends Section { return v[0]; } } + substituteMarkerString(s : string) : string { + return MarkerParser.toSentinelString(s, this.markers); + } }; /** diff --git a/common/web/types/src/kmx/string-list.ts b/common/web/types/src/kmx/string-list.ts index 4b5716b85d7..f9f4b46e912 100644 --- a/common/web/types/src/kmx/string-list.ts +++ b/common/web/types/src/kmx/string-list.ts @@ -1,3 +1,4 @@ +import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js'; import { Strs, StrsItem } from './kmx-plus.js'; /** @@ -22,7 +23,7 @@ export class ListIndex { * A string list in memory. This will be replaced with an index * into the string table at finalization. */ -export class ListItem extends Array { +export class ListItem extends Array implements OrderedStringList { /** * Construct a new list from an array of strings. * Use List. This is meant to be called by the List.allocString*() functions. @@ -41,6 +42,9 @@ export class ListItem extends Array { this.push(index); } } + getItemOrder(item: string): number { + return this.findIndex(({value}) => value.value === item); + } isEqual(a: ListItem | string[]): boolean { if (a.length != this.length) { return false; @@ -68,7 +72,12 @@ export class ListItem extends Array { return 0; } } + /** for debugging, print as single string */ toString(): string { - return this.map(v => v.value.value).join(' '); + return this.toStringArray().join(' '); + } + /** for debugging, map to string array */ + toStringArray(): string[] { + return this.map(v => v.value.value); } }; diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts index 1b41b9416c4..faee3b15946 100644 --- a/common/web/types/src/ldml-keyboard/pattern-parser.ts +++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts @@ -2,6 +2,7 @@ * Utilities for transform and marker processing */ +import { constants } from "@keymanapp/ldml-keyboard-constants"; import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js"; @@ -21,6 +22,12 @@ function matchArray(str: string, match: RegExp) : string[] { */ const COMMON_ID = /^[0-9A-Za-z_]{1,32}$/; +/** for use with markers, means an ordering can be determined */ +export interface OrderedStringList { + /** @returns the ordering of an item (0..), or -1 if not found */ + getItemOrder(item : string) : number; +} + /** * Class for helping with markers */ @@ -40,6 +47,25 @@ export class MarkerParser { */ public static readonly ANY_MARKER_ID = '.'; + /** + * Marker sentinel as a string - U+FFFF + */ + public static readonly SENTINEL = String.fromCodePoint(constants.marker_sentinel); + + /** + * Matches all markers. + */ + public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL; + + /** Minimum ID (trailing code unit) */ + public static readonly MIN_MARKER_INDEX = constants.marker_min_index; + /** Index meaning 'any marker' == `\m{.}` */ + public static readonly ANY_MARKER_INDEX = constants.marker_any_index; + /** Maximum usable marker index */ + public static readonly MAX_MARKER_INDEX = constants.marker_max_index; + /** Max count of markers */ + public static readonly MAX_MARKER_COUNT = constants.marker_max_count; + /** * Pattern for matching a marker reference, OR the special marker \m{.} */ @@ -51,8 +77,40 @@ export class MarkerParser { * @returns `[]` or an array of all markers referenced */ public static allReferences(str: string): string[] { + if (!str) { + return []; + } return matchArray(str, this.REFERENCE); } + + /** @returns string for marker #n */ + public static markerOutput(n: number): string { + if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) { + throw RangeError(`Internal Error: marker index out of range ${n}`); + } + return this.SENTINEL + String.fromCharCode(n); + } + + /** @returns all marker strings as sentinel values */ + public static toSentinelString(s: string, markers?: OrderedStringList) : string { + if (!s) return s; + return s.replaceAll(this.REFERENCE, (sub, arg) => { + if (arg === MarkerParser.ANY_MARKER_ID) { + return MarkerParser.SENTINEL_ALL_MARKERS; + } + if (!markers) { + throw RangeError(`Internal Error: Could not find marker \\m{${arg}} (no markers defined)`); + } + const order = markers.getItemOrder(arg); + if (order === -1) { + throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`); + } else if(order >= MarkerParser.MAX_MARKER_INDEX) { + throw RangeError(`Internal Error: marker \\m{${arg}} has out of range index ${order}`); + } else { + return MarkerParser.markerOutput(order+1); + } + }); + } } /** diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts index f8b881b6448..9d00ab82bf5 100644 --- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts +++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { ElementParser, ElementSegment, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; +import { ElementParser, ElementSegment, ElementType, MarkerParser, OrderedStringList, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; describe('Test of Pattern Parsers', () => { describe('should test MarkerParser', () => { @@ -51,6 +51,57 @@ describe('Test of Pattern Parsers', () => { assert.deepEqual(MarkerParser.allReferences(str), [], `expected no markers: ${str}`); } }); + it('should be able to emit sentinel values', () => { + assert.equal(MarkerParser.markerOutput(295), '\uFFFF\u0127', 'Wrong sentinel value emitted'); + assert.equal(MarkerParser.markerOutput(MarkerParser.ANY_MARKER_INDEX), MarkerParser.SENTINEL_ALL_MARKERS, 'Wrong sentinel value emitted for ffff'); + assert.throws(() => MarkerParser.markerOutput(0)); // below MIN + assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX + }); + it('should be able to output sentinel strings', () => { + // with nothing (no markers) + assert.equal( + MarkerParser.toSentinelString(`No markers here!`), + `No markers here!` + ); + assert.throws(() => + MarkerParser.toSentinelString(`Marker \\m{sorryNoMarkers}`) + ); + // with a custom class + class MyMarkers implements OrderedStringList { + getItemOrder(item: string): number { + const m : any = { + 'a': 0, + 'b': 1, + 'c': 2, + 'zzz': 0x2FFFFF, + }; + const o = m[item]; + if (o === undefined) return -1; + return o; + } + }; + const markers = new MyMarkers(); + assert.equal(MarkerParser.toSentinelString( + `No markers here!`, markers), + `No markers here!` + ); + assert.equal(MarkerParser.toSentinelString( + `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers), + `Give me \uFFFF\u0001 and \uFFFF\u0003, or \uFFFF\uFFFF.` + ); + assert.throws(() => + MarkerParser.toSentinelString( + `Want to see something funny? \\m{zzz}`, // out of range + markers + ) + ); + assert.throws(() => + MarkerParser.toSentinelString( + `Want to see something sad? \\m{nothing}`, // non existent + markers + ) + ); + }); }); describe('should test VariableParser', () => { // same test as for markers diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index a92731bfa3e..262a18e6f05 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -93,6 +93,11 @@ #define LDML_LENGTH_VARS_ITEM 0x10 #define LDML_LENGTH_VKEY 0xC #define LDML_LENGTH_VKEY_ITEM 0x8 +#define LDML_MARKER_ANY_INDEX 0xFFFF +#define LDML_MARKER_MAX_COUNT 0xFFFD +#define LDML_MARKER_MAX_INDEX 0xFFFE +#define LDML_MARKER_MIN_INDEX 0x1 +#define LDML_MARKER_SENTINEL 0xFFFF #define LDML_META_SETTINGS_FALLBACK_OMIT 0x1 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4 diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index 4d597008958..fdfc63da180 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -613,6 +613,19 @@ class Constants { } return chars.join(''); } + + // ---- marker stuff ---- + /** sentinel value indicating a marker follows */ + readonly marker_sentinel = 0xFFFF; + /** minimum usable marker index */ + readonly marker_min_index = 0x0001; + /** index value referring to the 'any' marker match */ + readonly marker_any_index = 0xFFFF; + /** maximum marker index prior to the 'any' value */ + readonly marker_max_index = this.marker_any_index - 1; + /** maximum count of markers (not including 'any') */ + readonly marker_max_count = this.marker_max_index - this.marker_min_index; + }; export const constants = new Constants(); diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index e5132193f56..b02fec77d85 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -18,6 +18,20 @@ namespace km { namespace kbp { namespace kmx { +/** + * \def KMXPLUS_DEBUG_LOAD set to 1 to print messages on KMXPLUS loading. + * Off by default. +*/ +#ifndef KMXPLUS_DEBUG_LOAD +#define KMXPLUS_DEBUG_LOAD 0 +#endif + +#if KMXPLUS_DEBUG_LOAD +#define DebugLoad(msg,...) DebugLog(msg, __VA_ARGS__) +#else +#define DebugLoad(msg,...) +#endif + // double check these modifier mappings static_assert(LCTRLFLAG == LDML_KEYS_MOD_CTRLL, "LDML modifier bitfield vs. kmx_file.h #define mismatch"); static_assert(RCTRLFLAG == LDML_KEYS_MOD_CTRLR, "LDML modifier bitfield vs. kmx_file.h #define mismatch"); @@ -231,7 +245,7 @@ COMP_KMXPLUS_DISP::valid(KMX_DWORD _kmn_unused(length)) const { DebugLog("disp: baseCharacter str#0x%X", baseCharacter); } for (KMX_DWORD i=0; i str0x%X", i, entries[i].to, entries[i].display); + DebugLoad("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display); if (entries[i].to == 0 || entries[i].display == 0) { DebugLog("disp to: or display: has a zero string"); assert(false); @@ -265,7 +279,7 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const { return false; } // TODO-LDML: validate valid UTF-16LE? - DebugLog("strs #0x%X: '%s'", i, Debug_UnicodeString(start)); + DebugLoad("strs #0x%X: '%s'", i, Debug_UnicodeString(start)); } return true; } @@ -740,7 +754,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { for(KMX_DWORD i = 0; is_valid && i < key2->keyCount; i++) { const auto &key = keys[i]; // is the count off the end? - DebugLog( " id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here + DebugLoad( " id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here if (key.flicks >0 && key.flicks >= key2->flicksCount) { DebugLog("key[%d] has invalid flicks index %d", i, key.flicks); is_valid = false; @@ -750,7 +764,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { for(KMX_DWORD i = 0; is_valid && i < key2->flicksCount; i++) { const auto &e = flickLists[i]; // is the count off the end? - DebugLog(" %d: index %d, count %d", i, e.flick, e.count); + DebugLoad(" %d: index %d, count %d", i, e.flick, e.count); if (i == 0) { if (e.flick != 0 || e.count != 0) { DebugLog("Error: Invalid Flick #0"); @@ -765,18 +779,22 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { } for(KMX_DWORD i = 0; is_valid && i < key2->flickCount; i++) { const auto &e = flickElements[i]; - // is the count off the end? - DebugLog(" %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags); + // validate to is present + if (e.to == 0 || e.directions == 0) { + DebugLog("flickElement[%d] has empty to=%0x%X or directions=%0x%X", i, e.to, e.directions); + is_valid = false; + assert(is_valid); + } + DebugLoad(" %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags); } // now the kmap - DebugLog(" kmap count: #0x%X", key2->kmapCount); + DebugLoad(" kmap count: #0x%X", key2->kmapCount); for (KMX_DWORD i = 0; i < key2->kmapCount; i++) { - // These are pretty noisy, drop them from the log - // DebugLog(" #0x%d\n", i); + DebugLoad(" #0x%d\n", i); auto &entry = kmap[i]; - // DebugLog(" vkey\t0x%X", entry.vkey); - // DebugLog(" mod\t0x%X", entry.mod); - // DebugLog(" key\t#0x%X", entry.key); + DebugLoad(" vkey\t0x%X", entry.vkey); + DebugLoad(" mod\t0x%X", entry.mod); + DebugLoad(" key\t#0x%X", entry.key); if (!LDML_IS_VALID_MODIFIER_BITS(entry.mod)) { DebugLog("Invalid modifier value"); assert(false); @@ -928,10 +946,12 @@ COMP_KMXPLUS_LIST_Helper::setList(const COMP_KMXPLUS_LIST *newList) { assert(is_valid); } } +#if KMXPLUS_DEBUG_LOAD for (KMX_DWORD i = 0; is_valid && i < list->indexCount; i++) { const auto &e = indices[i]; - DebugLog(" index %d: str 0x%X", i, e); + DebugLoad(" index %d: str 0x%X", i, e); } +#endif } // Return results DebugLog("COMP_KMXPLUS_LIST_Helper.setList(): %s", is_valid ? "valid" : "invalid"); @@ -969,7 +989,13 @@ COMP_KMXPLUS_USET::valid(KMX_DWORD _kmn_unused(length)) const { assert(false); return false; } - return true; + return true; // see helper +} + +COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(KMX_DWORD s, KMX_DWORD e) : start(s), end(e) { +} + +COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE &other) : start(other.start), end(other.end) { } COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(false), usets(nullptr), ranges(nullptr) { @@ -977,7 +1003,7 @@ COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(f bool COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) { - DebugLog("validating newUset=%p", newUset); + DebugLoad("validating newUset=%p", newUset); is_valid = true; if (newUset == nullptr) { // Note: kmx_plus::kmx_plus has already called section_from_bytes() @@ -1017,9 +1043,13 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) { } else { /** last lastEnd value */ KMX_DWORD lastEnd = 0x0; - for (KMX_DWORD r = 0; r < e.count; r++) { + for (KMX_DWORD r = 0; is_valid && r < e.count; r++) { const auto &range = ranges[e.range + r]; // already range-checked 'r' above - if (range.end < range.start) { + if (!Uni_IsValid(range.start, range.end)) { + DebugLog("uset[%d][%d] not valid: [U+%04X-U+%04X]", i, r, range.start, range.end); + is_valid = false; + assert(is_valid); + } else if (range.end < range.start) { // range swapped DebugLog("uset[%d]: range[%d+%d] end 0x%X= ch) { return true; } @@ -1058,6 +1090,30 @@ bool USet::contains(km_kbp_usv ch) const { return false; } +bool +USet::valid() const { + // double check + for (const auto &range : ranges) { + if (!Uni_IsValid(range.start, range.end)) { + DebugLog("Invalid UnicodeSet (contains noncharacters): [U+%04X,U+%04X]", (int)range.start, (int)range.end); + return false; + } + } + return true; +} + +void +USet::dump() const { + DebugLog(" - USet size=%d", ranges.size()); + for (const auto &range : ranges) { + if (range.start == range.end) { + DebugLog(" - [U+%04X]", (uint32_t)range.start); + } else { + DebugLog(" - [U+%04X-U+%04X]", (uint32_t)range.start, (uint32_t)range.end); + } + } +} + USet COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const { if (!valid() || i >= uset->usetCount) { @@ -1083,6 +1139,9 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length) : bksp(nullptr), disp(nullptr), elem(nullptr), key2(nullptr), layr(nullptr), list(nullptr), loca(nullptr), meta(nullptr), sect(nullptr), strs(nullptr), tran(nullptr), vars(nullptr), vkey(nullptr), valid(false) { DebugLog("kmx_plus: Got a COMP_KEYBOARD at %p\n", keyboard); +#if !KMXPLUS_DEBUG_LOAD + DebugLog("Note: define KMXPLUS_DEBUG_LOAD=1 at compile time for more verbosity in loading"); +#endif if (!(keyboard->dwFlags & KF_KMXPLUS)) { DebugLog("Err: flags COMP_KEYBOARD.dwFlags did not have KF_KMXPLUS set"); valid = false; diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index cbcdb4a323b..684da2a1920 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace km { namespace kbp { @@ -679,24 +680,27 @@ struct COMP_KMXPLUS_USET_USET { struct COMP_KMXPLUS_USET_RANGE { km_kbp_usv start; km_kbp_usv end; + public: + COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE& other); + COMP_KMXPLUS_USET_RANGE(KMX_DWORD start, KMX_DWORD end); }; /** * represents one of the uset elements - * Aliases, does not copy memory. - * The original KMX+ memory must stay around while this object is held. */ class USet { public: - /** construct a set over the specified range. */ + /** construct a set over the specified range. Data is copied. */ USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount); /** empty set */ USet(); /** true if the uset contains this char */ bool contains(km_kbp_usv ch) const; + /** debugging */ + void dump() const; + bool valid() const; private: - const COMP_KMXPLUS_USET_RANGE *ranges; - size_t count; + std::list ranges; }; class COMP_KMXPLUS_USET_Helper { diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h index fafa277ed79..9b479dac7ad 100644 --- a/core/src/kmx/kmx_xstring.h +++ b/core/src/kmx/kmx_xstring.h @@ -6,22 +6,43 @@ namespace km { namespace kbp { namespace kmx { +const char16_t Uni_LEAD_SURROGATE_START = 0xD800; +const char16_t Uni_LEAD_SURROGATE_END = 0xDBFF; +const char16_t Uni_TRAIL_SURROGATE_START = 0xDC00; +const char16_t Uni_TRAIL_SURROGATE_END = 0xDFFF; +const char16_t Uni_SURROGATE_START = Uni_LEAD_SURROGATE_START; +const char16_t Uni_SURROGATE_END = Uni_TRAIL_SURROGATE_END; +const char16_t Uni_FD_NONCHARACTER_START = 0xFDD0; +const char16_t Uni_FD_NONCHARACTER_END = 0xFDEF; +const char16_t Uni_FFFE_NONCHARACTER = 0xFFFE; +const char16_t Uni_FFFF_NONCHARACTER = 0xFFFF; +const char16_t Uni_BMP_END = 0xFFFF; +const km_kbp_usv Uni_SMP_START = 0x010000; +const km_kbp_usv Uni_PLANE_MASK = 0x1F0000; +const km_kbp_usv Uni_MAX_CODEPOINT = 0x10FFFF; + /** * @brief True if a lead surrogate * \def Uni_IsSurrogate1 */ -#define Uni_IsSurrogate1(ch) ((ch) >= 0xD800 && (ch) <= 0xDBFF) +#define Uni_IsSurrogate1(ch) ((ch) >= km::kbp::kmx::Uni_LEAD_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_LEAD_SURROGATE_END) /** * @brief True if a trail surrogate * \def Uni_IsSurrogate2 */ -#define Uni_IsSurrogate2(ch) ((ch) >= 0xDC00 && (ch) <= 0xDFFF) +#define Uni_IsSurrogate2(ch) ((ch) >= km::kbp::kmx::Uni_TRAIL_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_TRAIL_SURROGATE_END) + +/** + * @brief True if any surrogate + * \def UniIsSurrogate +*/ +#define Uni_IsSurrogate(ch) (Uni_IsSurrogate1(ch) || Uni_IsSurrogate2(ch)) /** * @brief Returns true if BMP (Plane 0) * \def Uni_IsBMP */ -#define Uni_IsBMP(ch) ((ch) < 0x10000) +#define Uni_IsBMP(ch) ((ch) <= km::kbp::kmx::Uni_BMP_END) /** * @brief Convert two UTF-16 surrogates into one UTF-32 codepoint @@ -29,17 +50,35 @@ namespace kmx { * @param cl trail surrogate - Uni_IsSurrogate2(cl) must == true * \def Uni_SurrogateToUTF */ -#define Uni_SurrogateToUTF32(ch, cl) (((ch) - 0xD800) * 0x400 + ((cl) - 0xDC00) + 0x10000) +#define Uni_SurrogateToUTF32(ch, cl) (((ch) - km::kbp::kmx::Uni_LEAD_SURROGATE_START) * 0x400 + ((cl) - km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + km::kbp::kmx::Uni_SMP_START) /** * @brief Convert UTF-32 BMP to UTF-16 BMP * @param ch codepoint - Uni_IsBMP(ch) must == true * \def Uni_UTF32BMPToUTF16 */ -#define Uni_UTF32BMPToUTF16(ch) (ch & 0xFFFF) +#define Uni_UTF32BMPToUTF16(ch) ((ch) & Uni_FFFF_NONCHARACTER) -#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800) -#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00) +#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) / 0x400 + km::kbp::kmx::Uni_LEAD_SURROGATE_START) +#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) % 0x400 + km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + +/** + * @returns true if the character is a noncharacter +*/ +bool Uni_IsNonCharacter(km_kbp_usv ch); + +/** + * @returns true if the character is a valid Unicode code point. + * Surrogates belong to UTF-16 and are invalid. +*/ +bool Uni_IsValid(km_kbp_usv ch); + +/** + * @returns true if the character is a valid Unicode code point range, that is, [start-end] are all + * valid. + * Surrogates belong to UTF-16 and are invalid. +*/ +bool Uni_IsValid(km_kbp_usv start, km_kbp_usv range); /** * char16_t array big enough to hold a single Unicode codepoint, @@ -133,6 +172,44 @@ u16string_to_u32string(const std::u16string &source) { return out; } +inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) { + return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE +} + +inline bool Uni_IsNoncharacter(km_kbp_usv ch) { + return (((ch) >= Uni_FD_NONCHARACTER_START && (ch) <= Uni_FD_NONCHARACTER_END) || Uni_IsEndOfPlaneNonCharacter(ch)); +} + +inline bool Uni_InCodespace(km_kbp_usv ch) { + return ((ch) <= Uni_MAX_CODEPOINT); +}; + +inline bool Uni_IsValid(km_kbp_usv ch) { + return (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch)); +} + +inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) { + if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) { + // start or end out of range, or inverted range + return false; + } else if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) { + // contains some of the surrogate range + return false; + } else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) { + // contains some of the noncharacter range + return false; + } else if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) { + // start and end are on different planes, meaning that the U+__FFFE/U+__FFFF noncharacters + // are contained. + // As a reminder, we already checked that start/end are themselves valid, + // so we know that 'end' is not on a noncharacter at end of plane. + return false; + } else { + return true; + } +} + + } // namespace kmx } // namespace kbp } // namespace km diff --git a/core/src/ldml/C9134_ldml_markers.md b/core/src/ldml/C9134_ldml_markers.md index 7a0f5d69597..0b2436b97f4 100644 --- a/core/src/ldml/C9134_ldml_markers.md +++ b/core/src/ldml/C9134_ldml_markers.md @@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas: #### Match only - `transform from=` to match markers -- `transform after=` to match markers - `display to=` for matching keys which contain markers ## Theory / Encoding diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 06257c8b11b..41ed5d419f2 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -240,51 +240,50 @@ ldml_processor::process_event( // Construct a context buffer of all the KM_KBP_BT_CHAR items // Extract the context into 'ctxt' for transforms to process if (!!transforms) { - // if no transforms, no reason to do this extraction + // if no transforms, no reason to do this extraction (ctxt will remain empty) auto &cp = state->context(); // We're only interested in as much of the context as is a KM_KBP_BT_CHAR. uint8_t last_type = KM_KBP_BT_UNKNOWN; for (auto c = cp.rbegin(); c != cp.rend(); c++) { last_type = c->type; if (last_type != KM_KBP_BT_CHAR) { - // not a char, get out + // not a char, stop here + // TODO-LDML: markers? break; } + ctxt.emplace_front(1, c->character); // extract UTF-32 to 1 or 2 UTF-16 chars in a string - km::kbp::kmx::char16_single buf; - const int len = km::kbp::kmx::Utf32CharToUtf16(c->character, buf); - const std::u16string str(buf.ch, len); - ctxt.push_front(str); // prepend to string } } // Look up the key const std::u16string str = keys.lookup(vk, modifier_state); if (str.empty()) { - // not found + // not found, so pass the keystroke on to the Engine state->actions().push_invalidate_context(); state->actions().push_emit_keystroke(); break; // ----- commit and exit } // found the correct string - push it into the context and actions const std::u32string str32 = kmx::u16string_to_u32string(str); - for(size_t i=0; icontext().push_character(str32[i]); - state->actions().push_character(str32[i]); + for (const auto &ch : str32) { + state->context().push_character(ch); + state->actions().push_character(ch); } // Now process transforms // Process the transforms if (!!transforms) { // add the newly added char to ctxt - ctxt.push_back(str); + ctxt.push_back(str32); + + std::u32string outputString; - std::u16string outputString; - // TODO-LDML: unroll ctxt into a str. Would be better to have transforms be able to process a vector - std::u16string ctxtstr; - for (size_t i = 0; i < ctxt.size(); i++) { - ctxtstr.append(ctxt[i]); + std::u32string ctxtstr; + for (const auto &ch : ctxt) { + ctxtstr.append(ch); } + // check if the context matched, and if so how much (at the end) const size_t matchedContext = transforms->apply(ctxtstr, outputString); if (matchedContext > 0) { @@ -296,10 +295,9 @@ ldml_processor::process_event( state->actions().push_backspace(KM_KBP_BT_CHAR, deletedChar); // Cause prior char to be removed } // Now, add in the updated text - const std::u32string outstr32 = kmx::u16string_to_u32string(outputString); - for (size_t i = 0; i < outstr32.length(); i++) { - state->context().push_character(outstr32[i]); - state->actions().push_character(outstr32[i]); + for (const auto &ch : outputString) { + state->context().push_character(ch); + state->actions().push_character(ch); } } } diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index 9d5275ab41a..4d469abfd97 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -9,6 +9,8 @@ #include "debuglog.h" #include #include +#include "kmx/kmx_xstring.h" + #ifndef assert #define assert(x) // TODO-LDML @@ -18,6 +20,22 @@ namespace km { namespace kbp { namespace ldml { +/** + * \def KMXPLUS_DEBUG_TRANSFORM + * define KMXPLUS_DEBUG_TRANSFORM=1 to enable verbose processing of transforms/reorders + * The default is 0, which only notes initialization and exceptional cases +*/ + +#ifndef KMXPLUS_DEBUG_TRANSFORM +#define KMXPLUS_DEBUG_TRANSFORM 0 +#endif + +#if KMXPLUS_DEBUG_TRANSFORM +#define DebugTran(msg, ...) DebugLog(msg, ##__VA_ARGS__) +#else +#define DebugTran(msg, ...) +#endif + element::element(const USet &new_u, KMX_DWORD new_flags) : chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) { } @@ -67,6 +85,16 @@ element::matches(km_kbp_usv ch) const { } } +void +element::dump() const { + if (is_uset()) { + DebugLog("element order=%d USET", (int)get_order()); + uset.dump(); + } else { + DebugLog("element order=%d U+%04X", (int)get_order(), (int)chr); + } +} + int reorder_sort_key::compare(const reorder_sort_key &other) const { int primaryResult = (int)primary - (int)other.primary; @@ -83,7 +111,10 @@ reorder_sort_key::compare(const reorder_sort_key &other) const { } else if (quaternaryResult) { return quaternaryResult; } else { - assert(quaternaryResult); // quaternary is a string index, should always be != + // We don't expect to get here. quaternaryResult is the string index, which + // should be unequal. + assert(quaternaryResult); + // We have the underlying character, so use the binary order as a tiebreaker. int identityResult = (int)ch - (int)other.ch; // tie breaker return identityResult; } @@ -94,6 +125,11 @@ reorder_sort_key::operator<(const reorder_sort_key &other) const { return (compare(other) < 0); } +bool +reorder_sort_key::operator>(const reorder_sort_key &other) const { + return (compare(other) > 0); +} + std::deque reorder_sort_key::from(const std::u32string &str) { // construct a 'baseline' sort key, that is, in the absence of @@ -102,6 +138,10 @@ reorder_sort_key::from(const std::u32string &str) { auto s = str.begin(); // str iterator size_t c = 0; // str index for (auto e = str.begin(); e < str.end(); e++, s++, c++) { + // primary weight: 0 + // seconary weight: c (the string index) + // tertiary weight: 0 + // quaternary weight: c (the index again) keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c}); } return keylist; @@ -116,9 +156,13 @@ reorder_sort_key::dump() const { size_t element_list::match_end(const std::u32string &str) const { if (str.size() < size()) { - return 0; // input string too short, can't possibly match + // input string too short, can't possibly match. + // This assumes each element is a single char, no string elements. + return 0; } // s: iterate from end to front of string + // For example, if str = 'abcd', we try to match 'd', then 'c', then 'b', then 'a' + // starting with the end of the element list. auto s = str.rbegin(); // e: end to front on elements. // we know the # of elements is <= length of string, @@ -137,43 +181,72 @@ element_list::match_end(const std::u32string &str) const { bool element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) { KMX_DWORD elementsLength; - auto elements = kplus.elem->getElementList(id, elementsLength); - assert((elementsLength == 0) || (elements != nullptr)); + auto elements = kplus.elem->getElementList(id, elementsLength); // pointer to beginning of element list + assert((elementsLength == 0) || (elements != nullptr)); // it could be a 0-length list for (size_t i = 0; i & element_list::update_sort_key(size_t offset, std::deque &key) const { + /** string index */ size_t c = 0; - for (auto e = begin(); e < end(); e++) { + for (auto e = begin(); e < end(); e++, c++) { + /** update this key */ auto &k = key.at(offset + c); + // we double check that the character matches. otherwise something + // has really gone awry, because we shouldn't be here if this element list doesn't apply. if (!e->matches(k.ch)) { - DebugLog("!! updateSortKey(%d+%d): element did not re-match the sortkey", offset, c); + DebugLog("!! Internal Error: updateSortKey(%d+%d): element did not re-match the sortkey", offset, c); k.dump(); + // TODO-LDML: assertion follows + assert(e->matches(k.ch)); // double check that this element matches } - assert(e->matches(k.ch)); // double check that this element matches + // we only update primary and tertiary weights k.primary = e->get_order(); - k.tertiary = e->get_tertiary(); // TODO-LDML: need more detailed tertiary work - c++; + // TODO-LDML: need more detailed tertiary work + k.tertiary = e->get_tertiary(); +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Updating at +%d", c); + k.dump(); +#endif } return key; } +void +element_list::dump() const { + DebugLog("element_list[%d]", size()); + for (const auto &e : *this) { + e.dump(); + } +} + reorder_entry::reorder_entry(const element_list &new_elements) : elements(new_elements), before() { } reorder_entry::reorder_entry(const element_list &new_elements, const element_list &new_before) : elements(new_elements), before(new_before) { @@ -182,11 +255,14 @@ reorder_entry::reorder_entry(const element_list &new_elements, const element_lis size_t reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const { auto substr = str.substr(offset, len); + // first, see if the elements match. If not, this entry doesn't apply size_t match_len = elements.match_end(substr); if (match_len == 0) { return 0; } + // Now we need to check if there is a "before=" element string that + // is also a precondition. if (!before.empty()) { // does not match before offset std::u32string prefix = substr.substr(0, substr.size() - match_len); @@ -208,95 +284,138 @@ reorder_group::apply(std::u32string &str) const { // get a baseline sort key auto sort_keys = reorder_sort_key::from(str); - // DebugLog("Baseline Keys:"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } - // apply ALL reorders in the group. - // size_t c = 0; - for (auto r = list.begin(); r < list.end(); r++) { + for (const auto &r : list) { // work backward from end of string forward + // That is, see if "abc" matches "abc" or "ab" or "a" for (size_t s = str.size(); s > 0; s--) { - size_t submatch = r->match_end(str, 0, s); + size_t submatch = r.match_end(str, 0, s); if (submatch != 0) { +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Matched: %S (off=%d, len=%d)", str.c_str(), 0, s); + r.elements.dump(); +#endif // update the sort key size_t sub_match_start = s - submatch; - r->elements.update_sort_key(sub_match_start, sort_keys); - some_match = true; + r.elements.update_sort_key(sub_match_start, sort_keys); + some_match = true; // record that there was a match } } - // c++; } if (!some_match) { - // DebugLog("Skip: No reorder elements matched."); + // get out if nothing matched. + // the sortkey won't be "interesting", and the sort + // will be a no-op. + DebugTran("Skip: No reorder elements matched."); return false; // nothing matched, so no work. } - size_t match_len = str.size(); // TODO-LDML: for now, assume entire match +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Updated sortkey"); + for (const auto &r : sort_keys) { + r.dump(); + } +#endif - // DebugLog("Updated Keys:"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } + // TODO-LDML: for now, assume matches entire string. + // A needed optimization here would be to detect a common substring + // at the end of the old and new strings, and keep the match_len + // minimal. This reduces thrash in core's context. + size_t match_len = str.size(); + // 'prefix' is the unmatched string before the match + // TODO-LDML: right now, this is empty. std::u32string prefix = str; prefix.resize(str.size() - match_len); // just the part before the matched part. - // just the suffix (the matched part) - std::u32string suffix = str.substr(prefix.size(), match_len); - // sort it! Here's where the reorder happens - // TODO: need to sort only between primary bases… - std::sort(sort_keys.begin(), sort_keys.end()); -#if 0 - // TODO-LDML :need to sort sub-runs - for(auto e = sort_keys.end(); !applied && e > sort_keys.begin(); e--) { - if (e->primary == 0) { - // Got it. - std::sort(e, sort_keys.end()); - // DebugLog("… sorting at q=%d", (int)e->quaternary); + + // Now, we need to actually do the sorting, but we must only sort + // 'runs' beginning with 0-weight keys. + + // Consider the 'roast' example in the spec, you might end up with the following: + // codepoint (pri, sec, ter, quat) + // U+1A21 (0, 0, 0, 0) + // U+1A60 (127, 1, 0, 1) + // U+1A45 (0, 2, 0, 2) + // U+1A6B (42, 3, 0, 3) + // U+1A76 (55, 4, 0, 4) + // This example happens to be in order, but must be sorted in two diferent ranges, + // with secondary (index) values of [0,1] and [2,4] + // + // Another example might look like the following: + // U+1A21 (0, 0, 0, 0) + // U+1A6B (42, 1, 0, 1) + // U+1A76 (55, 2, 0, 2) + // U+1A60 (10, 3, 0, 3) + // U+1A45 (10, 4, 0, 3) + // Here there is only a single range to sort [0,4] + + /** pointer to the beginning of the current run. */ + std::deque::iterator run_start = sort_keys.begin(); + for(auto e = run_start; e != sort_keys.end(); e++) { + if ((e->primary == 0) && (e != run_start)) { // it's a base + auto run_end = e - 1; + DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary); + std::sort(run_start, run_end); // reversed because it's a reverse iterator…? + // move the start + run_start = e; // next run starts here } } -#endif - // recombine into a str + // sort the last run in the string as well. + if (run_start != sort_keys.end()) { // TODO-LDML: skip if a single-char run + DebugTran("Sorting final subrange quaternary=[%d..]", run_start->quaternary); + std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…? + } + // recombine into a string by pulling out the 'ch' value + // that's in each sortkey element. std::u32string newSuffix; - size_t q = sort_keys.begin()->quaternary; // + size_t q = sort_keys.begin()->quaternary; // start with the first quaternary for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) { - if (q != e->quaternary) { // something rearranged in this subrange + if (q != e->quaternary) { + // something rearranged in this subrange, because the quaternary values are out of order. applied = true; } + // collect the characters newSuffix.append(1, e->ch); } if (applied) { - // DebugLog("Final Sort"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } str.resize(prefix.size()); str.append(newSuffix); } else { - // DebugLog("Skip: no reordering change detected"); + DebugTran("Skip: sorting caused no reordering"); + } +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Sorted sortkey"); + for (const auto &r : sort_keys) { + r.dump(); } +#endif return applied; } -transform_entry::transform_entry(const std::u16string &from, const std::u16string &to) : fFrom(from), fTo(to) { +transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) { } size_t -transform_entry::match(const std::u16string &input) const { +transform_entry::match(const std::u32string &input) const { if (input.length() < fFrom.length()) { + // TODO-LDML: regex + // Too small, can't match. return 0; } // string at end auto substr = input.substr(input.length() - fFrom.length(), fFrom.length()); if (substr != fFrom) { + // end of string doesn't match return 0; } + // match length == fFrom.length return substr.length(); } -std::u16string -transform_entry::apply(const std::u16string & /*input*/, size_t /*matchLen*/) const { +std::u32string +transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const { + // TODO-LDML: regex + // For now, we just return the 'to' string literally. return fTo; } @@ -325,7 +444,7 @@ transform_group::transform_group() { * return the first transform match in this group */ const transform_entry * -transform_group::match(const std::u16string &input, size_t &subMatched) const { +transform_group::match(const std::u32string &input, size_t &subMatched) const { for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) { // TODO-LDML: non regex implementation // is the match area too short? @@ -345,7 +464,7 @@ transform_group::match(const std::u16string &input, size_t &subMatched) const { * @return match length: number of chars at end of input string to modify. 0 if no match. */ size_t -transforms::apply(const std::u16string &input, std::u16string &output) { +transforms::apply(const std::u32string &input, std::u32string &output) { /** * Example: * Group0: za -> c, a -> bb @@ -380,7 +499,7 @@ transforms::apply(const std::u16string &input, std::u16string &output) { */ size_t matched = 0; /** modified copy of input */ - std::u16string updatedInput = input; + std::u32string updatedInput = input; for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) { // for each transform group // break out once there's a match @@ -392,13 +511,14 @@ transforms::apply(const std::u16string &input, std::u16string &output) { // find the first match in this group (if present) // TODO-LDML: check if reorder if (group->type == any_group_type::transform) { - auto transform = group->transform.match(updatedInput, subMatched); + auto entry = group->transform.match(updatedInput, subMatched); - if (transform != nullptr) { + if (entry != nullptr) { // now apply the found transform // update subOutput (string) and subMatched - std::u16string subOutput = transform->apply(updatedInput, subMatched); + // the returned string must replace the last "subMatched" chars of the string. + std::u32string subOutput = entry->apply(updatedInput, subMatched); // remove the matched part of the updatedInput updatedInput.resize(updatedInput.length() - subMatched); // chop of the subMatched part at end @@ -420,7 +540,17 @@ transforms::apply(const std::u16string &input, std::u16string &output) { } } } else if (group->type == any_group_type::reorder) { - // TODO-LDML reorder + // TODO-LDML: cheesy solution. We should be finding a smaller + // common match here. + std::u32string str2 = updatedInput; + if (group->reorder.apply(str2)) { + // pretend the whole thing matched + output.resize(0); + output.append(str2); + updatedInput.resize(0); + updatedInput.append(str2); + matched = output.length(); + } } // else: continue to next group } @@ -440,10 +570,10 @@ transforms::apply(const std::u16string &input, std::u16string &output) { return matched; } -// simple impl bool -transforms::apply(std::u16string &str) { - std::u16string output; +transforms::apply(std::u32string &str) { + // simple implementation for tests + std::u32string output; size_t matchLength = apply(str, output); if (matchLength == 0) { return false; @@ -453,23 +583,6 @@ transforms::apply(std::u16string &str) { return true; } -bool -transforms::apply(std::u32string &str) { - bool rc = false; - // TODO-LDML: PoC implementation for now, need to refactor into fcns - // ONLY reorder - for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) { - assert(group->type == reorder); // TODO-LDML - auto rgroup = group->reorder; - if (rgroup.apply(str)) { - rc = true; - } - } - return rc; -} - -// Loader - transforms * transforms::load( const kmx::kmx_plus &kplus, @@ -515,17 +628,18 @@ transforms::load( for (KMX_DWORD itemNumber = 0; itemNumber < group->count; itemNumber++) { const kmx::COMP_KMXPLUS_TRAN_TRANSFORM *element = tranHelper.getTransform(group->index + itemNumber); - const std::u16string fromStr = kplus.strs->get(element->from); - const std::u16string toStr = kplus.strs->get(element->to); + const std::u32string fromStr = kmx::u16string_to_u32string(kplus.strs->get(element->from)); + const std::u32string toStr = kmx::u16string_to_u32string(kplus.strs->get(element->to)); std::u16string mapFrom, mapTo; if (element->mapFrom && element->mapTo) { - // strings: variable name + // strings: variable name of from/to + // TODO-LDML: not implemented mapFrom = kplus.strs->get(element->mapFrom); mapTo = kplus.strs->get(element->mapTo); } - newGroup.emplace_back(fromStr, toStr); // creating a transform_entry + newGroup.emplace_back(fromStr, toStr /* ,mapFrom, mapTo */); // creating a transform_entry } transforms->addGroup(newGroup); } else if (group->type == LDML_TRAN_GROUP_TYPE_REORDER) { @@ -543,6 +657,7 @@ transforms::load( if (load_ok) { newGroup.list.emplace_back(elements, before); } else { + DebugLog("reorder elements(%d+%d) failed to load", group->index, itemNumber); return nullptr; } } diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 7904eeffc81..6296399f2b3 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -24,7 +24,7 @@ using km::kbp::kmx::USet; * Type of a group */ enum any_group_type { - transform = LDML_TRAN_GROUP_TYPE_REORDER, + transform = LDML_TRAN_GROUP_TYPE_TRANSFORM, reorder = LDML_TRAN_GROUP_TYPE_REORDER, }; @@ -33,24 +33,30 @@ enum any_group_type { */ class element { public: - /** from a USet */ + /** construct from a USet */ element(const USet &u, KMX_DWORD flags); - /** from a single char */ + /** construct from a single char */ element(km_kbp_usv ch, KMX_DWORD flags); /** @returns true if a USet type */ bool is_uset() const; + /** @returns true if prebase bit set*/ bool is_prebase() const; + /** @returns true if tertiary base bit set */ bool is_tertiary_base() const; - signed char get_tertiary() const; + /** @returns the primary order */ signed char get_order() const; + /** @returns the tertiary order */ + signed char get_tertiary() const; /** @returns raw elem flags */ KMX_DWORD get_flags() const; /** @returns true if matches this character*/ bool matches(km_kbp_usv ch) const; + /** debugging: dump this element via DebugLog() */ + void dump() const; private: - // TODO-LDML: support multi-char strings + // TODO-LDML: support multi-char strings? const km_kbp_usv chr; const USet uset; const KMX_DWORD flags; @@ -62,30 +68,30 @@ class element { class transform_entry { public: transform_entry( - const std::u16string &from, - const std::u16string &to + const std::u32string &from, + const std::u32string &to /*TODO-LDML: mapFrom, mapTo*/ ); /** * @returns length if it's a match */ - size_t match(const std::u16string &input) const; + size_t match(const std::u32string &input) const; /** * @returns output string */ - std::u16string apply(const std::u16string &input, size_t matchLen) const; + std::u32string apply(const std::u32string &input, size_t matchLen) const; private: - const std::u16string fFrom; // TODO-LDML: regex - const std::u16string fTo; + const std::u32string fFrom; // TODO-LDML: regex + const std::u32string fTo; }; /** * An ordered list of strings. */ -typedef std::deque string_list; +typedef std::deque string_list; /** * a group of entries - a @@ -100,7 +106,7 @@ class transform_group : public std::deque { * @param subMatched on output, the matched length * @returns alias to transform_entry or nullptr */ - const transform_entry *match(const std::u16string &input, size_t &subMatched) const; + const transform_entry *match(const std::u32string &input, size_t &subMatched) const; }; /** a single char, categorized according to reorder rules*/ @@ -111,13 +117,12 @@ struct reorder_sort_key { signed char tertiary; // tertiary value, defaults to 0 size_t quaternary; // index again - /** - * Return -1, 0, 1 depending on order - */ + /** @returns -1, 0, 1 depending on ordering */ int compare(const reorder_sort_key &other) const; bool operator<(const reorder_sort_key &other) const; + bool operator>(const reorder_sort_key &other) const; - /** create a 'baseline' sort key, all 0 primary weights */ + /** create a 'baseline' sort key, with each character having primary weight 0 */ static std::deque from(const std::u32string &str); /** TODO-LDML: for debugging. */ @@ -136,7 +141,7 @@ class element_list : public std::deque { * Update the deque (see reorder_sort_key::from()) with the weights from this element list * starting at the beginning of this element list * @param offset start at this offset in the deque. Still starts at the first element - * @param the key deque to update + * @param key key deque to update * @returns the key parameter */ std::deque &update_sort_key(size_t offset, std::deque &key) const; @@ -144,6 +149,9 @@ class element_list : public std::deque { /** construct from KMX+ elem id*/ bool load(const kmx::kmx_plus& kplus, kmx::KMXPLUS_ELEM id); + + /** TODO-LDML: for debugging */ + void dump() const; }; class reorder_entry { @@ -217,13 +225,7 @@ class transforms { * @param output if matched, contains the replacement output text * @return length in chars of the input (counting from the end) which matched context */ - size_t apply(const std::u16string &input, std::u16string &output); - - /** - * For tests - * @return true if str was altered - */ - bool apply(std::u16string &str); + size_t apply(const std::u32string &input, std::u32string &output); /** * For tests - TODO-LDML only supports reorder @@ -232,12 +234,13 @@ class transforms { bool apply(std::u32string &str); public: + /** load from a kmx_plus data section, either tran or bksp */ static transforms * - load(const kmx::kmx_plus &kplus, const kbp::kmx::COMP_KMXPLUS_TRAN *tran, const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper); + load(const kmx::kmx_plus &kplus, + const kbp::kmx::COMP_KMXPLUS_TRAN *tran, + const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper); }; -/** - * Loader for transform groups (from tran or bksp) - */ + } // namespace ldml } // namespace kbp } // namespace km diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp index d578fa7e381..e3c40776225 100644 --- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp @@ -1238,6 +1238,7 @@ test_xstrlen_ignoreifopt() { void test_utf32() { + std::cout << "== " << __FUNCTION__ << std::endl; const KMX_DWORD u295 = 0x0127; // ħ assert(Uni_IsBMP(u295)); @@ -1270,6 +1271,7 @@ test_utf32() { void test_u16string_to_u32string() { + std::cout << "== " << __FUNCTION__ << std::endl; // normal cases { const std::u32string str = u16string_to_u32string(u""); @@ -1327,6 +1329,56 @@ test_u16string_to_u32string() { } } +void test_is_valid() { + std::cout << "== " << __FUNCTION__ << std::endl; + // valid + assert_equal(Uni_IsValid(0x0000), true); + assert_equal(Uni_IsValid(0x0127), true); + assert_equal(Uni_IsValid(U'🙀'), true); + + // invalid + assert_equal(Uni_IsValid(0xDECAFBAD), false); // out of range + assert_equal(Uni_IsValid(0x566D4128), false); + assert_equal(Uni_IsValid(0xFFFF), false); // nonchar + assert_equal(Uni_IsValid(0xFFFE), false); // nonchar + assert_equal(Uni_IsValid(0x10FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x10FFFE), false); // nonchar + assert_equal(Uni_IsValid(0x01FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x01FFFE), false); // nonchar + assert_equal(Uni_IsValid(0x02FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x02FFFE), false); // nonchar + assert_equal(Uni_IsValid(0xFDD1), false); // nonchar + assert_equal(Uni_IsValid(0xFDD0), false); // nonchar + + + // positive range test + assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true); + assert_equal(Uni_IsValid(0x10, 0x20), true); + assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true); + + // all valid ranges in BMP + assert_equal(Uni_IsValid(0x0000, 0xD7FF), true); + assert_equal(Uni_IsValid(0xD800, 0xDFFF), false); + assert_equal(Uni_IsValid(0xE000, 0xFDCF), true); + assert_equal(Uni_IsValid(0xFDD0, 0xFDEF), false); + assert_equal(Uni_IsValid(0xFDF0, 0xFDFF), true); + assert_equal(Uni_IsValid(0xFDF0, 0xFFFD), true); + + // negative range test + assert_equal(Uni_IsValid(0, 0x10FFFF), false); // ends with nonchar + assert_equal(Uni_IsValid(0, 0x10FFFD), false); // contains lots o' nonchars + assert_equal(Uni_IsValid(0x20, 0x10), false); // swapped + assert_equal(Uni_IsValid(0xFDEF, 0xFDF0), false); // just outside range + assert_equal(Uni_IsValid(0x0000, 0x010000), false); // crosses noncharacter plane boundary and other stuff + assert_equal(Uni_IsValid(0x010000, 0x020000), false); // crosses noncharacter plane boundary + assert_equal(Uni_IsValid(0x0000, 0xFFFF), false); // crosses other BMP prohibited and plane boundary + assert_equal(Uni_IsValid(0x0000, 0xFFFD), false); // crosses other BMP prohibited + assert_equal(Uni_IsValid(0x0000, 0xE000), false); // crosses surrogate space + assert_equal(Uni_IsValid(0x0000, 0x20FFFF), false); // out of bounds + assert_equal(Uni_IsValid(0x10FFFD, 0x20FFFF), false); // out of bounds + + } + constexpr const auto help_str = u"\ test_kmx_xstring [--color]\n\ \n\ @@ -1349,6 +1401,7 @@ int main(int argc, char *argv []) { test_xstrlen_ignoreifopt(); test_utf32(); test_u16string_to_u32string(); + test_is_valid(); return 0; } diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml similarity index 85% rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml rename to core/tests/unit/ldml/keyboards/k_020_fr-test.xml index 21f9d64cc4f..04066ae220b 100644 --- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml +++ b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml @@ -1,10 +1,11 @@ + - + - + diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml b/core/tests/unit/ldml/keyboards/k_020_fr.xml similarity index 94% rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml rename to core/tests/unit/ldml/keyboards/k_020_fr.xml index 8a634bda65d..1f6fc2c32f1 100644 --- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml +++ b/core/tests/unit/ldml/keyboards/k_020_fr.xml @@ -1,4 +1,5 @@ + @@ -7,11 +8,11 @@ - + - - + + @@ -22,15 +23,15 @@ - + - + - + diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml index 5fcfd321e25..d23862ca2bb 100644 --- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml +++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml @@ -3,16 +3,30 @@ - + + + + + + + + + + - + diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml index da16c187171..8df0d5ffd7c 100644 --- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml +++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml @@ -13,16 +13,20 @@ + + + + - + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index b4ba719548c..3d1bfdddcce 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -9,8 +9,8 @@ # tests in resources/standards-data/ldml-keyboards/techpreview/test/ tests_from_cldr = [ 'ja-Latn', - 'pt-k0-abnt2', - # 'fr-t-k0-azerty', # vkey issues + # 'pt-k0-abnt2', #TODO-LDML: marker syntax fail! + 'fr-t-k0-azerty', ] tests_without_testdata = [ @@ -32,7 +32,7 @@ tests_without_testdata = [ # These tests have a k_001_tiny-test.xml file as well. tests_with_testdata = [ 'k_001_tiny', - 'fr-t-k0-azerty', # TODO-LDML: move to cldr above (fix vkey) + 'k_020_fr', # TODO-LDML: move to cldr above (fix vkey) 'k_200_reorder_nod_Lana', ] diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index c72fd6bcf8c..7097bcb64bf 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -40,7 +40,7 @@ test_transforms() { std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl; { // start with one - transform_entry te(std::u16string(u"e^"), std::u16string(u"E")); // keep it simple + transform_entry te(std::u32string(U"e^"), std::u32string(U"E")); // keep it simple // OK now make a group do it transforms tr; transform_group st; @@ -51,17 +51,17 @@ test_transforms() { // see if we can match the same { - std::u16string src(u"barQ^"); + std::u32string src(U"barQ^"); bool res = tr.apply(src); zassert_equal(res, false); - zassert_string_equal(src, std::u16string(u"barQ^")); // no change + zassert_string_equal(src, std::u32string(U"barQ^")); // no change } { - std::u16string src(u"fooe^"); + std::u32string src(U"fooe^"); bool res = tr.apply(src); zassert_equal(res, true); - zassert_string_equal(src, std::u16string(u"fooE")); + zassert_string_equal(src, std::u32string(U"fooE")); } } @@ -73,23 +73,23 @@ test_transforms() { // setup { transform_group st; - st.emplace_back(std::u16string(u"za"), std::u16string(u"c")); - st.emplace_back(std::u16string(u"a"), std::u16string(u"bb")); + st.emplace_back(std::u32string(U"za"), std::u32string(U"c")); + st.emplace_back(std::u32string(U"a"), std::u32string(U"bb")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"bb"), std::u16string(u"ccc")); + st.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"cc"), std::u16string(u"d")); + st.emplace_back(std::u32string(U"cc"), std::u32string(U"d")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"tcd"), std::u16string(u"e")); + st.emplace_back(std::u32string(U"tcd"), std::u32string(U"e")); tr.addGroup(st); } @@ -97,31 +97,31 @@ test_transforms() { // see if we can match the same { - std::u16string src(u"ta"); + std::u32string src(U"ta"); bool res = tr.apply(src); // pipe (|) symbol shows where the 'output' is delineated // t|a --> t|bb --> t|ccc --> t|cd --> |e - zassert_string_equal(src, std::u16string(u"e")); + zassert_string_equal(src, std::u32string(U"e")); zassert_equal(res, true); } { - std::u16string src(u"qza"); + std::u32string src(U"qza"); bool res = tr.apply(src); // pipe (|) symbol shows where the 'output' is delineated // q|za -> q|c - zassert_string_equal(src, std::u16string(u"qc")); + zassert_string_equal(src, std::u32string(U"qc")); zassert_equal(res, true); } { - std::u16string src(u"qa"); + std::u32string src(U"qa"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"qcd")); + zassert_string_equal(src, std::u32string(U"qcd")); zassert_equal(res, true); } { - std::u16string src(u"tb"); + std::u32string src(U"tb"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"tb")); + zassert_string_equal(src, std::u32string(U"tb")); zassert_equal(res, false); } } @@ -132,13 +132,13 @@ test_transforms() { transforms tr; { transform_group st; - st.emplace_back(std::u16string(u"िह"), std::u16string(u"हि")); + st.emplace_back(std::u32string(U"िह"), std::u32string(U"हि")); tr.addGroup(st); } { - std::u16string src(u"िह"); + std::u32string src(U"िह"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"हि")); + zassert_string_equal(src, std::u32string(U"हि")); zassert_equal(res, true); } } @@ -163,7 +163,7 @@ test_reorder_standalone() { const std::u32string expect = roasts[0]; // now setup the rules const COMP_KMXPLUS_USET_RANGE ranges[] = {// 0 - {0x1A75, 0x1A79}}; + COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)}; const COMP_KMXPLUS_USET_USET usets[] = {{0, 1, 0xFFFFFFFF}}; const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0]; const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count); @@ -288,7 +288,7 @@ test_reorder_standalone() { // element_list e0; - e0.emplace_back(U'\u1A6B', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT); + e0.emplace_back(U'\u1A60', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT); rg.list.emplace_back(e0); // @@ -334,19 +334,41 @@ test_reorder_standalone() { std::cout << __FILE__ << ":" << __LINE__ << " - back to nod-Lana " << std::endl; // TODO-LDML: move this into test code perhaps for (size_t r = 0; r < sizeof(roasts) / sizeof(roasts[0]); r++) { - std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << std::endl; const auto &roast = roasts[r]; + std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << "=" << roast << std::endl; + // try apply with string + { + std::cout << "- try apply(text, output)" << std::endl; + std::u32string text = roast; + std::u32string output; + size_t len = tr.apply(text, output); + if (len == 0) { + std::cout << " (did not apply)" << std::endl; + } else { + std::cout << " applied, matchLen= " << len << std::endl; + text.resize(text.size()-len); // shrink + text.append(output); + std::cout << " = " << text << std::endl; + } + zassert_string_equal(text, expect); + } // try all-at-once { + std::cout << "- try apply(text)" << std::endl; std::u32string text = roast; if (!tr.apply(text)) { std::cout << " (did not apply)" << std::endl; + } else if (text == roast) { + std::cout << " (suboptimal: apply returned true but made no change)" << std::endl; + } else { + std::cout << " changed to " << text; } zassert_string_equal(text, expect); std::cout << " matched (converting all at once)!" << std::endl; } // simulate typing this one char at a time; { + std::cout << "- try key-at-a-time" << std::endl; std::u32string text; for (auto ch = roast.begin(); ch < roast.end(); ch++) { // append the string @@ -362,6 +384,24 @@ test_reorder_standalone() { std::cout << std::endl; } } + // special test + { + std::cout << __FILE__ << ":" << __LINE__ << " - special test " << std::endl; + const std::u32string expect = U"\u1A21\u1A60\u1A45"; // this string shouldn't mutate at all. + { + std::u32string text = expect; + tr.apply(text); + zassert_string_equal(text, expect); + } + { + // try submatch + std::u32string text = expect; + std::u32string output; + size_t len = tr.apply(text, output); + zassert_string_equal(output, U""); + assert_equal(len, 0); + } + } } return EXIT_SUCCESS; } diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 499c74abe9a..987aea874ff 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -46,10 +46,17 @@ let callbackProcIdentifier = 0; const callbackPrefix = 'kmnCompilerCallbacks_'; +interface MallocAndFree { + malloc(sz: number) : number; + free(p: number) : null; +}; + + export class KmnCompiler implements UnicodeSetParser { private Module: any; callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; + wasmExports: MallocAndFree; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); @@ -61,6 +68,7 @@ export class KmnCompiler implements UnicodeSetParser { if(!this.Module) { try { this.Module = await loadWasmHost(); + this.wasmExports = (this.Module.wasmExports ?? this.Module.asm); } catch(e: any) { /* c8 ignore next 3 */ this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({e})); @@ -237,25 +245,23 @@ export class KmnCompiler implements UnicodeSetParser { return null; } - const buf = this.Module.asm.malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT); // TODO-LDML: Catch OOM - /** return code, if positive: range count */ + const buf = this.wasmExports.malloc(rangeCount * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT); + /** If <= 0: return code. If positive: range count */ const rc = this.Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2); if (rc >= 0) { const ranges = []; const startu = (buf / this.Module.HEAPU32.BYTES_PER_ELEMENT); for (let i = 0; i < rc; i++) { - const low = this.Module.HEAPU32[startu + (i * 2) + 0]; - const high = this.Module.HEAPU32[startu + (i * 2) + 1]; - ranges.push([low, high]); + const start = this.Module.HEAPU32[startu + (i * 2) + 0]; + const end = this.Module.HEAPU32[startu + (i * 2) + 1]; + ranges.push([start, end]); } - // TODO-LDML: no free?? - // Module.asm.free(buf); + this.wasmExports.free(buf); return new UnicodeSet(pattern, ranges); } else { - // translate error - // TODO-LDML: no free?? - // Module.asm.free(buf); + this.wasmExports.free(buf); + // translate error code into callback this.callbacks.reportMessage(getUnicodeSetError(rc)); return null; } @@ -265,6 +271,7 @@ export class KmnCompiler implements UnicodeSetParser { /* c8 ignore next 2 */ return null; } + // call with rangeCount = 0 to invoke in 'preflight' mode. const rc = this.Module.kmcmp_parseUnicodeSet(pattern, 0, 0); if (rc >= 0) { return rc; diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts index 2a05c24f269..a924f9fe05a 100644 --- a/developer/src/kmc-kmn/test/test-wasm-uset.ts +++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts @@ -65,12 +65,25 @@ describe('Compiler UnicodeSet function', function() { '[[]': CompilerMessages.ERROR_UnicodeSetSyntaxError, }; for(const [pat, expected] of Object.entries(failures)) { - callbacks.clear(); - assert.notOk(compiler.parseUnicodeSet(pat, 1)); - assert.equal(callbacks.messages.length, 1); - const firstMessage = callbacks.messages[0]; - const code = firstMessage.code; - assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for ${pat}`); + { + // verify fails parse + callbacks.clear(); + assert.notOk(compiler.parseUnicodeSet(pat, 1)); + assert.equal(callbacks.messages.length, 1); + const firstMessage = callbacks.messages[0]; + const code = firstMessage.code; + assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for parsing ${pat}`); + } + // skip 'out of range' because that one won't fail during sizing. + if (expected !== CompilerMessages.FATAL_UnicodeSetOutOfRange) { + // verify fails size + callbacks.clear(); + assert.equal(compiler.sizeUnicodeSet(pat), -1, `sizing ${pat}`); + assert.equal(callbacks.messages.length, 1); + const firstMessage = callbacks.messages[0]; + const code = firstMessage.code; + assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for sizing ${pat}`); + } } }); }); diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 1ed2e14b145..241c5af6856 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -1,5 +1,5 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types'; import { CompilerMessages } from "./messages.js"; import { SectionCompiler } from "./section-compiler.js"; @@ -7,8 +7,14 @@ import { SectionCompiler } from "./section-compiler.js"; import DependencySections = KMXPlus.DependencySections; import Disp = KMXPlus.Disp; import DispItem = KMXPlus.DispItem; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; export class DispCompiler extends SectionCompiler { + static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean { + keyboard.displays?.display?.forEach(({ to }) => + mt.add(MarkerUse.match, MarkerParser.allReferences(to))); + return true; + } public get id() { return constants.section.disp; @@ -38,9 +44,11 @@ export class DispCompiler extends SectionCompiler { // displayOptions result.baseCharacter = sections.strs.allocAndUnescapeString(this.keyboard.displays?.displayOptions?.baseCharacter); + // TODO-LDML: substitute variables! + // displays result.disps = this.keyboard.displays?.display.map(display => ({ - to: sections.strs.allocAndUnescapeString(display.to), + to: sections.strs.allocAndUnescapeString(sections.vars.substituteMarkerString(display.to)), display: sections.strs.allocAndUnescapeString(display.display), })) || []; // TODO-LDML: need coverage for the [] diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index e167c784b2d..93476d44970 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -1,6 +1,7 @@ import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { SectionCompiler } from "./section-compiler.js"; import { LDMLKeyboard, KMXPlus, CompilerCallbacks } from "@keymanapp/common-types"; +import { VarsCompiler } from './vars.js'; /** * Compiler for typrs that don't actually consume input XML @@ -69,6 +70,6 @@ export class UsetCompiler extends EmptyCompiler { } /** - * For test use. The top three compilers. + * For test use. The top compilers. */ -export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler ]; +export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ]; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index b3e13b70fc7..89d93178d34 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -1,5 +1,5 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; -import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types'; +import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types'; import { CompilerMessages } from './messages.js'; import { SectionCompiler } from "./section-compiler.js"; @@ -8,8 +8,18 @@ import Keys = KMXPlus.Keys; import ListItem = KMXPlus.ListItem; import KeysFlicks = KMXPlus.KeysFlicks; import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js'; +import { MarkerTracker, MarkerUse } from './marker-tracker.js'; export class KeysCompiler extends SectionCompiler { + static validateMarkers( + keyboard: LDMLKeyboard.LKKeyboard, + mt: MarkerTracker + ): boolean { + keyboard.keys?.key?.forEach(({ to }) => + mt.add(MarkerUse.emit, MarkerParser.allReferences(to)) + ); + return true; + } public get id() { return constants.section.keys; @@ -20,7 +30,7 @@ export class KeysCompiler extends SectionCompiler { * @returns just the non-touch layers. */ public hardwareLayers() { - return this.keyboard.layers?.filter(({form}) => form !== 'touch'); + return this.keyboard.layers?.filter(({ form }) => form !== "touch"); } public validate() { @@ -30,7 +40,7 @@ export class KeysCompiler extends SectionCompiler { const usedKeys = allUsedKeyIdsInLayers(this.keyboard?.layers); const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]); for (let key of uniqueKeys) { - const {id, flicks} = key; + const { id, flicks } = key; if (!usedKeys.has(id)) { continue; // unused key, ignore } @@ -38,10 +48,14 @@ export class KeysCompiler extends SectionCompiler { if (!flicks) { continue; // no flicks } - const flickEntry = this.keyboard.keys?.flicks?.find(x => x.id === flicks); - if (!flickEntry ) { + const flickEntry = this.keyboard.keys?.flicks?.find( + (x) => x.id === flicks + ); + if (!flickEntry) { valid = false; - this.callbacks.reportMessage(CompilerMessages.Error_MissingFlicks({flicks, id})); + this.callbacks.reportMessage( + CompilerMessages.Error_MissingFlicks({ flicks, id }) + ); } } @@ -53,8 +67,9 @@ export class KeysCompiler extends SectionCompiler { if (hardwareLayers.length >= 1) { // validate all errors for (let layers of hardwareLayers) { - for(let layer of layers.layer) { - valid = this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found + for (let layer of layers.layer) { + valid = + this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found } } // TODO-LDML: } else { touch? @@ -84,11 +99,13 @@ export class KeysCompiler extends SectionCompiler { /* c8 ignore next 3 */ if (hardwareLayers.length > 1) { // validation should have already caught this - throw Error(`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`); + throw Error( + `Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}` + ); } else if (hardwareLayers.length === 1) { const theLayers = hardwareLayers[0]; const { form } = theLayers; - for(let layer of theLayers.layer) { + for (let layer of theLayers.layer) { this.compileHardwareLayerToKmap(sections, layer, sect, form); } } // else: TODO-LDML do nothing if only touch layers @@ -98,7 +115,9 @@ export class KeysCompiler extends SectionCompiler { public loadFlicks(sections: DependencySections, sect: Keys) { for (let lkflicks of this.keyboard.keys.flicks) { - let flicks: KeysFlicks = new KeysFlicks(sections.strs.allocString(lkflicks.id)); + let flicks: KeysFlicks = new KeysFlicks( + sections.strs.allocString(lkflicks.id) + ); for (let lkflick of lkflicks.flick) { let flags = 0; @@ -106,10 +125,14 @@ export class KeysCompiler extends SectionCompiler { if (!to.isOneChar) { flags |= constants.keys_flick_flags_extend; } - let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions); + let directions: ListItem = sections.list.allocListFromSpaces( + sections.strs, + lkflick.directions + ); flicks.flicks.push({ directions, flags, + // TODO-LDML: markers,variables to, }); } @@ -132,19 +155,33 @@ export class KeysCompiler extends SectionCompiler { if (!!key.gap) { flags |= constants.keys_key_flags_gap; } - if (key.transform === 'no') { + if (key.transform === "no") { flags |= constants.keys_key_flags_notransform; } const id = sections.strs.allocString(key.id); - const longPress: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.longPress); - const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault); - const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap); + const longPress: ListItem = sections.list.allocListFromEscapedSpaces( + sections.strs, + // TODO-LDML: markers,variables + key.longPress + ); + const longPressDefault = sections.strs.allocAndUnescapeString( + // TODO-LDML: markers,variables + key.longPressDefault + ); + const multiTap: ListItem = sections.list.allocListFromEscapedSpaces( + sections.strs, + // TODO-LDML: markers,variables + key.multiTap + ); const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word - const to = sections.strs.allocAndUnescapeString(key.to, true); + const toRaw = key.to; + // TODO-LDML: variables + let toCooked = sections.vars.substituteMarkerString(toRaw); + const to = sections.strs.allocAndUnescapeString(toCooked, true); if (!to.isOneChar) { flags |= constants.keys_key_flags_extend; } - const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 + const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 sect.keys.push({ flags, flicks, @@ -166,12 +203,17 @@ export class KeysCompiler extends SectionCompiler { * @param layer * @returns */ - private validateHardwareLayerForKmap(hardware: string, layer: LDMLKeyboard.LKLayer) { + private validateHardwareLayerForKmap( + hardware: string, + layer: LDMLKeyboard.LKLayer + ) { let valid = true; const { modifier } = layer; if (!validModifier(modifier)) { - this.callbacks.reportMessage(CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id })); + this.callbacks.reportMessage( + CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id }) + ); valid = false; } @@ -179,21 +221,31 @@ export class KeysCompiler extends SectionCompiler { /* c8 ignore next 5 */ if (!keymap) { // not reached due to XML validation - this.callbacks.reportMessage(CompilerMessages.Error_InvalidHardware({ form: hardware })); + this.callbacks.reportMessage( + CompilerMessages.Error_InvalidHardware({ form: hardware }) + ); valid = false; } const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]); if (layer.row.length > keymap.length) { - this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows()); + this.callbacks.reportMessage( + CompilerMessages.Error_HardwareLayerHasTooManyRows() + ); valid = false; } for (let y = 0; y < layer.row.length && y < keymap.length; y++) { - const keys = layer.row[y].keys.split(' '); + const keys = layer.row[y].keys.split(" "); if (keys.length > keymap[y].length) { - this.callbacks.reportMessage(CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware, modifier })); + this.callbacks.reportMessage( + CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ + row: y + 1, + hardware, + modifier, + }) + ); valid = false; } @@ -201,14 +253,24 @@ export class KeysCompiler extends SectionCompiler { for (let key of keys) { x++; - let keydef = uniqueKeys.find(x => x.id == key); + let keydef = uniqueKeys.find((x) => x.id == key); if (!keydef) { - this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({ keyId: key, col: x + 1, row: y + 1, layer: layer.id, form: 'hardware' })); + this.callbacks.reportMessage( + CompilerMessages.Error_KeyNotFoundInKeyBag({ + keyId: key, + col: x + 1, + row: y + 1, + layer: layer.id, + form: "hardware", + }) + ); valid = false; continue; } if (!keydef.to && !keydef.gap && !keydef.switch) { - this.callbacks.reportMessage(CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })); + this.callbacks.reportMessage( + CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }) + ); valid = false; continue; } @@ -222,7 +284,7 @@ export class KeysCompiler extends SectionCompiler { sections: DependencySections, layer: LDMLKeyboard.LKLayer, sect: Keys, - hardware: string, + hardware: string ): Keys { const mod = translateLayerAttrToModifier(layer); const keymap = Constants.HardwareToKeymap.get(hardware); @@ -231,7 +293,7 @@ export class KeysCompiler extends SectionCompiler { for (let row of layer.row) { y++; - const keys = row.keys.split(' '); + const keys = row.keys.split(" "); let x = -1; for (let key of keys) { x++; diff --git a/developer/src/kmc-ldml/src/compiler/marker-tracker.ts b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts new file mode 100644 index 00000000000..8007f29e5bf --- /dev/null +++ b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts @@ -0,0 +1,72 @@ +/** + * Verb for MarkerTracker.add() + */ +export enum MarkerUse { + /** outputs this marker into context (e.g. transform to= or key to=) */ + emit, + /** consumes this marker out of the context (e.g. transform from=) */ + consume, + /** matches the marker, but doesn't consume (e.g. display to=) */ + match, + /** variable definition: might consume, emit, or match. */ + variable, +} + +type MarkerSet = Set; + +/** Tracks usage of markers */ +export class MarkerTracker { + /** markers that were emitted */ + emitted: MarkerSet; + /** markers that were consumed and removed from the context */ + consumed: MarkerSet; + /** markers that were matched, but not necessarily consumed */ + matched: MarkerSet; + /** all markers */ + all: MarkerSet; + + constructor() { + this.emitted = new Set(); + this.consumed = new Set(); + this.matched = new Set(); + this.all = new Set(); + } + + /** + * + * @param verb what kind of use we are adding + * @param markers list of markers to add + */ + add(verb: MarkerUse, markers: string[]) { + if (!markers.length) { + return; // skip if empty + } + if (verb == MarkerUse.emit) { + markers.forEach((m) => { + this.emitted.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.consume) { + markers.forEach((m) => { + this.consumed.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.match) { + markers.forEach((m) => { + this.matched.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.variable) { + markers.forEach((m) => { + // we don't know, so add it to all three + this.matched.add(m); + this.emitted.add(m); + this.consumed.add(m); + this.all.add(m); + }); + /* c8 skip next 3 */ + } else { + throw Error(`Internal error: unsupported verb ${verb} for match`); + } + } +} diff --git a/developer/src/kmc-ldml/src/compiler/messages.ts b/developer/src/kmc-ldml/src/compiler/messages.ts index f0feb6f5996..e3ec8bcbf24 100644 --- a/developer/src/kmc-ldml/src/compiler/messages.ts +++ b/developer/src/kmc-ldml/src/compiler/messages.ts @@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; -// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.KeyboardCompiler; +// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevFatal = CompilerErrorSeverity.Fatal | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -35,9 +35,9 @@ export class CompilerMessages { m(this.HINT_LocaleIsNotMinimalAndClean, `Locale '${o.sourceLocale}' is not minimal or correctly formatted and should be '${o.locale}'`); static HINT_LocaleIsNotMinimalAndClean = SevHint | 0x0008; - static Error_VkeyIsNotValid = (o:{vkey: string}) => - m(this.ERROR_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`); - static ERROR_VkeyIsNotValid = SevError | 0x0009; + static Hint_VkeyIsNotValid = (o:{vkey: string}) => + m(this.HINT_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`); + static HINT_VkeyIsNotValid = SevHint | 0x0009; static Hint_VkeyIsRedundant = (o:{vkey: string}) => m(this.HINT_VkeyIsRedundant, `Virtual key '${o.vkey}' is mapped to itself, which is redundant.`); @@ -130,5 +130,9 @@ export class CompilerMessages { static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) => m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`); static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020; + + static Error_MissingMarkers = (o: { ids: string[] }) => + m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`); + static ERROR_MissingMarkers = SevError | 0x0021; } diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts index 5150108e6c8..9e187fafe62 100644 --- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts @@ -32,7 +32,8 @@ export class SectionCompiler { const defaults = new Set([ constants.section.strs, constants.section.list, - constants.section.elem + constants.section.elem, + constants.section.vars, ]); return defaults; } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index d0173b7bfb3..fa1f828622a 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -1,5 +1,5 @@ import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types'; import { SectionCompiler } from "./section-compiler.js"; import Bksp = KMXPlus.Bksp; @@ -15,10 +15,21 @@ import LKTransform = LDMLKeyboard.LKTransform; import LKTransforms = LDMLKeyboard.LKTransforms; import { verifyValidAndUnique } from "../util/util.js"; import { CompilerMessages } from "./messages.js"; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; type TransformCompilerType = 'simple' | 'backspace'; -class TransformCompiler extends SectionCompiler { +export class TransformCompiler extends SectionCompiler { + + static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean { + keyboard?.transforms?.forEach(transforms => + transforms.transformGroup.forEach(transformGroup => { + transformGroup.transform?.forEach(({ to, from }) => { + mt.add(MarkerUse.emit, MarkerParser.allReferences(to)); + mt.add(MarkerUse.consume, MarkerParser.allReferences(from)); + })})); + return true; + } protected type: T; @@ -130,6 +141,10 @@ class TransformCompiler cookedTo = sections.vars.substituteStrings(cookedTo, sections); } + // add in markers. idempotent if no markers. + cookedFrom = sections.vars.substituteMarkerString(cookedFrom); // TODO-LDML: need to support \m{.} here, maybe other edge cases + cookedTo = sections.vars.substituteMarkerString(cookedTo); + result.from = sections.strs.allocAndUnescapeString(cookedFrom); // TODO-LDML: not unescaped here, done previously result.to = sections.strs.allocAndUnescapeString(cookedTo); // TODO-LDML: not unescaped here, done previously return result; diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts index 19451c4db43..b6e61367508 100644 --- a/developer/src/kmc-ldml/src/compiler/vars.ts +++ b/developer/src/kmc-ldml/src/compiler/vars.ts @@ -1,5 +1,5 @@ import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types'; import { VariableParser } from '@keymanapp/common-types'; import { SectionCompiler } from "./section-compiler.js"; import Vars = KMXPlus.Vars; @@ -9,6 +9,10 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem; import DependencySections = KMXPlus.DependencySections; import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile; import { CompilerMessages } from "./messages.js"; +import { KeysCompiler } from "./keys.js"; +import { TransformCompiler } from "./tran.js"; +import { DispCompiler } from "./disp.js"; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; export class VarsCompiler extends SectionCompiler { public get id() { return constants.section.vars; @@ -17,7 +21,8 @@ export class VarsCompiler extends SectionCompiler { public get dependencies(): Set { const defaults = new Set([ constants.section.strs, - constants.section.elem + constants.section.elem, + constants.section.list, ]); defaults.delete(this.id); return defaults; @@ -29,7 +34,6 @@ export class VarsCompiler extends SectionCompiler { public validate(): boolean { let valid = true; - // TODO-LDML scan for markers? // Check for duplicate ids const allIds = new Set(); @@ -130,14 +134,61 @@ export class VarsCompiler extends SectionCompiler { })); valid = false; } + + valid = this.validateMarkers() && valid; // accumulate validity + + return valid; + } + + private collectMarkers(mt : MarkerTracker) : boolean { + let valid = true; + + // call our friends to validate + valid = this.validateVarsMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = KeysCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = TransformCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = DispCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + return valid; } + private validateMarkers(): boolean { + const mt = new MarkerTracker(); + let valid = this.collectMarkers(mt); + // see if there are any matched-but-not-emitted + const matchedNotEmitted : Set = new Set(); + for (const m of mt.matched.values()) { + if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker + if (!mt.emitted.has(m)) { + matchedNotEmitted.add(m); + } + } + for (const m of mt.consumed.values()) { + if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker + if (!mt.emitted.has(m)) { + matchedNotEmitted.add(m); + } + } + + // report once + if (matchedNotEmitted.size > 0) { + this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() })); + valid = false; + } + return valid; + } + + validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker) : boolean { + keyboard?.variables?.string?.forEach(({value}) => + mt.add(MarkerUse.variable, MarkerParser.allReferences(value))); + return true; + } + public compile(sections: DependencySections): Vars { const result = new Vars(); const variables = this.keyboard?.variables; - + // we always have vars, it's depended on by other sections if (!variables) return result; // Empty vars, to simplify other sections // we already know the variables do not conflict with each other @@ -153,6 +204,14 @@ export class VarsCompiler extends SectionCompiler { variables?.unicodeSet?.forEach((e) => this.addUnicodeSet(result, e, sections)); + // reload markers - TODO-LDML: double work! + const mt = new MarkerTracker(); + this.collectMarkers(mt); + + // collect all markers, excluding the match-all + const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort(); + result.markers = sections.list.allocList(sections.strs, allMarkers); + return result.valid() ? result : null; } diff --git a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts index d951080b9b4..0d0c8020954 100644 --- a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts @@ -44,6 +44,10 @@ export class LdmlKeyboardVisualKeyboardCompiler { let keydef = source.keyboard.keys?.key?.find(x => x.id == key); + if (!keydef) { + throw Error(`Internal Error: could not find key id="${key}" in layer "${layer.id || ''}", row "${y}"`); + } + vk.keys.push({ flags: VisualKeyboard.VisualKeyboardKeyFlags.kvkkUnicode, shift: shift, diff --git a/developer/src/kmc-ldml/src/compiler/vkey.ts b/developer/src/kmc-ldml/src/compiler/vkey.ts index 1b7d71f3008..2d6d44d9d01 100644 --- a/developer/src/kmc-ldml/src/compiler/vkey.ts +++ b/developer/src/kmc-ldml/src/compiler/vkey.ts @@ -19,13 +19,15 @@ export class VkeyCompiler extends SectionCompiler { this.keyboard.vkeys.vkey.forEach(vk => { if(LdmlVkeyNames[vk.from] === undefined) { - this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.from})); - valid = false; + // TODO-LDML: When we do #7135 this may need to change back to an error. + this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.from})); + return; } if(LdmlVkeyNames[vk.to] === undefined) { - this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.to})); - valid = false; + // TODO-LDML: When we do #7135 this may need to change back to an error. + this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.to})); + return; } if(vk.from == vk.to) { diff --git a/developer/src/kmc-ldml/test/fixtures/basic.txt b/developer/src/kmc-ldml/test/fixtures/basic.txt index 0fe6d67a308..cbbb4038915 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic.txt +++ b/developer/src/kmc-ldml/test/fixtures/basic.txt @@ -257,7 +257,7 @@ block(keys) # struct COMP_KMXPLUS_KEYS { index(strNull,strHmaqtugha,2) # KMXPLUS_STR 'hmaqtugha' 00 00 00 00 # KMXPLUS_STR switch 0A 00 00 00 # KMX_DWORD width*10 - 01 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e' + 02 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e' 00 00 00 00 # STR longPressDefault 00 00 00 00 # TODO: index(listNull,listNull,4) # LIST multiTap 00 00 00 00 # flicks 0 @@ -324,21 +324,25 @@ block(layr) # struct COMP_KMXPLUS_LAYR { block(list) # struct COMP_KMXPLUS_LAYR_LIST { 6c 69 73 74 # KMX_DWORD header.ident; // 0000 Section name - list diff(list,endList) # KMX_DWORD header.size; // 0004 Section length - 02 00 00 00 # KMX_DWORD listCount (should be 2) - 02 00 00 00 # KMX_DWORD indexCount (should be 2) + 03 00 00 00 # KMX_DWORD listCount (should be 2) + 03 00 00 00 # KMX_DWORD indexCount (should be 2) # list #0 the null list block(listNull) 00 00 00 00 #index(indexNull,indexNull,2) # KMX_DWORD list index (0) 00 00 00 00 # KMX_DWORD lists[0].count - # list #1 the ae list + block(listA) + 00 00 00 00 # first index + 01 00 00 00 #count block(listAe) - 00 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0) + 01 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0) 02 00 00 00 # KMX_DWORD count block(endLists) # indices #block(indexNull) # No null index # index(strNull,strNull,2) # KMXPLUS_STR string index + block(indexA) + index(strNull,strA,2) # a block(indexAe) index(strNull,strA,2) # KMXPLUS_STR a index(strNull,strElemBkspFrom2,2) # KMXPLUS_STR e @@ -401,6 +405,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { diff(strs,strName) sizeof(strName,2) diff(strs,strFromSet) sizeof(strFromSet,2) diff(strs,strUSet) sizeof(strUSet,2) + diff(strs,strAmarker) sizeof(strAmarker,2) diff(strs,strElemTranFrom1) sizeof(strElemTranFrom1,2) diff(strs,strElemTranFrom1a) sizeof(strElemTranFrom1a,2) diff(strs,strElemTranFrom1b) sizeof(strElemTranFrom1b,2) @@ -421,6 +426,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { diff(strs,strTranTo) sizeof(strTranTo,2) diff(strs,strKeys) sizeof(strKeys,2) diff(strs,strIndicator) sizeof(strIndicator,2) + diff(strs,strSentinel0001) sizeof(strSentinel0001,2) # String table -- block(x) is used to store the null u16char at end of each string @@ -433,6 +439,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd' block(strFromSet) 5B 00 5C 00 75 00 31 00 41 00 37 00 35 00 2D 00 5C 00 75 00 31 00 41 00 37 00 39 00 5D 00 block(x) 00 00 # [\u1a75-\u1a79] block(strUSet) 5b 00 61 00 62 00 63 00 5d 00 block(x) 00 00 # '[abc]' + block(strAmarker) 5C 00 6D 00 7B 00 61 00 7D 00 block(x) 00 00 # '\m{a}' block(strElemTranFrom1) 5E 00 block(x) 00 00 # '^' block(strElemTranFrom1a) 5E 00 61 00 block(x) 00 00 # '^a' block(strElemTranFrom1b) 5E 00 65 00 block(x) 00 00 # '^e' @@ -458,7 +465,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { block(strKeys) 90 17 b6 17 block(x) 00 00 # 'ថា' # block(strIndicator) 3d d8 40 de block(x) 00 00 # '🙀' - + block(strSentinel0001) FF FF 01 00 block(x) 00 00 # U+FFFF U+0001 @@ -478,10 +485,15 @@ block(tran) # struct COMP_KMXPLUS_TRAN { block(tranGroupStart) # COMP_KMXPLUS_TRAN_GROUP # group 0 00 00 00 00 # KMX_DWORD type = transform - 01 00 00 00 # KMX_DWORD count + 02 00 00 00 # KMX_DWORD count diff(tranTransformStart,tranTransform0,16) # KMX_DWORD index # group 1 + 00 00 00 00 # KMX_DWORD type = transform + 01 00 00 00 # KMX_DWORD count + diff(tranTransformStart,tranTransform2,16) # KMX_DWORD index + + # group 2 01 00 00 00 # KMX_DWORD type = reorder 01 00 00 00 # KMX_DWORD count diff(tranReorderStart,tranReorder0,8) # KMX_DWORD index @@ -494,6 +506,18 @@ block(tran) # struct COMP_KMXPLUS_TRAN { index(strNull,strNull,2) # mapFrom index(strNull,strNull,2) # mapTo + block(tranTransform1) + index(strNull,strA,2) # KMXPLUS_STR from; 'a' + index(strNull,strSentinel0001,2) # KMXPLUS_STR to; \m{a} + index(strNull,strNull,2) # mapFrom + index(strNull,strNull,2) # mapTo + + block(tranTransform2) # Next group + index(strNull,strSentinel0001,2) # KMXPLUS_STR from; (\m{a}) + index(strNull,strNull,2) # KMXPLUS_STR to; (none) + index(strNull,strNull,2) # mapFrom + index(strNull,strNull,2) # mapTo + # reorders block(tranReorderStart) # COMP_KMXPLUS_TRAN_REORDER block(tranReorder0) @@ -520,23 +544,29 @@ block(uset) block(vars) # struct COMP_KMXPLUS_VARS { 76 61 72 73 # KMX_DWORD header.ident; // 0000 Section name - vars diff(vars,varsEnd) # KMX_DWORD header.size; // 0004 Section length - 00 00 00 00 # KMX_DWORD markers - list + 01 00 00 00 # KMX_DWORD markers - list 1 ['a'] diff(varsBegin,varsEnd,16) # KMX_DWORD varCount - # var 0 block(varsBegin) + # var 0 + 00 00 00 00 # KMX_DWORD type = str + index(strNull,strA,2) # KMXPLUS_STR id 'a' + index(strNull,strAmarker,2) # KMXPLUS_STR value '\m{a}' + 00 00 00 00 # KMXPLUS_ELEM + + # var 1 01 00 00 00 # KMX_DWORD type = set index(strNull,strVse,2) # KMXPLUS_STR id 'vse' index(strNull,strSet,2) # KMXPLUS_STR value 'a b c' 01 00 00 00 # KMXPLUS_ELEM elem 'a b c' see 'elemSet' - # var 1 + # var 2 00 00 00 00 # KMX_DWORD type = string index(strNull,strVst,2) # KMXPLUS_STR id 'vst' index(strNull,strSet2,2) # KMXPLUS_STR value 'abc' 00 00 00 00 # KMXPLUS_ELEM elem - # var 2 + # var 3 02 00 00 00 # KMX_DWORD type = string index(strNull,strVus,2) # KMXPLUS_STR id 'vus' index(strNull,strUSet,2) # KMXPLUS_STR value '[abc]' diff --git a/developer/src/kmc-ldml/test/fixtures/basic.xml b/developer/src/kmc-ldml/test/fixtures/basic.xml index 0eb6676490f..2d345324d9a 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic.xml +++ b/developer/src/kmc-ldml/test/fixtures/basic.xml @@ -37,6 +37,7 @@ + @@ -45,6 +46,11 @@ + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml new file mode 100644 index 00000000000..fd63c36b025 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml new file mode 100644 index 00000000000..cbe36d4db1c --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 30dde83e268..a6e05ebf39b 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -19,6 +19,7 @@ import DependencySections = KMXPlus.DependencySections; import Section = KMXPlus.Section; import { ElemCompiler, ListCompiler, StrsCompiler } from '../../src/compiler/empty-compiler.js'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; +import { VarsCompiler } from '../../src/compiler/vars.js'; // import Vars = KMXPlus.Vars; /** @@ -92,7 +93,7 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section const parentId = parentCompiler.id; if (!dependencies) { // default dependencies - dependencies = [ StrsCompiler, ListCompiler, ElemCompiler ]; + dependencies = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ]; } for (const dep of dependencies) { const compiler = new dep(source, callbacks); diff --git a/developer/src/kmc-ldml/test/test-tran.ts b/developer/src/kmc-ldml/test/test-tran.ts index 6e25cb4ac87..ec77ab02f2d 100644 --- a/developer/src/kmc-ldml/test/test-tran.ts +++ b/developer/src/kmc-ldml/test/test-tran.ts @@ -1,7 +1,6 @@ import 'mocha'; import { assert } from 'chai'; import { TranCompiler, BkspCompiler } from '../src/compiler/tran.js'; -import { VarsCompiler } from '../src/compiler/vars.js'; import { BASIC_DEPENDENCIES, UsetCompiler } from '../src/compiler/empty-compiler.js'; import { CompilerMessages } from '../src/compiler/messages.js'; import { compilerTestCallbacks, testCompilationCases } from './helpers/index.js'; @@ -10,7 +9,7 @@ import { KMXPlus } from '@keymanapp/common-types'; import Tran = KMXPlus.Tran;// for tests… import Bksp = KMXPlus.Bksp;// for tests… import { constants } from '@keymanapp/ldml-keyboard-constants'; -const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler, VarsCompiler ]; +const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler ]; const bkspDependencies = tranDependencies; describe('tran', function () { diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts index a3cb303b825..b18f546f278 100644 --- a/developer/src/kmc-ldml/test/test-vars.ts +++ b/developer/src/kmc-ldml/test/test-vars.ts @@ -5,6 +5,9 @@ import { CompilerMessages } from '../src/compiler/messages.js'; import { CompilerMessages as KmnCompilerMessages } from '@keymanapp/kmc-kmn'; import { testCompilationCases } from './helpers/index.js'; import { KMXPlus } from '@keymanapp/common-types'; +import { BASIC_DEPENDENCIES } from '../src/compiler/empty-compiler.js'; +// now that 'everything' depends on vars, we need an explicit dependency here +const varsDependencies = BASIC_DEPENDENCIES.filter(c => c !== VarsCompiler); import Vars = KMXPlus.Vars; @@ -182,5 +185,32 @@ describe('vars', function () { CompilerMessages.Error_MissingStringVariable({id: 'missingStringInSet'}) ], }, - ]); + ], varsDependencies); + describe('markers', function () { + this.slow(500); // 0.5 sec -- json schema validation takes a while + + testCompilationCases(VarsCompiler, [ + { + subpath: 'sections/vars/markers-maximal.xml', + callback(sect) { + const vars = sect; + assert.ok(vars.markers); + assert.sameDeepOrderedMembers(vars.markers.toStringArray(), + ['m','x']); + }, + }, + { + subpath: 'sections/vars/fail-markers-badref-0.xml', + errors: [ + CompilerMessages.Error_MissingMarkers({ + ids: [ + 'doesnt_exist_1', + 'doesnt_exist_2', + 'doesnt_exist_3', + ] + }), + ], + }, + ], varsDependencies); + }); }); diff --git a/developer/src/kmc-ldml/test/test-vkey.ts b/developer/src/kmc-ldml/test/test-vkey.ts index 03c27cbd3f6..dec24125e2a 100644 --- a/developer/src/kmc-ldml/test/test-vkey.ts +++ b/developer/src/kmc-ldml/test/test-vkey.ts @@ -37,19 +37,19 @@ describe('vkey compiler', function () { assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Info_MultipleVkeysHaveSameTarget({vkey: 'Q'})); }); - it('should error on invalid "from" vkey', async function() { + it('should hint on invalid "from" vkey', async function() { let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-from-vkey.xml', compilerTestCallbacks) as Vkey; - assert.isNull(vkey); + assert.isNotNull(vkey); assert.equal(compilerTestCallbacks.messages.length, 2); - assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'q'})); - assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Error_VkeyIsNotValid({vkey: 'HYFEN'})); + assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'q'})); + assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'HYFEN'})); }); - it('should error on invalid "to" vkey', async function() { + it('should hint on invalid "to" vkey', async function() { let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-to-vkey.xml', compilerTestCallbacks) as Vkey; - assert.isNull(vkey); + assert.isNotNull(vkey); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'A-ACUTE'})); + assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'A-ACUTE'})); }); it('should error on repeated vkeys', async function() { diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 605292583d7..8d68c549790 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -28,7 +28,15 @@ if cpp_compiler.get_id() == 'emscripten' # wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+ flags += ['-fwasm-exceptions'] lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6'] - links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + links += ['-fwasm-exceptions', '--bind'] + if cpp_compiler.version().version_compare('>=3.1.44') + # emscripten 3.1.44 removes .asm object and so we need to export `wasmExports` + # #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523 + links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']'] + else + # emscripten < 3.1.44 does not include `wasmExports` + links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + endif endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index ee45d4eb4a7..3c4f4c57c1c 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -4,28 +4,30 @@ On Linux, you can build the following projects: -* [Keyman for Linux](#keyman-for-linux) -* [Keyman Core](#keyman-core) (Linux only) (aka core) -* [Keyman for Android](#keyman-for-android) - -* Keyman Core (wasm targets) -* Common/Web -* KeymanWeb +- [Keyman Core](#keyman-core) (aka core) +- [Keyman for Linux](#keyman-for-linux) +- [Keyman Web](#keyman-web) +- [Keyman for Android](#keyman-for-android) + +- Common/Web The following projects **cannot** be built on Linux: -* Keyman for Windows -* Keyman Developer -* Keyman for macOS -* Keyman for iOS +- Keyman for Windows +- Keyman Developer +- Keyman for macOS +- Keyman for iOS -## System Requirements +## Requirements -* Minimum Ubuntu version: Ubuntu 20.04 +### System Requirements + +- Minimum Ubuntu version: Ubuntu 20.04 Other Linux distributions will also work if appropriate dependencies are installed. -## Repository Paths +### Repository Paths Recommended filesystem layout: @@ -38,7 +40,7 @@ $HOME/keyman/ ... ``` -## Prerequisites +### Prerequisites The current list of dependencies can be found in the `Build-Depends` section of `linux/debian/control`. They are most easily installed with the `mk-build-deps` tool: @@ -49,101 +51,139 @@ sudo apt install devscripts equivs sudo mk-build-deps --install linux/debian/control ``` -### Node.js +#### Node.js -Node.js v18 is required for Core build, Web tests, and Developer command line tools. +Node.js v18 is required for Core builds, Web builds, and Developer command line tool builds and usage. -## Keyman for Linux +You can install it with: + +```shell +curl -sL https://deb.nodesource.com/setup_18.x | bash +apt-get -q -y install nodejs +``` + +#### Emscripten -All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites). +You'll also have to install `emscripten` (version 3.1.44 is known to work): -Building: +```shell +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install latest +./emsdk activate latest +export EMSCRIPTEN_BASE=$(pwd)/upstream/emscripten +``` -* [Building Keyman for Linux](../../linux/README.md) +**NOTE:** Don't put EMSDK on the path, i.e. don't source `emsdk_env.sh`. ## Keyman Core -All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites). +All dependencies are already installed if you followed the instructions under +[Prerequisites](#prerequisites). -Building: +### Building Keyman Core -* [Building Keyman Core](../../core/doc/BUILDING.md) +Keyman Core can be built with the `core/build.sh` script. -## Docker Builder +- [Building Keyman Core](../../core/doc/BUILDING.md) -The Docker builder allows you to perform a linux build from anywhere Docker is supported. -To build the docker image: +## Keyman for Linux -```shell -cd linux -docker pull ubuntu:latest -docker build . -t keymanapp/keyman-linux-builder:latest -``` +All dependencies are already installed if you followed the instructions +under [Prerequisites](#prerequisites). -Once the image is built, it may be used to build parts of Keyman. +### Building Keyman for Linux -- core +Keyman for Linux can be built with the `linux/build.sh` script. -```shell -# build 'core' in docker -cd ../core -# keep linux build artifacts separate -mkdir -p build/linux -docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keymanapp/keyman-linux-builder:latest bash -c 'core/build.sh --debug' +- [Building Keyman for Linux](../../linux/README.md) + +## Keyman Web + +Most dependencies are already installed if you followed the instructions under +[Prerequisites](#prerequisites). You'll still have to install Chrome: + +```bash +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +sudo apt install ./google-chrome-stable_current_amd64.deb ``` -- linux +And add the `CHROME_BIN` environment variable to `.bashrc: -```shell -# build 'linux' installation in docker -cd keymanapp/keyman -docker run -it --rm -v $(pwd):/home/build/src/keyman -w /home/build/src/keyman keymanapp/keyman-linux-builder:latest bash -c "DESTDIR=. linux/build.sh --debug build install" +```bash +export CHROME_BIN=/opt/google/chrome/chrome ``` +### Environment variables for Keyman Web + +`CHROME_BIN` pointing to the Google Chrome binary. + +### Building Keyman Web + +Keyman Web can be built with the `web/build.sh` script. + +- [Building Keyman Web](../../web/README.md) + ## Keyman for Android **Dependencies:** -* [Base](#base-dependencies) -* [Web](./windows#web-dependencies) +Most dependencies are already installed if you followed the instructions +under [Prerequisites](#prerequisites). **Additional requirements:** -* Android SDK -* [Android Studio](https://developer.android.com/studio/install#linux) -* Gradle -* Maven -* OpenJDK 11 (for Keyman 17.0+) -* pandoc +- [Android Studio](https://developer.android.com/studio/install#linux) + or sdkmanager +- Maven +- pandoc +- Android SDK +- Gradle +- jq + +If you only use the command line you don't need Android Studio, however +to do development it's recommended to install it. Run Android Studio once after installation to install additional components such as emulator images and SDK updates. -**Required environment variable:** +Maven, jq and pandoc can be installed with: + +```shell +sudo apt update +sudo apt install maven pandoc jq +``` -* `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`) +If necessary, Android SDK and Gradle will be installed by the build script. +In order for that to work, run the following command once. You won't need +this if you install Android SDK through Android Studio. -**Recommended environment variable:** +```shell +sudo apt install sdkmanager +sudo sdkmanager platform-tools +sudo chown -R $USER:$USER /opt/android-sdk/ +sdkmanager --licenses +``` -* [`JAVA_HOME`](#java_home) +### Environment variables for Keyman for Android -Building: +**Required environment variable:** -* [Building Keyman for Android](../../android/README.md) +- `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`) -## Prerequisites +**Recommended environment variable:** -Many dependencies are only required for specific projects. +- [`JAVA_HOME`](#java_home) -### Base Dependencies +### Building Keyman for Android -**Environment variables:** +Keyman for Android can be built with the `android/build.sh` script. -* -- +- [Building Keyman for Android](../../android/README.md) -## Notes on Environment Variables +### Notes on Environment Variables -### JAVA_HOME +#### JAVA_HOME This environment variable tells Gradle what version of Java to use for building Keyman for Android. OpenJDK 11 is used for master. @@ -166,3 +206,59 @@ older versions, you can set `JAVA_HOME_11` to the OpenJDK 11 path and from command line. But note that you do need to update your `JAVA_HOME` env var to the associated version before opening Android Studio and loading any Android projects. `JAVA_HOME_11` is mostly used by CI. + +## Docker Builder + +The Docker builder allows you to perform a build from anywhere Docker is supported. + +To build the docker image: + +```shell +cd linux +docker pull ubuntu:latest +docker build . -t keymanapp/keyman-linux-builder:latest +``` + +Once the image is built, it may be used to build parts of Keyman. + +**Note** that it's not yet possible to run tests in the Docker container. + +- core + + ```shell + # build 'Keyman Core' in docker + # keep linux build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/core/build/linux + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/core/build/linux:/home/build/build/core/build \ + keymanapp/keyman-linux-builder:latest \ + core/build.sh --debug + ``` + +- linux + + ```shell + # build 'Keyman for Linux' installation in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + --entrypoint /bin/bash keymanapp/keyman-linux-builder:latest \ + -c 'DESTDIR=/home/build /usr/bin/bashwrapper linux/build.sh --debug build install' + ``` + +- Keyman Web + + ```shell + # build 'Keyman Web' in docker + docker run --privileged -it --rm \ + -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-linux-builder:latest \ + web/build.sh --debug + ``` + +- Keyman for Android + + ```shell + # build 'Keyman for Android' in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-linux-builder:latest \ + android/build.sh --debug + ``` diff --git a/docs/build/windows.md b/docs/build/windows.md index fd66445b81e..9faebc3d632 100644 --- a/docs/build/windows.md +++ b/docs/build/windows.md @@ -153,6 +153,7 @@ choco install git jq python ninja pandoc refreshenv # choco meson (0.55) is too old, 1.0 required: python -m pip install meson +``` **Environment variables**: * [`KEYMAN_ROOT`](#keyman_root) diff --git a/docs/linux/README.md b/docs/linux/README.md index 18e65699ad0..d848e215ed1 100644 --- a/docs/linux/README.md +++ b/docs/linux/README.md @@ -2,27 +2,18 @@ ## Projects -- [keyman-config](../../linux/keyman-config) - km-config and some other tools to install, uninstall - and view information about Keyman keyboard packages. +- [keyman-config](../../linux/keyman-config) - `km-config` and some other tools + to install, uninstall and view information about Keyman keyboard packages. - [ibus-keyman](../../linux/ibus-keyman) - IBUS integration to use .kmp Keyman keyboards +- [keyman-system-service](../../linux/keyman-system-service) - A DBus system service + that allows to perform keyboard related actions when running under Wayland. - [core](../../core) - common keyboardprocessor library See [license information](../../linux/LICENSE.md) about licensing. ## Linux Requirements/Setup -- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo - -- Install packages required for building and developing Keyman for Linux. - The list of required packages can be seen in `linux/debian/control`. - It is easiest to use the `mk-build-deps` tool to install the - dependencies: - - ```bash - sudo apt update - sudo apt install devscripts equivs - sudo mk-build-deps --install linux/debian/control - ``` +See [document in ../build](../build/linux-ubuntu.md). ## Compiling from Command Line @@ -61,15 +52,18 @@ for details on building Linux packages for Keyman. ## Testing -### keyman-config - -The unit tests can be run with the following command: +The tests can be run with the following command: ```bash -cd linux/keyman-config -./run-tests.sh +linux/build.sh test ``` +To just run the unit tests without integration tests, add the +`--no-integration` parameter. + +It's also possible to only run the tests for one of the subprojects. You +can use `build.sh` in the subdirectory for that. + ### ibus-keyman If you want to run the ibus-keyman tests with Wayland, you'll have to diff --git a/docs/linux/packaging.md b/docs/linux/packaging.md index bfb6a9d13d3..c133575d049 100644 --- a/docs/linux/packaging.md +++ b/docs/linux/packaging.md @@ -10,7 +10,7 @@ We use different channels to build and distribute the Linux packages: [alpha](https://launchpad.net/~keymanapp/+archive/ubuntu/keyman-alpha) versions - [pso](http://packages.sil.org/) and [llso](http://linux.lsdev.sil.org/ubuntu/) for stable, beta, and alpha versions -- artifacts on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/view/change-requests/) +- artifacts on [GitHub](https://github.com/keymanapp/keyman/actions/workflows/deb-packaging.yml) for pull requests Packages on [llso](http://linux.lsdev.sil.org/ubuntu/) are uploaded automatically and are @@ -21,191 +21,63 @@ pso enabled. ## Package builds Package builds happen on [Launchpad](#package-builds-on-launchpad) and -[Jenkins](#package-builds-on-jenkins). Package builds for the official Ubuntu/Debian +[GitHub](#github-actions-package-builds). Package builds for the official Ubuntu/Debian repos happen outside of our control. However, we [upload source packages](#uploading-debian-source-packages) to the Debian community. -## Package builds on Jenkins +## GitHub Actions package builds ### Build jobs -The definition of the packaging jobs, the triggering of the jobs and the necessary build scripts -are scattered over several source repos: - -- [ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) contains the definition of - a meta job (multi-branch pipeline job) that gets triggered when a change gets pushed to the - [Keyman GitHub repo](https://github.com/keymanapp/keyman). The meta job creates a new build - configuration/job for each branch/pull request on GitHub. The new job gets triggered to initialize - itself, but then exits immediately. We use the Jenkins - [Job DSL plugin](https://github.com/jenkinsci/job-dsl-plugin/wiki) to define the meta job. - - ci-builder-scripts also contains several generic scripts to set up a package build environment - (using `sbuilder`) and for building source and binary packages. These scripts are shared with - other projects. - - The Keyman GitHub repo defines a webhook that triggers the meta job on Jenkins. - - Changes to ci-builder-scripts go through [Gerrit](https://gerrit.lsdev.sil.org). See - [CONTRIBUTING.md](https://github.com/sillsdev/ci-builder-scripts/blob/master/CONTRIBUTING.md) - for details. - - File structure: - - - [groovy/KeymanPackagingJobs.groovy](https://github.com/sillsdev/ci-builder-scripts/blob/master/groovy/KeymanPackagingJobs.groovy) - contains the meta job definition - - The [bash/](https://github.com/sillsdev/ci-builder-scripts/tree/master/bash) subdirectory - contains `bash` scripts: - - - [setup.sh](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) - - setup sbuild chroot environment - - [update](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) - - update the sbuild chroot environment - - [build-package](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) - - create a binary package - -- [lsdev-pipeline-library](https://github.com/sillsdev/lsdev-pipeline-library) contains a reusable - Jenkins pipeline library. The - [vars/keymanPackaging.groovy](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) - file contains the bulk of the logic of the Keyman packaging job. - -- The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various scripts that are - used to trigger a build and as part of the package build, and of course the source code for the - packages: - - - [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh) - runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) to trigger the - builds for the various platforms, among them the Jenkins package build. - - [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh) - runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?) - and increments the version number before triggering the builds for the various platforms. - - [linux/Jenkinsfile](https://github.com/keymanapp/keyman/blob/master/linux/Jenkinsfile) is a flag - for the meta job. If the meta job finds this file, it will create a new build configuration. This - file simply calls the packaging functionality defined in `lsdev-pipeline-library` and passes the - distributions and architectures to build as parameters. - - [linux/build/agent/install-deps](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps) - installs dependencies on the current build agent. - - The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) subdirectory - contains `bash` scripts that are used during the package build. Some are only needed for - Launchpad builds. - - - [jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh) - gets called from `lsdev-pipeline-library` to create a source package. - - - [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - this is the `debian` - subdirectory for Keyman for Linux with the meta data for the Linux package. - See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) for - details to the various files. +The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various +scripts that are used to trigger a build and as part of the package build, +and of course the source code for the packages: + +- [.github/workflows/deb-packaging.yml](https://github.com/keymanapp/keyman/blob/master/.github/workflows/deb-packaging.yml) + contains the definition of the packaging GHA +- [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh) + runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) + to trigger the builds for the various platforms, among them the GHA package build. +- [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh) + runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?) + and increments the version number before triggering the builds for the + various platforms. +- The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) + subdirectory contains `bash` scripts that are used during the package build. + Some are only needed for Launchpad builds. + + - [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) + gets called by the packaging GHA to install dependencies, create the source + package and to verify the API. + +- [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - + this is the `debian` subdirectory for Keyman for Linux with the meta data + for the Linux package. + See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) + for details to the various files in the `debian` directory. ### Flow of a Linux package build -- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) or - [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta) - trigger a build on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/) -- Jenkins verifies the build parameters and starts the matching build configuration for the PR or - branch -- The [build job](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) runs several checks: - - - it exits immediately if the build is not manually triggered and no parameters are passed in - (i.e. it got triggered by the GitHub webhook) - - it doesn't build if this is a PR, didn't get triggered manually and the PR is not from a trusted - user - - it doesn't build if no Linux-relevant files changed unless the parameter `force` was passed - - manually triggered builds will always build - -- build job installs - [dependencies](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps) - on the current build agent -- build job creates a source package for the linux packages (keyman, kmflcomp, - libkmfl, and ibus-kmfl). This is done by calling - [scripts/jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh). -- build job creates the binary package for each linux package on each distribution (currently - bionic, focal, and groovy) and each architecture (amd64, i386 only for bionic) -- at the end of the build if it is not a build of a PR, the `.deb` file gets uploaded to llso - (alpha packages to e.g. `bionic-experimental`, beta packages to `bionic-proposed` and - packages build from the stable branch to the main section `bionic`) +- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) + or [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta) + trigger a packaging GHA build +- packaging GHA calls [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) + which installs dependencies and creates the source package +- packaging GHA creates the binary package for each linux package on each + distribution +- packaging GHA verifies that the API didn't change with the help of + [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) +- at the end of the build if it is not a build of a PR, the `.deb` files get + uploaded to llso (alpha packages to e.g. `jammy-experimental`, beta + packages to `jammy-proposed` and packages build from the stable branch + to the main section `jammy`) - if the build is successful the job archives the artifacts -The Jenkins build progress is visible in two ways: - -- [traditional view](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/) -- [blue ocean view](https://jenkins.lsdev.sil.org/blue/organizations/jenkins/pipeline-keyman-packaging/activity) - -**Note:** TC release builds pass the git tag to build to the Jenkins job. The same tag -gets passed twice as parameters `tag` and `tag2`. The first parameter gets persisted between -builds, allowing to retrigger a tag-build. The second parameter is necessary to distinguish -if this is a retriggered build of a tag-build. - -### Local package builds - -It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who -only occasionally deals with packaging it might be easier to use the scripts that Jenkins runs: - -#### Prerequisites for local package builds - -Install `sbuild` (and probably some other packages that I forgot). - -You’ll need a chroot image before you can use sbuild. The scripts in -[ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) will help -with that. [`setup.sh`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) -can setup such chroots: - -```bash -bash/setup.sh --dists "focal bionic" --arches "amd64 i386" -``` - -[`update`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) is used to -later update those chroots: - -```bash -bash/update --dists "focal bionic" --arches "amd64 i386" -``` - -Set the `DEBSIGNKEY` environment variable to your public GPG key that will be used to sign -the packages. - -#### Building packages - -Building packages happen in the [Keyman source tree](https://github.com/keymanapp/keyman). - -The Keyman -[`linux/scripts/jenkins.sh`](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh) -script can be used to create a source package. - -```bash -cd linux -./scripts/jenkins.sh keyman ${DEBSIGNKEY} -``` - -This creates a source package (`keyman_-1.dsc`) and some `*.tar.?z` -files in the source root directory for `keyman`. - -ci-builder-script's [`build-package`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) -script creates the binary packages: - -```bash -cd $KEYMAN_ROOT -~/ci-builder-scripts/bash/build-package \ - --dists "focal bionic" --arches "amd64 i386" \ - --debkeyid ${DEBSIGNKEY} --build-in-place --no-upload -``` - -This will create the binary package `keyman_-1+1_.deb`. - -To speed up package building you might want to limit the build to a single dist -(e.g. `--dists "bionic"`) and arch (e.g. `--arches "amd64"`). - -After building packages it might be a good idea to clean up the source tree -before doing further work: - -```bash -git clean -dxf -``` - -### Local package builds (Docker) +### Local package builds with Docker It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who only occasionally deals with packaging it might be easier to use -the scripts that run on GitHub actions: +Docker and the scripts that run on GitHub actions: #### Prerequisites for local package builds with Docker diff --git a/linux/.pbuilderrc b/linux/.pbuilderrc index 1384eec4535..6a77a0ab081 100644 --- a/linux/.pbuilderrc +++ b/linux/.pbuilderrc @@ -14,7 +14,7 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC "experimental" "unstable" "testing" "stable") # List of Ubuntu suites. Update these when needed. -UBUNTU_SUITES=("lunar" "kinetic" "jammy" "focal") +UBUNTU_SUITES=("mantic" "lunar" "jammy" "focal") # Mirrors to use. Update these to your preferred mirror. DEBIAN_MIRROR="deb.debian.org" diff --git a/linux/Dockerfile b/linux/Dockerfile index d6cf7d0c63c..6852b542384 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2022 SIL International. All rights reserved. +# Copyright (c) 2022-2023 SIL International. All rights reserved. # # builder image for a linux build # see ../docs/build/linux-ubuntu.md @@ -7,27 +7,80 @@ FROM --platform=amd64 ubuntu:latest LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Linux Build Image" + # We will switch to a build user after some installation USER root -RUN useradd -c "Build user" -d $HOME -m build ENV HOME /home/build -VOLUME /home/build -WORKDIR /home/build +RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build +VOLUME /home/build/build +WORKDIR /home/build/build ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_PRIORITY critical ENV DEBCONF_NOWARNINGS yes # Update to the latest RUN apt-get -q -y update && \ - apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common && \ + apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common curl && \ add-apt-repository ppa:keymanapp/keyman && \ - add-apt-repository ppa:keymanapp/keyman-alpha && \ + add-apt-repository ppa:keymanapp/keyman-alpha +RUN apt-get -q -y update && \ apt-get -q -y upgrade +# Install dependencies ADD debian/control /tmp/control # Answer 'yes' to install questions -RUN (yes | mk-build-deps --install /tmp/control) || true -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -RUN apt-get -q -y install nodejs +RUN (yes | mk-build-deps --install /tmp/control) || true && \ + rm /tmp/control + +# Install Node +RUN curl -sL https://deb.nodesource.com/setup_18.x | bash && \ + apt-get -q -y install nodejs + +# Install emscripten +RUN cd /usr/share && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install latest && \ + ./emsdk activate latest && \ + echo "#!/bin/bash" > /usr/bin/bashwrapper && \ + echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper + +# Keyman Web +RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get -q -y install ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb && \ + echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper + +# Keyman for Android +RUN apt-get -q -y install gradle maven pandoc sdkmanager jq && \ + sdkmanager platform-tools && \ + yes | sdkmanager --licenses && \ + chown -R build:build /opt/android-sdk/ && \ + echo "export ANDROID_HOME=/opt/android-sdk" >> /usr/bin/bashwrapper && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> /usr/bin/bashwrapper + +# Finish bashwrapper script and adjust permissions +RUN echo "\${@:-bash}" >> /usr/bin/bashwrapper && \ + chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + # now, switch to build user USER build + +# Pre-install gradle. This will put files in ~/.gradle which will speed up builds. +RUN mkdir -p $HOME/tmp/gradle/wrapper && \ + # KMEA uses gradle-7.5.1-bin + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + # Some projects use gradle-7.5.1-all, so we pre-install that as well + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + rm -rf $HOME/tmp + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/linux/Jenkinsfile b/linux/Jenkinsfile deleted file mode 100644 index b70cb95d706..00000000000 --- a/linux/Jenkinsfile +++ /dev/null @@ -1,11 +0,0 @@ -#!groovy -// Copyright (c) 2019-2022 SIL International -// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) - -@Library('lsdev-pipeline-library') _ - -keymanPackaging { - distributionsToPackage = 'focal jammy kinetic lunar' - arches = 'amd64 i386' - packagesToBuild = ['keyman'] -} diff --git a/linux/README.md b/linux/README.md index 3dfb3404779..e4b50cdc905 100644 --- a/linux/README.md +++ b/linux/README.md @@ -1,3 +1,5 @@ # Keyman for Linux See [/docs/linux/README.md](../docs/linux/README.md) for documentation. + +. \ No newline at end of file diff --git a/linux/debian/changelog b/linux/debian/changelog index 9f69cc9a12f..1ad0f861fb3 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,18 @@ +keyman (16.0.141-1) unstable; urgency=medium + + * Work around mips64el build failure (#1041499) + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Thu, 27 Jul 2023 16:30:04 +0200 + +keyman (16.0.140-1) unstable; urgency=medium + + * New upstream release (closes: #1037707). + * Re-release to Debian + + -- Eberhard Beilharz Mon, 24 Jul 2023 11:41:07 +0200 + keyman (16.0.139-4) unstable; urgency=medium * debian/tests: Revert previous change and ignore s390x from autopkgtests diff --git a/linux/keyman-system-service/README.md b/linux/keyman-system-service/README.md index 358004af8fc..8b6ca2eba69 100644 --- a/linux/keyman-system-service/README.md +++ b/linux/keyman-system-service/README.md @@ -1,8 +1,8 @@ # keyman-system-service -A DBus system service that allows to access /dev/input/* devices -to toggle capslock and perform other keyboard related actions when -running under Wayland. +A DBus system service that allows to access `/dev/input/*` devices +to toggle capslock and perform other keyboard related actions. This is +required when running under Wayland, but also used with X11. See , , diff --git a/linux/scripts/cow.sh b/linux/scripts/cow.sh index e175a666ef6..201896a7e7c 100755 --- a/linux/scripts/cow.sh +++ b/linux/scripts/cow.sh @@ -3,7 +3,7 @@ # If needed set cowbuilder up for building Keyman Debian packages # Then cowbuilder update -distributions='focal jammy kinetic lunar' +distributions='focal jammy lunar mantic' if ! dpkg-query -l cowbuilder; then echo "installing pbuilder and cowbuilder" diff --git a/linux/scripts/deb.sh b/linux/scripts/deb.sh index 3c87af0de8c..35a476a55cb 100755 --- a/linux/scripts/deb.sh +++ b/linux/scripts/deb.sh @@ -11,7 +11,7 @@ set -e -all_distributions="focal jammy" +all_distributions="focal jammy lunar mantic" distributions="" echo "all_distributions: ${all_distributions}" diff --git a/linux/scripts/dist.sh b/linux/scripts/dist.sh index f8d63b63e35..d19abf7f782 100755 --- a/linux/scripts/dist.sh +++ b/linux/scripts/dist.sh @@ -49,7 +49,7 @@ dpkg-source --tar-ignore=*~ --tar-ignore=.git --tar-ignore=.gitattributes \ --tar-ignore=core/build \ --tar-ignore=developer --tar-ignore=docs --tar-ignore=ios \ --tar-ignore=linux/keyman-config/buildtools/build-langtags.py --tar-ignore=__pycache__ \ - --tar-ignore=linux/help --tar-ignore=linux/Jenkinsfile \ + --tar-ignore=linux/help \ --tar-ignore=mac --tar-ignore=node_modules --tar-ignore=oem \ --tar-ignore=linux/build \ --tar-ignore=linux/builddebs \ diff --git a/linux/scripts/jenkins.sh b/linux/scripts/jenkins.sh deleted file mode 100755 index 5c796259559..00000000000 --- a/linux/scripts/jenkins.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# $1 - project name with appended tier, e.g. keyman-alpha -# $2 - GPG key used for signing the source package - -set -e -set -u - -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" - -. "$THIS_SCRIPT_PATH/package-build.inc.sh" - -keyman_projects="keyman" - -tier="stable" - -if [[ "$1" =~ "-alpha" ]]; then - tier="alpha" -elif [[ "$1" =~ "-beta" ]]; then - tier="beta" -fi - -proj="$1" -proj=${proj%"-alpha"} -proj=${proj%"-beta"} - -fullsourcename="keyman" -sourcedir="$KEYMAN_ROOT" -sourcename=${fullsourcename%"-alpha"} -sourcename=${sourcename%"-beta"} - -# set Debian/changelog environment -export DEBFULLNAME="${fullsourcename} Package Signing Key" -export DEBEMAIL='jenkins@sil.org' - -checkAndInstallRequirements - -# clean up prev deb builds -builder_heading "cleaning previous builds of $1" - -rm -rf builddebs -rm -rf "$sourcedir/${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log} -rm -rf "$sourcedir/../${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log} - -builder_heading "Make source package for $fullsourcename" -builder_heading "reconfigure" -TIER="$tier" ./scripts/reconf.sh - -builder_heading "Make origdist" -./scripts/dist.sh origdist -builder_heading "Make deb source" -./scripts/deb.sh sourcepackage - -#sign source package -for file in builddebs/*.dsc; do - builder_heading "Signing source package $file" - debsign -k"$2" "$file" -done - -mv builddebs/* .. diff --git a/linux/scripts/launchpad.sh b/linux/scripts/launchpad.sh index 30495d2b0e1..7437272c030 100755 --- a/linux/scripts/launchpad.sh +++ b/linux/scripts/launchpad.sh @@ -33,7 +33,7 @@ else fi echo "ppa: ${ppa}" -distributions="${DIST:-focal jammy kinetic lunar}" +distributions="${DIST:-focal jammy lunar mantic}" packageversion="${PACKAGEVERSION:-1~sil1}" BASEDIR=$(pwd) diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index 6c249891c16..fcfe492b433 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -49,6 +49,14 @@ function downloadSource() { sha256sum -c --ignore-missing SHA256SUMS |grep "${proj}" } +function wait_for_apt_deb { + # from https://gist.github.com/hrpatel/117419dcc3a75e46f79a9f1dce99ef52 + while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock &>/dev/null 2>&1; do + echo "Waiting for apt/dpkg lock to release, sleeping 10s" + sleep 10 + done +} + function checkAndInstallRequirements() { local TOINSTALL="" @@ -63,12 +71,12 @@ function checkAndInstallRequirements() export DEBIAN_FRONTEND=noninteractive if [ -n "$TOINSTALL" ]; then - sudo apt-get update + wait_for_apt_deb && sudo apt-get update # shellcheck disable=SC2086 - sudo apt-get -qy install $TOINSTALL + wait_for_apt_deb && sudo apt-get -qy install $TOINSTALL fi sudo mk-build-deps debian/control - sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb + wait_for_apt_deb && sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb sudo rm -f keyman-buid-deps_* } diff --git a/linux/scripts/upload-to-debian.sh b/linux/scripts/upload-to-debian.sh index af443cac08a..d7665f29f8f 100755 --- a/linux/scripts/upload-to-debian.sh +++ b/linux/scripts/upload-to-debian.sh @@ -105,7 +105,7 @@ git add debian/changelog git commit -m "chore(linux): Update debian changelog" if [ -n "$PUSH" ]; then $NOOP git push --force-with-lease origin chore/linux/changelog - $NOOP gh pr create --draft --base "$stable_branch" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip" + $NOOP gh pr create --draft --base "${stable_branch#origin/}" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip" fi if $ISBETA; then @@ -118,7 +118,7 @@ git checkout -B chore/linux/cherry-pick/changelog ${CLBRANCH} git cherry-pick -x chore/linux/changelog if [ -n "$PUSH" ]; then $NOOP git push --force-with-lease origin chore/linux/cherry-pick/changelog - $NOOP gh pr create --draft --base ${CLBRANCH} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip" + $NOOP gh pr create --draft --base ${CLBRANCH#origin/} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip" fi builder_heading "Finishing" diff --git a/package-lock.json b/package-lock.json index 89ca9b035dc..f22a434ee38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10686,9 +10686,9 @@ } }, "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "dev": true, "engines": { "node": ">=0.10.0" diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 7ae8db2ae1e..cdb19442461 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -74,24 +74,31 @@ function findVersion() { VERSION_TAG= fi - if [ -z "${TEAMCITY_VERSION-}" -a -z "${JENKINS_HOME-}" ]; then - # Local dev machine, not TeamCity + if [ -z "${TEAMCITY_VERSION-}" ] && [ -z "${GITHUB_ACTIONS-}" ]; then + # Local dev machine, not TeamCity or GitHub Action VERSION_TAG="$VERSION_TAG-local" VERSION_ENVIRONMENT=local - else + elif [ -n "${TEAMCITY_PR_NUMBER-}" ]; then # On TeamCity: are we running a pull request build or a master/beta/stable build? - if [ ! -z "${TEAMCITY_PR_NUMBER-}" ]; then - VERSION_ENVIRONMENT=test - # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y' - # This indicates we are running a Test build. - if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then - VERSION_TAG="$VERSION_TAG-test" - else - VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER" - fi + VERSION_ENVIRONMENT="test" + # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y' + # This indicates we are running a Test build. + if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then + VERSION_TAG="$VERSION_TAG-test" else - VERSION_ENVIRONMENT="$TIER" + VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER" fi + elif [ -n "${GITHUB_ACTIONS-}" ] && ${GHA_TEST_BUILD-}; then + VERSION_ENVIRONMENT="test" + # Note GHA_BRANCH can be 'master', 'beta', or 'stable-x.y' + # This indicates we are running a Test build. + if [[ ${GHA_BRANCH-} =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then + VERSION_TAG="${VERSION_TAG}-test" + else + VERSION_TAG="${VERSION_TAG}-test-${GHA_BRANCH-unset}" + fi + else + VERSION_ENVIRONMENT="$TIER" fi VERSION_WITH_TAG="$VERSION$VERSION_TAG" diff --git a/resources/build/increment-version.sh b/resources/build/increment-version.sh index d54990194c6..445459da161 100755 --- a/resources/build/increment-version.sh +++ b/resources/build/increment-version.sh @@ -143,7 +143,7 @@ if [ "$action" == "commit" ]; then popd > /dev/null # - # Trigger builds for the previous version on TeamCity, Jenkins and GitHub + # Trigger builds for the previous version on TeamCity and GitHub # triggerBuilds diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh index 218383d1597..300bd831567 100755 --- a/resources/build/run-required-test-builds.sh +++ b/resources/build/run-required-test-builds.sh @@ -54,11 +54,7 @@ function triggerTestBuilds() { eval test_builds='(${'bc_test_$platform'[@]})' for test_build in "${test_builds[@]}"; do if [[ $test_build == "" ]]; then continue; fi - if [ "${test_build:(-8)}" == "_Jenkins" ]; then - local job=${test_build%_Jenkins} - echo " -- Triggering build configuration $job/$branch on Jenkins" - triggerJenkinsBuild "$job" "$branch" "$force" - elif [ "${test_build:(-7)}" == "_GitHub" ]; then + if [ "${test_build:(-7)}" == "_GitHub" ]; then local job=${test_build%_GitHub} echo " -- Triggering GitHub action build $job/$branch" triggerGitHubActionsBuild true "$job" "$branch" diff --git a/resources/build/trigger-builds.inc.sh b/resources/build/trigger-builds.inc.sh index ac8d519d4a8..8d6297b9e1f 100644 --- a/resources/build/trigger-builds.inc.sh +++ b/resources/build/trigger-builds.inc.sh @@ -17,11 +17,7 @@ function triggerBuilds() { eval builds='(${'bc_${bcbase}_${platform}'[@]})' for build in "${builds[@]}"; do if [[ $build == "" ]]; then continue; fi - if [ "${build:(-8)}" == "_Jenkins" ]; then - local job=${build%_Jenkins} - echo Triggering Jenkins build "$job" "$base" "true" - triggerJenkinsBuild "$job" "$base" "true" - elif [ "${build:(-7)}" == "_GitHub" ]; then + if [ "${build:(-7)}" == "_GitHub" ]; then local job=${build%_GitHub} echo Triggering GitHub action build "$job" "$base" triggerGitHubActionsBuild false "$job" "$base" @@ -69,62 +65,6 @@ function triggerTeamCityBuild() { -d "$command" } -function triggerJenkinsBuild() { - local JENKINS_JOB="$1" - local JENKINS_BRANCH="${2:-master}" - - local JENKINS_SERVER=https://jenkins.lsdev.sil.org - - local FORCE="" - if [ "${3:-false}" == "true" ]; then - FORCE=", \"force\": true" - fi - - local TAG="" - # This will only be true if we created and pushed a tag - if [ "${action:-""}" == "commit" ]; then - TAG=", \"tag\": \"$VERSION_GIT_TAG\", \"tag2\": \"$VERSION_GIT_TAG\"" - fi - - if [[ $JENKINS_BRANCH != stable-* ]] && [[ $JENKINS_BRANCH =~ [0-9]+ ]]; then - JENKINS_BRANCH="PR-${JENKINS_BRANCH}" - fi - - local OUTPUT=$(curl --silent --write-out '\n' \ - -X POST \ - --header "token: $JENKINS_TOKEN" \ - --header "Content-Type: application/json" \ - $JENKINS_SERVER/generic-webhook-trigger/invoke \ - --data "{ \"project\": \"$JENKINS_JOB/$JENKINS_BRANCH\", \"branch\": \"$JENKINS_BRANCH\" $TAG $FORCE }") - - if echo "$OUTPUT" | grep -q "\"triggered\":true"; then - echo -n " job triggered: " - else - echo "##teamcity[buildProblem description='Triggering Jenkins build failed']" - echo -n " triggering failed: " - fi - - # Strip {"jobs":{ from the beginning of OUTPUT - OUTPUT=${OUTPUT#\{\"jobs\":\{} - # Split json string to lines with one job each - local jobs count - count=0 - IFS='|' jobs=(${OUTPUT//\},\"pipeline/\},|\"pipeline}) - # Find job that actually got triggered (or that we should have triggered) - for line in "${jobs[@]}"; do - if [[ $line == \"$JENKINS_JOB/$JENKINS_BRANCH* ]]; then - echo "$line" - count=$((++count)) - fi - done - if [[ $count < 1 ]]; then - # DEBUG - echo -n $OUTPUT - - echo - fi -} - function triggerGitHubActionsBuild() { local IS_TEST_BUILD="$1" local GITHUB_ACTION="$2" diff --git a/resources/build/trigger-definitions.inc.sh b/resources/build/trigger-definitions.inc.sh index 9370dadd825..4698719d4d7 100644 --- a/resources/build/trigger-definitions.inc.sh +++ b/resources/build/trigger-definitions.inc.sh @@ -34,9 +34,6 @@ watch_common_linux='common/linux|common/web' # These bc_x_y variables ARE used in trigger-builds.inc.sh by pattern so the names are important, # and you won't find them directly in a grep search. # -# _Jenkins should be appended to any build configuration (pipeline) name that is from Jenkins, -# not TeamCity. -# # _GitHub should be appended to any build configuration name that is from GitHub, not TeamCity. # Test Build Configurations @@ -46,7 +43,7 @@ bc_test_all=() bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects) bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects) -bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux pipeline-keyman-packaging_Jenkins deb-pr-packaging_GitHub) +bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub) bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS) bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows) bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_LMLayer_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM) @@ -66,7 +63,7 @@ vcs_test=HttpsGithubComKeymanappKeymanPRs bc_master_android=(KeymanAndroid_Build) bc_master_ios=(Keyman_iOS_Master) -bc_master_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_master_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_master_mac=(KeymanMac_Master) bc_master_windows=(Keyman_Build) bc_master_web=(Keymanweb_Build) @@ -78,7 +75,7 @@ vcs_master=HttpsGithubComKeymanappKeyman bc_beta_android=(KeymanAndroid_Build) bc_beta_ios=(Keyman_iOS_Master) -bc_beta_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_beta_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_beta_mac=(KeymanMac_Master) bc_beta_windows=(Keyman_Build) bc_beta_web=(Keymanweb_Build) @@ -90,7 +87,7 @@ vcs_beta=HttpsGithubComKeymanappKeyman bc_stable_14_0_android=(KeymanAndroid_Build) bc_stable_14_0_ios=(Keyman_iOS_Master) -bc_stable_14_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_stable_14_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_stable_14_0_mac=(KeymanMac_Master) bc_stable_14_0_windows=(Keyman_Build) bc_stable_14_0_web=(Keymanweb_Build) @@ -105,7 +102,7 @@ vcs_stable_14_0=HttpsGithubComKeymanappKeyman bc_stable_15_0_android=(KeymanAndroid_Build) bc_stable_15_0_ios=(Keyman_iOS_Master) -bc_stable_15_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_stable_15_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_stable_15_0_mac=(KeymanMac_Master) bc_stable_15_0_windows=(Keyman_Build) bc_stable_15_0_web=(Keymanweb_Build) @@ -116,7 +113,7 @@ vcs_stable_15_0=HttpsGithubComKeymanappKeyman bc_stable_16_0_android=(KeymanAndroid_Build) bc_stable_16_0_ios=(Keyman_iOS_Master) -bc_stable_16_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins) +bc_stable_16_0_linux=(KeymanLinux_Master) bc_stable_16_0_mac=(KeymanMac_Master) bc_stable_16_0_windows=(Keyman_Build) bc_stable_16_0_web=(Keymanweb_Build) diff --git a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml index 7a1916d1df9..156a195c058 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml +++ b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml @@ -63,12 +63,9 @@ - diff --git a/web/src/app/browser/src/contextManager.ts b/web/src/app/browser/src/contextManager.ts index 42e7edd729f..492c627146d 100644 --- a/web/src/app/browser/src/contextManager.ts +++ b/web/src/app/browser/src/contextManager.ts @@ -211,6 +211,12 @@ export default class ContextManager extends ContextManagerBasespacebar. - [Keyman Configuration - Keyboard Layouts Tab](config/keyboards) - [How To - Download and Install a Keyman Keyboard](../start/download-and-install-keyboard) - [Keyboard Task - Enable or Disable a Keyboard](enable-or-disable-keyboard) -- [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](../troubleshooting/hidden) diff --git a/windows/src/desktop/help/basic/uninstall.md b/windows/src/desktop/help/basic/uninstall.md index d9952a4279f..7e7cb2274c7 100644 --- a/windows/src/desktop/help/basic/uninstall.md +++ b/windows/src/desktop/help/basic/uninstall.md @@ -15,18 +15,3 @@ title: Software Task - Uninstall Keyman 5. Click Uninstall. 6. Follow the prompts to complete the uninstall. - -## Uninstall Keyman from Windows 7 or 8 - -1. Exit Keyman. - -2. Open Windows Control Panel. - -3. Select \'Add or Remove Programs\' or \'Programs and Features\' or - \'Uninstall a program\'. - -4. Click Keyman in the list. - -5. Click Remove or Uninstall or right-click \'Uninstall\'. - -6. Follow the prompts to complete the uninstall. diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md index ca1ead7b271..32bf778813e 100644 --- a/windows/src/desktop/help/common/os.md +++ b/windows/src/desktop/help/common/os.md @@ -2,26 +2,16 @@ title: What operating systems does Keyman support? --- -Keyman fully supports 32-bit *and* 64-bit versions of the following +Keyman for Windows fully supports the following versions of the Windows operating systems: -- Windows Server 2008 - -- Windows 7 - -- Windows 8 - -- Windows 8.1 - -- Windows 10 +- Windows 10 (32-bit and 64-bit editions) - Windows 11 -- Windows Server 2012 and 2012 R2 +- Windows Server 2019 and 2022 + +Keyman also runs on Android, iOS, Linux, macOS, and in websites. Visit https://keyman.com for more downloads. -**Note:** -Keyman works slightly differently in different versions of Windows. -Older versions of Windows have more language limitations and need extra -configuration. Windows Vista and later versions of Windows include added -security measures and some configuration actions will require you to -confirm security prompts. +* **Note:** Keyman does not yet support Windows Insider Preview ARM64. +* Keyman works slightly differently in different versions of Windows. diff --git a/windows/src/desktop/help/common/requirements.md b/windows/src/desktop/help/common/requirements.md index cdd3d81c234..11fefc6dfdf 100644 --- a/windows/src/desktop/help/common/requirements.md +++ b/windows/src/desktop/help/common/requirements.md @@ -3,4 +3,4 @@ title: What are Keyman's hardware requirements? --- Keyman has minimal resource requirements. Any computer that can run -Windows 7 or later should be able to run Keyman without trouble. +Windows 10 or later should be able to run Keyman without trouble. diff --git a/windows/src/desktop/help/troubleshooting/hidden.md b/windows/src/desktop/help/troubleshooting/hidden.md deleted file mode 100644 index 11d35a151fa..00000000000 --- a/windows/src/desktop/help/troubleshooting/hidden.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: How To - Fix A Problem with an Active Keyman Keyboard Not Typing ---- - -## Symptoms -You may experience an issue in Windows 7 where: -* Keyman is on; and -* your Keyman keyboard is active, your Keyman keyboard is active, with its icon showing in the Keyman menu; but -* your Keyman keyboard does not work. - -## Resolution -Unhide the Keyman menu as described in [Step 2](../start/tutorial#step-2-) of the Getting Started tutorial. - -This can also be resolved by switching language using the Windows Language Bar or the Language Icon - -## Background -When the Keyman icon is in the hidden notification area of the Windows taskbar, opening the notification area and selecting the icon causes it to change the keyboard for the taskbar, not for your active application. Clicking back to the active application results in the keyboard turning off again. - -## Related Topics -- [Keyboard Task - Turn on a Keyboard](../basic/select-keyboard) diff --git a/windows/src/desktop/help/troubleshooting/index.md b/windows/src/desktop/help/troubleshooting/index.md index 03e9745fec9..54416ad9ba0 100644 --- a/windows/src/desktop/help/troubleshooting/index.md +++ b/windows/src/desktop/help/troubleshooting/index.md @@ -10,6 +10,4 @@ title: Troubleshooting * [How To - Resolve Security Software Conflicts with keyman32.dll](securitysoftware) * [How To - Use the Keyman Setup Bootstrapper](bootstrapper) -* [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](hidden) - * [How To - Allow Windows 10 to Install Apps From Anywhere](install-app-from-anywhere)