Merge branch 'master' into mbien/upgrade-garden-forms-package

README.md

@@ -178,11 +178,11 @@ function MyComponent() {
 Providing the default English value in the code makes it possible to use it as a fallback value when strings are not yet translated and to extract the strings from the source code to the translations YAML file.
 
 #### Plurals 
-When using [plurals](https://www.i18next.com/translation-function/plurals), we need to provide default values for the `zero`, `one`, and `other` values, as requested by our translation system. This can be done by passing the default values in the [options](https://www.i18next.com/translation-function/essentials#overview-options) of the `t` function.
+When using [plurals](https://www.i18next.com/translation-function/plurals), you need to provide default values for at least the `one` and `other` forms. You can also pass a default value for the `zero` form, if you want it to be different from the `other` form. This can be done by passing the default values in the [options](https://www.i18next.com/translation-function/essentials#overview-options) of the `t` function.
 
 ```ts
 t("my-key", {
-  "defaultValue.zero": "{{count}} items",
+  "defaultValue.zero": "No items",
   "defaultValue.one": "{{count}} item",
   "defaultValue.other": "{{count}} items",
   count: ...
@@ -191,23 +191,42 @@ t("my-key", {
 
 #### String extraction
 
-The `bin/extract-strings.mjs` script can be used to extract translation strings from the source code and put them in the YAML file that is picked up by our internal translation system. The usage of the script is documented in the script itself.
+The `bin/extract-strings.mjs` script can be used to extract translation strings from the source code and put them in the YAML file that is picked up by our internal translation system.
 
-The script wraps the `i18next-parser` tool and converts its output to the YAML format used internally. It is possible to use a similar approach in a custom theme, either using the standard `i18next-parser` output as the source for translations or implementing a custom transformer.
+To extract the strings for all the modules, run:
 
-#### Updating translation files
+```bash
+yarn i18n:extract
+```
+
+To extract the strings for a specific module, run:
 
-Use the `bin/update-modules-translations.mjs` to download the latest translations for all the modules. All files are then bundled by the build process in a single `[MODULE]-translations-bundle.js` file.
+```bash
+yarn i18n:extract --module=module-name
+```
 
-The first time that translations are added to a module, you need to add a mapping between the module folder and the package name on the translations systems to the `MODULE` variable in the script. For example, if a module is located in `src/modules/my-module` and the package name is `cph-theme-my-module`, you need to add:
+You can also pass the `--mark-obsolete` flag to mark removed strings as obsolete:
 
-```js
-const MODULES = {
-  ...,
-  "my-module": "cph-theme-my-module"
-}
+```bash
+yarn i18n:extract --mark-obsolete
+```
+
+If you are adding a new module, you need first to create the initial `src/modules/[MODULE]/translations/en-us.yml` file with the header containing the title and package name:
+
+```yaml
+title: "My Module"
+packages:
+  - "package-name"
 ```
 
+The script wraps the `i18next-parser` tool and converts its output to the YAML format used internally. It is possible to use a similar approach in a custom theme, either using the standard `i18next-parser` output as the source for translations or implementing a custom transformer.
+
+#### Updating translation files
+
+You can run `yarn i18n:update-translations` to download the latest translations files for all the modules. The script downloads all the locale files for each module, fetching the package name from the `src/modules/[MODULE]/translations/en-us.yml` file, and saves them in the `src/modules/[MODULE]/translations/locales` folder.
+
+All files are then bundled by the build process in a single `[MODULE]-translations-bundle.js` file.
+
 # Accessibility testing
 
 We use a custom node script that runs [lighthouse](https://github.com/GoogleChrome/lighthouse) for automated accessibility testing.

bin/extract-strings.mjs

@@ -13,28 +13,61 @@
  *       value: "My value"
  * ```
  *
- * If a string present in the source YAML file is not found in the source code, it will be marked as obsolete
+ * If a string present in the source YAML file is not found in the source code, it will be marked as obsolete if the
+ * `--mark-obsolete` flag is passed.
  *
  * If the value in the YAML file differs from the value in the source code, a warning will be printed in the console,
  * since the script cannot know which one is correct and cannot write back in the source code files. This can happen for
  * example after a "reverse string sweep", and can be eventually fixed manually.
  *
  * The script uses the i18next-parser library for extracting the strings and it adds a custom transformer for creating
  * the file in the required YAML format.
+ *
+ * For usage instructions, run `node extract-strings.mjs --help`
  */
-// Usage:   node bin/extract-strings.mjs [PACKAGE_NAME] [SOURCE_FILES_GLOB] [OUTPUT_PATH]
-// Example: node bin/extract-strings.mjs new-request-form 'src/modules/new-request-form/**/*.{ts,tsx}' src/modules/new-request-form/translations
 import vfs from "vinyl-fs";
 import Vinyl from "vinyl";
 import { transform as I18NextTransform } from "i18next-parser";
 import { Transform } from "node:stream";
-import { existsSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { load, dump } from "js-yaml";
 import { resolve } from "node:path";
+import { glob } from "glob";
+import { parseArgs } from "node:util";
+
+const { values: args } = parseArgs({
+  options: {
+    "mark-obsolete": {
+      type: "boolean",
+    },
+    module: {
+      type: "string",
+    },
+    help: {
+      type: "boolean",
+    },
+  },
+});
+
+if (args.help) {
+  const helpMessage = `
+  Usage: extract-strings.mjs [options]
+
+  Options:
+    --mark-obsolete   Mark removed strings as obsolete in the source YAML file
+    --module          Extract strings only for the specified module. The module name should match the folder name in the src/modules folder
+                      If not specified, the script will extract strings for all modules
+    --help            Display this help message
+
+  Examples:
+    node extract-strings.mjs
+    node extract-strings.mjs --mark-obsolete
+    node extract-strings.mjs --module=ticket-fields
+  `;
 
-const PACKAGE_NAME = process.argv[2];
-const INPUT_GLOB = `${process.cwd()}/${process.argv[3]}`;
-const OUTPUT_DIR = resolve(process.cwd(), process.argv[4]);
+  console.log(helpMessage);
+  process.exit(0);
+}
 
 const OUTPUT_YML_FILE_NAME = "en-us.yml";
 
@@ -50,10 +83,10 @@ const OUTPUT_BANNER = `#
 
 /** @type {import("i18next-parser").UserConfig} */
 const config = {
-  // Our translation system requires that we add ".zero", ".one", ".other" keys for plurals
+  // Our translation system requires that we add all 6 forms (zero, one, two, few, many, other) keys for plurals
   // i18next-parser extracts plural keys based on the target locale, so we are passing a
-  // locale that need exactly the ".zero", ".one", ".other" keys, even if we are extracting English strings
-  locales: ["lv"],
+  // locale that need exactly the 6 forms, even if we are extracting English strings
+  locales: ["ar"],
   keySeparator: false,
   namespaceSeparator: false,
   pluralSeparator: ".",
@@ -62,37 +95,46 @@ const config = {
 };
 
 class SourceYmlTransform extends Transform {
-  #defaultContent = {
-    title: "",
-    packages: [PACKAGE_NAME],
-    parts: [],
-  };
+  #parsedInitialContent;
+  #outputDir;
 
   #counters = {
     added: 0,
     obsolete: 0,
     mismatch: 0,
   };
 
-  constructor() {
+  constructor(outputDir) {
     super({ objectMode: true });
+
+    this.#outputDir = outputDir;
+    this.#parsedInitialContent = this.#getSourceYmlContent();
   }
 
   _transform(file, encoding, done) {
     try {
       const strings = JSON.parse(file.contents.toString(encoding));
-      const outputContent = this.#getSourceYmlContent();
 
-      // Mark removed keys as obsolete
+      const outputContent = {
+        ...this.#parsedInitialContent,
+        parts: this.#parsedInitialContent.parts || [],
+      };
+
+      // Find obsolete keys
       for (const { translation } of outputContent.parts) {
         if (!(translation.key in strings) && !translation.obsolete) {
-          translation.obsolete = this.#getObsoleteDate();
           this.#counters.obsolete++;
+
+          if (args["mark-obsolete"]) {
+            translation.obsolete = this.#getObsoleteDate();
+          }
         }
       }
 
       // Add new keys to the source YAML or log mismatched value
-      for (const [key, value] of Object.entries(strings)) {
+      for (let [key, value] of Object.entries(strings)) {
+        value = this.#fixPluralValue(key, value, strings);
+
         const existingPart = outputContent.parts.find(
           (part) => part.translation.key === key
         );
@@ -124,12 +166,7 @@ class SourceYmlTransform extends Transform {
         ),
       });
       this.push(virtualFile);
-
-      console.log(`String extraction completed!
-      Added strings: ${this.#counters.added}
-      Removed strings (marked as obsolete): ${this.#counters.obsolete}
-      Strings with mismatched value: ${this.#counters.mismatch}`);
-
+      this.#printInfo();
       done();
     } catch (e) {
       console.error(e);
@@ -138,23 +175,82 @@ class SourceYmlTransform extends Transform {
   }
 
   #getSourceYmlContent() {
-    const outputPath = resolve(OUTPUT_DIR, OUTPUT_YML_FILE_NAME);
-    if (existsSync(outputPath)) {
-      return load(readFileSync(outputPath, "utf-8"));
-    }
-
-    return this.#defaultContent;
+    const outputPath = resolve(this.#outputDir, OUTPUT_YML_FILE_NAME);
+    return load(readFileSync(outputPath, "utf-8"));
   }
 
   #getObsoleteDate() {
     const today = new Date();
     const obsolete = new Date(today.setMonth(today.getMonth() + 3));
     return obsolete.toISOString().split("T")[0];
   }
+
+  #printInfo() {
+    const message = `Package ${this.#parsedInitialContent.packages[0]}
+    Added strings: ${this.#counters.added}
+    ${this.#getObsoleteInfoMessage()}
+    Strings with mismatched value: ${this.#counters.mismatch}
+    `;
+
+    console.log(message);
+  }
+
+  #getObsoleteInfoMessage() {
+    if (args["mark-obsolete"]) {
+      return `Removed strings (marked as obsolete): ${this.#counters.obsolete}`;
+    }
+
+    let result = `Obsolete strings: ${this.#counters.obsolete}`;
+    if (this.#counters.obsolete > 0) {
+      result += " - Use --mark-obsolete to mark them as obsolete";
+    }
+
+    return result;
+  }
+
+  // if the key ends with .zero, .one, .two, .few, .many, .other and the value is empty
+  // find the same key with the `.other` suffix in strings and return the value
+  #fixPluralValue(key, value, strings) {
+    if (key.endsWith(".zero") && value === "") {
+      return strings[key.replace(".zero", ".other")] || "";
+    }
+
+    if (key.endsWith(".one") && value === "") {
+      return strings[key.replace(".one", ".other")] || "";
+    }
+
+    if (key.endsWith(".two") && value === "") {
+      return strings[key.replace(".two", ".other")] || "";
+    }
+
+    if (key.endsWith(".few") && value === "") {
+      return strings[key.replace(".few", ".other")] || "";
+    }
+
+    if (key.endsWith(".many") && value === "") {
+      return strings[key.replace(".many", ".other")] || "";
+    }
+
+    return value;
+  }
 }
 
-vfs
-  .src([INPUT_GLOB])
-  .pipe(new I18NextTransform(config))
-  .pipe(new SourceYmlTransform())
-  .pipe(vfs.dest(OUTPUT_DIR));
+const sourceFilesGlob = args.module
+  ? `src/modules/${args.module}/translations/en-us.yml`
+  : "src/modules/**/translations/en-us.yml";
+
+const sourceFiles = await glob(sourceFilesGlob);
+for (const sourceFile of sourceFiles) {
+  const moduleName = sourceFile.split("/")[2];
+  const inputGlob = `src/modules/${moduleName}/**/*.{ts,tsx}`;
+  const outputDir = resolve(
+    process.cwd(),
+    `src/modules/${moduleName}/translations`
+  );
+
+  vfs
+    .src([inputGlob])
+    .pipe(new I18NextTransform(config))
+    .pipe(new SourceYmlTransform(outputDir))
+    .pipe(vfs.dest(outputDir));
+}

bin/update-modules-translations.mjs

@@ -3,15 +3,10 @@
  * This script is used for downloading the latest Zendesk official translation files for the modules in the `src/module` folder.
  *
  */
-import { writeFile } from "node:fs/promises";
+import { writeFile, readFile, mkdir } from "node:fs/promises";
 import { resolve } from "node:path";
-
-/**
- *  Maps each folder in the `src/modules` directory with its package name on the translation system
- */
-const MODULES = {
-  "new-request-form": "new-request-form",
-};
+import { glob } from "glob";
+import { load } from "js-yaml";
 
 const BASE_URL = `https://static.zdassets.com/translations`;
 
@@ -38,6 +33,8 @@ async function fetchModuleTranslations(moduleName, packageName) {
       `src/modules/${moduleName}/translations/locales`
     );
 
+    await mkdir(outputDir, { recursive: true });
+
     const { json } = await manifestResponse.json();
 
     await Promise.all(
@@ -50,6 +47,23 @@ async function fetchModuleTranslations(moduleName, packageName) {
   }
 }
 
-for (const [moduleName, packageName] of Object.entries(MODULES)) {
+// search for `src/modules/**/translations/en-us.yml` files, read it contents and return a map of module names and package names
+async function getModules() {
+  const result = {};
+  const files = await glob("src/modules/**/translations/en-us.yml");
+
+  for (const file of files) {
+    const content = await readFile(file);
+    const parsedContent = load(content);
+    const moduleName = file.split("/")[2];
+    result[moduleName] = parsedContent.packages[0];
+  }
+
+  return result;
+}
+
+const modules = await getModules();
+
+for (const [moduleName, packageName] of Object.entries(modules)) {
   await fetchModuleTranslations(moduleName, packageName);
 }

package.json

@@ -11,7 +11,7 @@
     "download-locales": "node ./bin/update-translations",
     "test": "jest",
     "test-a11y": "node bin/lighthouse/index.js",
-    "i18n:extract": "node bin/extract-strings.mjs new-request-form 'src/modules/new-request-form/**/*.{ts,tsx}' src/modules/new-request-form/translations",
+    "i18n:extract": "node bin/extract-strings.mjs",
     "i18n:update-translations": "node bin/update-modules-translations.mjs",
     "zcli": "zcli"
   },
@@ -82,6 +82,7 @@
     "eslint-plugin-prettier": "4.2.1",
     "eslint-plugin-react": "^7.32.2",
     "eslint-plugin-react-hooks": "^4.6.0",
+    "glob": "^10.4.5",
     "husky": "8.0.2",
     "i18next-parser": "^8.13.0",
     "jest": "^29.6.1",