diff --git a/.env.example b/.env.example index 983c975bb368..e69de29bb2d1 100644 --- a/.env.example +++ b/.env.example @@ -1 +0,0 @@ -ALLOW_TRANSLATION_COMMITS= diff --git a/.github/actions-scripts/msft-create-translation-batch-pr.js b/.github/actions-scripts/msft-create-translation-batch-pr.js deleted file mode 100755 index df5f739b6036..000000000000 --- a/.github/actions-scripts/msft-create-translation-batch-pr.js +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env node - -import fs from 'fs' -import github from '@actions/github' - -const OPTIONS = Object.fromEntries( - ['BASE', 'BODY_FILE', 'GITHUB_TOKEN', 'HEAD', 'LANGUAGE', 'TITLE', 'GITHUB_REPOSITORY'].map( - (envVarName) => { - const envVarValue = process.env[envVarName] - if (!envVarValue) { - throw new Error(`You must supply a ${envVarName} environment variable`) - } - return [envVarName, envVarValue] - } - ) -) - -if (!process.env.GITHUB_REPOSITORY) { - throw new Error('GITHUB_REPOSITORY environment variable not set') -} - -const RETRY_STATUSES = [ - 422, // Retry the operation if the PR already exists - 502, // Retry the operation if the API responds with a `502 Bad Gateway` error. -] -const RETRY_ATTEMPTS = 3 -const { - // One of the default environment variables provided by Actions. - GITHUB_REPOSITORY, - - // These are passed in from the step in the workflow file. - TITLE, - BASE, - HEAD, - LANGUAGE, - BODY_FILE, - GITHUB_TOKEN, -} = OPTIONS -const [OWNER, REPO] = GITHUB_REPOSITORY.split('/') - -const octokit = github.getOctokit(GITHUB_TOKEN) - -/** - * @param {object} config Configuration options for finding the PR. - * @returns {Promise} The PR number. - */ -async function findPullRequestNumber(config) { - // Get a list of PRs and see if one already exists. - const { data: listOfPullRequests } = await octokit.rest.pulls.list({ - owner: config.owner, - repo: config.repo, - head: `${config.owner}:${config.head}`, - }) - - return listOfPullRequests[0]?.number -} - -/** - * When this file was first created, we only introduced support for creating a pull request for some translation batch. - * However, some of our first workflow runs failed during the pull request creation due to a timeout error. - * There have been cases where, despite the timeout error, the pull request gets created _anyway_. - * To accommodate this reality, we created this function to look for an existing pull request before a new one is created. - * Although the "find" check is redundant in the first "cycle", it's designed this way to recursively call the function again via its retry mechanism should that be necessary. - * - * @param {object} config Configuration options for creating the pull request. - * @returns {Promise} The PR number. - */ -async function findOrCreatePullRequest(config) { - const found = await findPullRequestNumber(config) - - if (found) { - return found - } - - try { - const { data: pullRequest } = await octokit.rest.pulls.create({ - owner: config.owner, - repo: config.repo, - base: config.base, - head: config.head, - title: config.title, - body: config.body, - draft: false, - }) - - return pullRequest.number - } catch (error) { - if (!error.response || !config.retryCount) { - throw error - } - - if (!config.retryStatuses.includes(error.response.status)) { - throw error - } - - console.error(`Error creating pull request: ${error.message}`) - console.warn(`Retrying in 5 seconds...`) - await new Promise((resolve) => setTimeout(resolve, 5000)) - - config.retryCount -= 1 - - return findOrCreatePullRequest(config) - } -} - -/** - * @param {object} config Configuration options for labeling the PR - * @returns {Promise} - */ -async function labelPullRequest(config) { - await octokit.rest.issues.update({ - owner: config.owner, - repo: config.repo, - issue_number: config.issue_number, - labels: config.labels, - }) -} - -async function main() { - const options = { - title: TITLE, - base: BASE, - head: HEAD, - body: fs.readFileSync(BODY_FILE, 'utf8'), - labels: ['translation-batch', `translation-batch-${LANGUAGE}`], - owner: OWNER, - repo: REPO, - retryStatuses: RETRY_STATUSES, - retryCount: RETRY_ATTEMPTS, - } - - options.issue_number = await findOrCreatePullRequest(options) - const pr = `${GITHUB_REPOSITORY}#${options.issue_number}` - console.log(`Created PR ${pr}`) - - // metadata parameters aren't currently available in `github.rest.pulls.create`, - // but they are in `github.rest.issues.update`. - await labelPullRequest(options) - console.log(`Updated ${pr} with these labels: ${options.labels.join(', ')}`) -} - -main() diff --git a/.github/workflows/msft-create-translation-batch-pr.yml b/.github/workflows/msft-create-translation-batch-pr.yml deleted file mode 100644 index bab7a09566f0..000000000000 --- a/.github/workflows/msft-create-translation-batch-pr.yml +++ /dev/null @@ -1,193 +0,0 @@ -name: Create translation Batch Pull Request (Microsoft) - -# **What it does**: -# - Creates one pull request per language after running a series of automated checks, -# removing translations that are broken in any known way -# **Why we have it**: -# - To deploy translations -# **Who does it impact**: It automates what would otherwise be manual work, -# helping docs engineering focus on higher value work - -on: - workflow_dispatch: - schedule: - - cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:20 PST - -permissions: - contents: write - -jobs: - create-translation-batch: - name: Create translation batch - if: github.repository == 'github/docs-internal' - runs-on: ubuntu-latest - # A sync's average run time is ~3.2 hours. - # This sets a maximum execution time of 300 minutes (5 hours) to prevent the workflow from running longer than necessary. - timeout-minutes: 300 - strategy: - fail-fast: false - max-parallel: 1 - matrix: - include: - - language: es - language_dir: translations/es-ES - language_repo: github/docs-internal.es-es - - - language: ja - language_dir: translations/ja-JP - language_repo: github/docs-internal.ja-jp - - - language: pt - language_dir: translations/pt-BR - language_repo: github/docs-internal.pt-br - - - language: zh - language_dir: translations/zh-CN - language_repo: github/docs-internal.zh-cn - - # We'll be ready to add the following languages in a future effort. - - - language: ru - language_dir: translations/ru-RU - language_repo: github/docs-internal.ru-ru - - - language: ko - language_dir: translations/ko-KR - language_repo: github/docs-internal.ko-kr - - - language: fr - language_dir: translations/fr-FR - language_repo: github/docs-internal.fr-fr - - - language: de - language_dir: translations/de-DE - language_repo: github/docs-internal.de-de - - steps: - - name: Set branch name - id: set-branch - run: | - echo "BRANCH_NAME=msft-translation-batch-${{ matrix.language }}-$(date +%Y-%m-%d__%H-%M)" >> $GITHUB_OUTPUT - - run: git config --global user.name "docubot" - - run: git config --global user.email "67483024+docubot@users.noreply.github.com" - - - name: Checkout the docs-internal repo - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 - with: - fetch-depth: 0 - lfs: true - - - name: Create a branch for the current language - run: git checkout -b ${{ steps.set-branch.outputs.BRANCH_NAME }} - - - name: Remove unwanted git hooks - run: rm .git/hooks/post-checkout - - - name: Remove all language translations - run: | - git rm -rf --quiet ${{ matrix.language_dir }}/content - git rm -rf --quiet ${{ matrix.language_dir }}/data - - - name: Checkout the language-specific repo - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 - with: - repository: ${{ matrix.language_repo }} - token: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }} - path: ${{ matrix.language_dir }} - - - name: Remove .git from the language-specific repo - run: rm -rf ${{ matrix.language_dir }}/.git - - - name: Commit translated files - run: | - git add ${{ matrix.language_dir }} - git commit -m "Add translations" || echo "Nothing to commit" - - - name: 'Setup node' - uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516 - with: - node-version: '16.17.0' - - - run: npm ci - - - name: Homogenize frontmatter - run: | - node script/i18n/homogenize-frontmatter.js - git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/homogenize-frontmatter.js" || echo "Nothing to commit" - - - name: Fix translation errors - run: | - node script/i18n/fix-translation-errors.js - git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/fix-translation-errors.js" || echo "Nothing to commit" - - - name: Check rendering - run: | - node script/i18n/lint-translation-files.js --check rendering | tee -a /tmp/batch.log | cat - git add ${{ matrix.language_dir }} && git commit -m "Run script/i18n/lint-translation-files.js --check rendering" || echo "Nothing to commit" - - - name: Reset files with broken liquid tags - run: | - node script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }} | tee -a /tmp/batch.log | cat - git add ${{ matrix.language_dir }} && git commit -m "run script/i18n/msft-reset-files-with-broken-liquid-tags.js --language=${{ matrix.language }}" || echo "Nothing to commit" - - - name: Check in CSV report - run: | - mkdir -p translations/log - csvFile=translations/log/msft-${{ matrix.language }}-resets.csv - script/i18n/msft-report-reset-files.js --report-type=csv --language=${{ matrix.language }} --log-file=/tmp/batch.log > $csvFile - git add -f $csvFile && git commit -m "Check in ${{ matrix.language }} CSV report" || echo "Nothing to commit" - - - name: Write the reported files that were reset to /tmp/pr-body.txt - run: script/i18n/msft-report-reset-files.js --report-type=pull-request-body --language=${{ matrix.language }} --log-file=/tmp/batch.log --csv-path=${{ steps.set-branch.outputs.BRANCH_NAME }}/translations/log/msft-${{ matrix.language }}-resets.csv > /tmp/pr-body.txt - - - name: Push filtered translations - run: git push origin ${{ steps.set-branch.outputs.BRANCH_NAME }} - - - name: Close existing stale batches - uses: lee-dohm/close-matching-issues@e9e43aad2fa6f06a058cedfd8fb975fd93b56d8f - with: - token: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} - query: 'type:pr label:translation-batch-${{ matrix.language }}' - - - name: Create translation batch pull request - env: - GITHUB_TOKEN: ${{ secrets.DOCUBOT_REPO_PAT }} - TITLE: 'New translation batch for ${{ matrix.language }}' - BASE: 'main' - HEAD: ${{ steps.set-branch.outputs.BRANCH_NAME }} - LANGUAGE: ${{ matrix.language }} - BODY_FILE: '/tmp/pr-body.txt' - run: .github/actions-scripts/msft-create-translation-batch-pr.js - - - name: Approve PR - if: github.ref_name == 'main' - env: - GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} - run: gh pr review --approve || echo "Nothing to approve" - - - name: Set auto-merge - if: github.ref_name == 'main' - env: - GITHUB_TOKEN: ${{ secrets.OCTOMERGER_PAT_WITH_REPO_AND_WORKFLOW_SCOPE }} - run: gh pr merge ${{ steps.set-branch.outputs.BRANCH_NAME }} --auto --squash || echo "Nothing to merge" - - # When the maximum execution time is reached for this job, Actions cancels the workflow run. - # This emits a notification for the first responder to triage. - - name: Send Slack notification if workflow is cancelled - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - if: cancelled() - with: - channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}🎉 - color: failure - text: 'The new translation batch for ${{ matrix.language }} was cancelled.' - - # Emit a notification for the first responder to triage if the workflow failed. - - name: Send Slack notification if workflow failed - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - if: failure() - with: - channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} - color: failure - text: 'The new translation batch for ${{ matrix.language }} failed.' diff --git a/.github/workflows/triage-unallowed-contributions.yml b/.github/workflows/triage-unallowed-contributions.yml index 61a21c7918bd..bd1ee42b3388 100644 --- a/.github/workflows/triage-unallowed-contributions.yml +++ b/.github/workflows/triage-unallowed-contributions.yml @@ -20,7 +20,6 @@ on: - 'lib/webhooks/**' - 'package*.json' - 'script/**' - - 'translations/**' - 'content/actions/deployment/security-hardening-your-deployments/**' permissions: @@ -49,8 +48,6 @@ jobs: # Returns list of changed files matching each filter filters: | - translation: - - 'translations/**' openapi: - 'lib/rest/static/**' notAllowed: @@ -67,7 +64,6 @@ jobs: - 'lib/webhooks/**' - 'package*.json' - 'scripts/**' - - 'translations/**' - 'content/actions/deployment/security-hardening-your-deployments/**' # When there are changes to files we can't accept, leave a comment @@ -91,7 +87,6 @@ jobs: 'lib/webhooks/**', 'package*.json', 'scripts/**', - 'translations/**', 'content/actions/deployment/security-hardening-your-deployments/**', ] diff --git a/.prettierignore b/.prettierignore index b9d22bdd8913..2802f7d8a872 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,3 @@ -/translations/ includes/ data/release-notes/ script/bookmarklets/ diff --git a/.vscode/settings.json b/.vscode/settings.json index d331b10571f7..ab59a4f37f78 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,4 @@ { - "files.exclude": { - "translations/**": true - }, "workbench.editor.enablePreview": false, "workbench.editor.enablePreviewFromQuickOpen": false } diff --git a/nodemon.json b/nodemon.json index 03f870fc3b5a..7ed242020748 100644 --- a/nodemon.json +++ b/nodemon.json @@ -3,7 +3,6 @@ "ignore": [ "assets", "script", - "translations", "stylesheets", "tests", "content", diff --git a/script/README.md b/script/README.md index 91c1b084dbfa..7ffe0f1c772e 100644 --- a/script/README.md +++ b/script/README.md @@ -451,68 +451,6 @@ A helper that returns an array of files for a given path and file extension. --- -### [`i18n/fix-translation-errors.js`](i18n/fix-translation-errors.js) - -Run this script to fix known frontmatter errors by copying values from english file Currently only fixing errors in: 'type', 'changelog' Please double check the changes created by this script before committing. - ---- - - -### [`i18n/homogenize-frontmatter.js`](i18n/homogenize-frontmatter.js) - -Run this script to fix known frontmatter errors by copying values from english file Translatable properties are designated in the frontmatter JSON schema - ---- - - -### [`i18n/lint-translation-files.js`](i18n/lint-translation-files.js) - -Use this script as part of the translation merge process to output a list of either parsing or rendering errors in translated files and run script/i18n/reset-translated-file.js on them. - ---- - - -### [`i18n/msft-report-reset-files.js`](i18n/msft-report-reset-files.js) - - - ---- - - -### [`i18n/msft-reset-files-with-broken-liquid-tags.js`](i18n/msft-reset-files-with-broken-liquid-tags.js) - - - ---- - - -### [`i18n/msft-tokens.js`](i18n/msft-tokens.js) - - - ---- - - -### [`i18n/prune-stale-files.js`](i18n/prune-stale-files.js) - - - ---- - - -### [`i18n/reset-translated-file.js`](i18n/reset-translated-file.js) - -This is a convenience script for replacing the contents of translated files with the English content from their corresponding source file. - -Usage: script/i18n/reset-translated-file.js - -Examples: - -$ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md - ---- - - ### [`i18n/test-html-pages.js`](i18n/test-html-pages.js) @@ -520,13 +458,6 @@ $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index. --- -### [`i18n/test-render-translation.js`](i18n/test-render-translation.js) - -Run this script to test-render all the translation files that have been changed (when compared to the `main` branch). - ---- - - ### [`kill-server-for-jest.js`](kill-server-for-jest.js) @@ -577,13 +508,6 @@ This script is intended to be used as a git "prepush" hook. If the current branc --- -### [`prevent-translation-commits.js`](prevent-translation-commits.js) - -This script is run as a git precommit hook (installed by husky after npm install). It detects changes to files the in the translations folder and prevents the commit if any changes exist. - ---- - - ### [`purge-fastly`](purge-fastly) Run this script to manually purge the Fastly cache. Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file. diff --git a/script/content-migrations/README.md b/script/content-migrations/README.md index bf6a3f59fa52..1cc264b2d040 100644 --- a/script/content-migrations/README.md +++ b/script/content-migrations/README.md @@ -2,6 +2,4 @@ This directory stores scripts that modify content and/or data files. Because writers are updating content all the time, scripts in here require more -cross-team coordination and planning before they are run. Make sure to consider -whether a script added here also needs to be run on translation files or if we -can wait for the changes to come in through out translation automation. +cross-team coordination and planning before they are run. Make sure to consider if we can wait for the changes to come in through out translation automation. diff --git a/script/i18n/fix-translation-errors.js b/script/i18n/fix-translation-errors.js deleted file mode 100755 index 2d48e354ddca..000000000000 --- a/script/i18n/fix-translation-errors.js +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// Run this script to fix known frontmatter errors by copying values from english file -// Currently only fixing errors in: 'type', 'changelog' -// Please double check the changes created by this script before committing. -// -// [end-readme] - -import path from 'path' -import { execSync } from 'child_process' -import { get, set } from 'lodash-es' -import fs from 'fs' -import fm from '../../lib/frontmatter.js' -import matter from 'gray-matter' -import chalk from 'chalk' -import yaml from 'js-yaml' -import releaseNotesSchema from '../../tests/helpers/schemas/release-notes-schema.js' -import revalidator from 'revalidator' - -main() - -async function main() { - const fixableFmProps = Object.keys(fm.schema.properties) - .filter((property) => !fm.schema.properties[property].translatable) - .sort() - const fixableYmlProps = ['date'] - - const loadAndValidateContent = async (path, schema) => { - let fileContents - try { - fileContents = await fs.promises.readFile(path, 'utf8') - } catch (e) { - if (fs.existsSync(path)) { - console.error(e.message) - } - return null - } - - if (path.endsWith('yml')) { - let data - let errors = [] - try { - data = yaml.load(fileContents) - } catch {} - if (data && schema) { - ;({ errors } = revalidator.validate(data, schema)) - } - return { data, errors, content: null } - } else { - return fm(fileContents) - } - } - - const cmd = - 'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/(content/.+.md|data/release-notes/.*.yml)$"' - - const maxBuffer = 1024 * 1024 * 2 // twice the default value - const changedFilesRelPaths = execSync(cmd, { maxBuffer }).toString().split('\n') - - for (const relPath of changedFilesRelPaths) { - // Skip READMEs - if (!relPath || relPath.endsWith('README.md')) continue - - // find the corresponding english file by removing the first 2 path segments: /translation/ - const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep) - - const localisedResult = await loadAndValidateContent(relPath, releaseNotesSchema) - if (!localisedResult) continue - const { data, errors, content } = localisedResult - - const fixableProps = relPath.endsWith('yml') ? fixableYmlProps : fixableFmProps - - const fixableErrors = errors.filter(({ property }) => { - const prop = property.split('.') - return fixableProps.includes(prop[0]) - }) - - if (!data || fixableErrors.length === 0) continue - - const engResult = await loadAndValidateContent(engAbsPath) - if (!engResult) continue - const { data: engData } = engResult - - console.log(chalk.bold(relPath)) - - const newData = data - - fixableErrors.forEach(({ property, message }) => { - const correctValue = get(engData, property) - console.log(chalk.red(` error message: [${property}] ${message}`)) - console.log(` fix property [${property}]: ${get(data, property)} -> ${correctValue}`) - set(newData, property, correctValue) - }) - - let toWrite - if (content) { - toWrite = matter.stringify(content, newData, { lineWidth: 10000, forceQuotes: true }) - } else { - toWrite = yaml.dump(newData, { lineWidth: 10000, forceQuotes: true }) - } - - fs.writeFileSync(relPath, toWrite) - } -} diff --git a/script/i18n/homogenize-frontmatter.js b/script/i18n/homogenize-frontmatter.js deleted file mode 100755 index 213b9776ee97..000000000000 --- a/script/i18n/homogenize-frontmatter.js +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// Run this script to fix known frontmatter errors by copying values from english file -// Translatable properties are designated in the frontmatter JSON schema -// -// [end-readme] - -import path from 'path' -import fs from 'fs/promises' -import matter from 'gray-matter' -import walk from 'walk-sync' -import fm from '../../lib/frontmatter.js' - -// Run! -main() - -async function main() { - const translatedMarkdownFiles = walk('translations') - .filter((filename) => { - return ( - filename.includes('/content/') && - filename.endsWith('.md') && - !filename.endsWith('README.md') - ) - }) - .map((filename) => `translations/${filename}`) - - console.log( - ( - await Promise.all( - translatedMarkdownFiles.map(async (relPath) => - updateTranslatedMarkdownFile(relPath).catch((e) => `Error in ${relPath}: ${e.message}`) - ) - ) - ) - .filter(Boolean) - .join('\n') - ) -} - -async function extractFrontmatter(path) { - const fileContents = await fs.readFile(path, 'utf8') - return fm(fileContents) -} - -async function updateTranslatedMarkdownFile(relPath) { - // find the corresponding english file by removing the first 2 path segments: /translations/ - const engAbsPath = relPath.split(path.sep).slice(2).join(path.sep) - - // Load frontmatter from the source english file - let englishFrontmatter - try { - englishFrontmatter = await extractFrontmatter(engAbsPath) - } catch { - // This happens when an English file has been moved or deleted and translations are not in sync. - // It does mean this script will not homogenous those translated files, but the docs site does not - // load translated files that don't correlate to an English file, so those translated files can't break things. - // return `${relPath}: English file does not exist: ${engAbsPath}` - return // silence - } - - const localisedFrontmatter = await extractFrontmatter(relPath) - if (!localisedFrontmatter) return `${relPath}: No localised frontmatter` - - // Look for differences between the english and localised non-translatable properties - let overwroteSomething = false - for (const prop in localisedFrontmatter.data) { - if ( - !fm.schema.properties[prop].translatable && - englishFrontmatter.data[prop] && - localisedFrontmatter.data[prop] !== englishFrontmatter.data[prop] - ) { - localisedFrontmatter.data[prop] = englishFrontmatter.data[prop] - overwroteSomething = true - } - } - - // rewrite the localised file, if it changed - if (overwroteSomething) { - const toWrite = matter.stringify(localisedFrontmatter.content, localisedFrontmatter.data, { - lineWidth: 10000, - forceQuotes: true, - }) - await fs.writeFile(relPath, toWrite) - - // return `${relPath}: updated` - // silence - } -} diff --git a/script/i18n/msft-report-reset-files.js b/script/i18n/msft-report-reset-files.js deleted file mode 100755 index 13b1c5d65695..000000000000 --- a/script/i18n/msft-report-reset-files.js +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node - -import { program } from 'commander' -import fs from 'fs' -import languages from '../../lib/languages.js' - -const defaultWorkflowUrl = [ - process.env.GITHUB_SERVER_URL, - process.env.GITHUB_REPOSITORY, - 'actions/runs', - process.env.GITHUB_RUN_ID, -].join('/') - -const reportTypes = { - 'pull-request-body': pullRequestBodyReport, - csv: csvReport, -} - -program - .description('Reads a translation batch log and generates a report') - .requiredOption('--language ', 'The language to compare') - .requiredOption('--log-file ', 'The batch log file') - .requiredOption( - '--report-type ', - 'The batch log file, I.E: ' + Object.keys(reportTypes).join(', ') - ) - .option('--workflow-url ', 'The workflow url', defaultWorkflowUrl) - .option('--csv-path ', 'The path to the CSV file') - .parse(process.argv) - -const options = program.opts() -const language = languages[options.language] -const { logFile, workflowUrl, reportType, csvPath } = options - -if (!Object.keys(reportTypes).includes(reportType)) { - throw new Error(`Invalid report type: ${reportType}`) -} - -const logFileContents = fs.readFileSync(logFile, 'utf8') - -const revertLines = logFileContents - .split('\n') - .filter((line) => line.match(/^(-> reverted to English)|^(-> removed)/)) - .filter((line) => line.match(language.dir)) - -const reportEntries = revertLines.sort().map((line) => { - const [, file, reason] = line.match(/^-> (?:reverted to English|removed): (.*) Reason: (.*)$/) - return { file, reason } -}) - -function pullRequestBodyReport() { - return [ - `New translation batch for ${language.name}. Product of [this workflow](${workflowUrl}). - -## ${reportEntries.length} files reverted. - -You can see the log in [\`${csvPath}\`](${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/blob/${csvPath}).`, - ].join('\n') -} - -function csvReport() { - const lines = reportEntries.map(({ file, reason }) => { - return [file, reason].join(',') - }) - - return ['file,reason', lines].flat().join('\n') -} - -console.log(reportTypes[reportType]()) diff --git a/script/i18n/msft-reset-files-with-broken-liquid-tags.js b/script/i18n/msft-reset-files-with-broken-liquid-tags.js deleted file mode 100755 index 7cecbed2665e..000000000000 --- a/script/i18n/msft-reset-files-with-broken-liquid-tags.js +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env node - -import { program } from 'commander' -import { execFileSync } from 'child_process' -import { languageFiles, compareLiquidTags } from './msft-tokens.js' -import languages from '../../lib/languages.js' - -program - .description('show-liquid-tags-diff') - .requiredOption('-l, --language ', 'The language to compare') - .option('-d, --dry-run', 'Just pretend to reset files') - .parse(process.argv) - -function resetFiles(files) { - console.log(`Reseting ${files.length} files:`) - - const dryRun = program.opts().dryRun ? '--dry-run' : '' - - files.forEach((file) => { - const cmd = 'script/i18n/reset-translated-file.js' - const args = [file, '--reason', 'broken liquid tags', dryRun] - execFileSync(cmd, args, { stdio: 'inherit' }) - }) -} - -function deleteFiles(files) { - console.log(`Deleting ${files.length} files:`) - - const dryRun = program.opts().dryRun ? '--dry-run' : '' - - files.forEach((file) => { - const cmd = 'script/i18n/reset-translated-file.js' - const args = [ - file, - '--remove', - '--reason', - 'file deleted because it no longer exists in main', - dryRun, - ] - execFileSync(cmd, args, { stdio: 'inherit' }) - }) -} - -async function main() { - const options = program.opts() - const language = languages[options.language] - - if (!language) { - throw new Error(`Language ${options.language} not found`) - } - - // languageFiles() returns an array indexed as follows: - // [0]: intersection of the files that exist in both main and the language-specific branch - // [1]: files that exist only in the language-specific branch, not in main - const allContentFiles = languageFiles(language, 'content') - const allDataFiles = languageFiles(language, 'data') - const files = [allContentFiles[0], allDataFiles[0]].flat() - const nonexitentFiles = [allContentFiles[1], allDataFiles[1]].flat() - const brokenFiles = [] - - files.forEach((file) => { - try { - // it throws error if the the syntax is invalid - const comparison = compareLiquidTags(file, language) - - if (comparison.diff.count === 0) { - return - } - - brokenFiles.push(comparison.translation) - } catch (e) { - brokenFiles.push(e.filePath) - } - }) - - await resetFiles(brokenFiles) - await deleteFiles(nonexitentFiles) -} - -main() diff --git a/script/i18n/msft-tokens.js b/script/i18n/msft-tokens.js deleted file mode 100644 index eee1ede1816b..000000000000 --- a/script/i18n/msft-tokens.js +++ /dev/null @@ -1,90 +0,0 @@ -import walk from 'walk-sync' -import { Tokenizer } from 'liquidjs' -import { readFileSync } from 'fs' -import gitDiff from 'git-diff' -import _ from 'lodash' - -function getGitDiff(a, b) { - return gitDiff(a, b, { flags: '--text --ignore-all-space' }) -} - -function getMissingLines(diff) { - return diff - .split('\n') - .filter((line) => line.startsWith('-')) - .map((line) => line.replace('-', '')) -} - -function getExceedingLines(diff) { - return diff - .split('\n') - .filter((line) => line.startsWith('+')) - .map((line) => line.replace('+', '')) -} - -export function languageFiles(language, folder = 'content') { - const englishFiles = walk(folder, { directories: false }) - const languageFiles = walk(`${language.dir}/${folder}`, { directories: false }) - return [ - _.intersection(englishFiles, languageFiles).map((file) => `${folder}/${file}`), - _.difference(languageFiles, englishFiles).map((file) => `${language.dir}/${folder}/${file}`), // returns languageFiles not included in englishFiles - ] -} - -export function compareLiquidTags(file, language) { - const translation = `${language.dir}/${file}` - const sourceTokens = getTokensFromFile(file).rejectType('html') - const otherFileTokens = getTokensFromFile(translation).rejectType('html') - const diff = sourceTokens.diff(otherFileTokens) - - return { - file, - translation, - diff, - } -} - -function getTokens(contents) { - const tokenizer = new Tokenizer(contents) - return new Tokens(...tokenizer.readTopLevelTokens()) -} - -export function getTokensFromFile(filePath) { - const contents = readFileSync(filePath, 'utf8') - try { - return new Tokens(...getTokens(contents)) - } catch (e) { - const error = new Error(`Error parsing ${filePath}: ${e.message}`) - error.filePath = filePath - throw error - } -} - -export class Tokens extends Array { - rejectType(tagType) { - return this.filter( - (token) => token.constructor.name.toUpperCase() !== `${tagType}Token`.toUpperCase() - ) - } - - onlyText() { - return this.map((token) => token.getText()) - } - - diff(otherTokens) { - const a = this.onlyText().sort() - const b = otherTokens.onlyText().sort() - - const diff = getGitDiff(a.join('\n'), b.join('\n')) - - if (!diff) { - return { count: 0, missing: [], exceeding: [], output: '' } - } - - const missing = getMissingLines(diff) - const exceeding = getExceedingLines(diff) - const count = exceeding.length + missing.length - - return { count, missing, exceeding, output: diff } - } -} diff --git a/script/i18n/prune-stale-files.js b/script/i18n/prune-stale-files.js deleted file mode 100755 index d251913e0d0f..000000000000 --- a/script/i18n/prune-stale-files.js +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env node -import fs from 'fs' -import path from 'path' -import walk from 'walk-sync' -import { program } from 'commander' -import languages from '../../lib/languages.js' - -program - .description( - `Removes any file in the translations directory that doesn't have a 1-1 mapping with an English file in the content directory` - ) - .option('-d, --dry-run', `List the files that will be deleted, but don't remove them).`) - .parse(process.argv) - -const languageDir = Object.keys(languages) - .filter((language) => !languages[language].wip && language !== 'en') - .map((language) => languages[language].dir) - -main() - -async function main() { - const listOfContentFiles = walk(path.join(process.cwd(), 'content'), { - includeBasePath: false, - directories: false, - }) - - const translatedFilePaths = [] - languageDir.forEach((directory) => { - const listOfFiles = walk(path.join(directory, 'content'), { - includeBasePath: true, - directories: false, - }).map((path) => path.replace(process.cwd(), '')) - translatedFilePaths.push(...listOfFiles) - }) - - let outOfSyncFilesCount = 0 - translatedFilePaths.forEach((translatedFilePath) => { - const translationRelativePath = translatedFilePath.split('/content/')[1] - - // If there is a 1:1 mapping of translated file to english file - // we're in sync, don't log - if (listOfContentFiles.includes(translationRelativePath)) { - return - } - - outOfSyncFilesCount++ - if (!program.opts().dryRun) { - fs.unlinkSync(translatedFilePath) - } else { - console.log(translatedFilePath) - } - }) - - console.log(`Out of sync file size: ${outOfSyncFilesCount}`) -} diff --git a/script/i18n/reset-translated-file.js b/script/i18n/reset-translated-file.js deleted file mode 100755 index 4a88da8c48e8..000000000000 --- a/script/i18n/reset-translated-file.js +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// This is a convenience script for replacing the contents of translated -// files with the English content from their corresponding source file. -// -// Usage: -// script/i18n/reset-translated-file.js -// -// Examples: -// -// $ script/i18n/reset-translated-file.js translations/es-XL/content/actions/index.md -// -// [end-readme] - -import { program } from 'commander' -import { execSync } from 'child_process' -import assert from 'assert' -import fs from 'fs' -import path from 'path' -import chalk from 'chalk' - -program - .description('reset translated files') - .option( - '-m, --prefer-main', - 'Reset file to the translated file, try using the file from `main` branch first, if not found (usually due to renaming), fall back to English source.' - ) - .option('-rm, --remove', 'Remove the translated files altogether') - .option('-d, --dry-run', 'Just pretend to reset files') - .option('-r, --reason ', 'A reason why the file is getting reset') - .parse(process.argv) - -const dryRun = program.opts().dryRun -const reason = program.opts().reason -const reasonMessage = reason ? `Reason: ${reason}` : '' - -const resetToEnglishSource = (translationFilePath) => { - assert( - translationFilePath.startsWith('translations/'), - 'path argument must be in the format `translations//path/to/file`' - ) - - if (program.opts().remove) { - if (!dryRun) { - const fullPath = path.join(process.cwd(), translationFilePath) - fs.unlinkSync(fullPath) - } - console.log('-> removed: %s %s', translationFilePath, reasonMessage) - return - } - if (!fs.existsSync(translationFilePath)) { - return - } - - const relativePath = translationFilePath.split(path.sep).slice(2).join(path.sep) - const englishFile = path.join(process.cwd(), relativePath) - - if (!dryRun && !fs.existsSync(englishFile)) { - fs.unlinkSync(translationFilePath) - return - } - - if (!dryRun) { - // it is important to replace the file with English source instead of - // removing it, and relying on the fallback, because redired_from frontmatter - // won't work in fallbacks - const englishContent = fs.readFileSync(englishFile, 'utf8') - fs.writeFileSync(translationFilePath, englishContent) - } - - console.log( - '-> reverted to English: %s %s', - path.relative(process.cwd(), translationFilePath), - reasonMessage - ) -} - -const [pathArg] = program.args -assert(pathArg, 'first arg must be a target filename') - -// Is the arg a fully-qualified path? -const relativePath = fs.existsSync(pathArg) ? path.relative(process.cwd(), pathArg) : pathArg - -if (program.opts().preferMain) { - try { - if (!dryRun) { - execSync(`git checkout main -- ${relativePath}`, { stdio: 'pipe' }) - } - console.log('-> reverted to file from main branch: %s %s', relativePath, reasonMessage) - } catch (e) { - if (e.message.includes('pathspec')) { - console.warn( - chalk.red( - `cannot find ${relativePath} in main branch (likely because it was renamed); falling back to English source file.` - ) - ) - resetToEnglishSource(relativePath) - } else { - console.warn(e.message) - } - } -} else { - resetToEnglishSource(relativePath) -} diff --git a/script/i18n/test-render-translation.js b/script/i18n/test-render-translation.js deleted file mode 100755 index 0ce20c328a78..000000000000 --- a/script/i18n/test-render-translation.js +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// Run this script to test-render all the translation files that have been changed (when compared to the `main` branch). -// -// [end-readme] - -import renderContent from '../../lib/render-content/index.js' -import loadSiteData from '../../lib/site-data.js' -import { loadPages } from '../../lib/page-data.js' -import languages from '../../lib/languages.js' -import { promisify } from 'util' -import ChildProcess, { execSync } from 'child_process' -import fs from 'fs' -import frontmatter from '../../lib/frontmatter.js' -import chalk from 'chalk' -import { YAMLException } from 'js-yaml' - -const fmSchemaProperties = frontmatter.schema.properties -const exec = promisify(ChildProcess.exec) - -main() - -async function main() { - const siteData = await loadAndPatchSiteData() - const pages = await loadPages() - const contextByLanguage = {} - for (const lang in languages) { - const langObj = languages[lang] - const [langCode] = langObj.dir === '' ? 'en' : langObj.dir.split('/').slice(1) - if (!langCode) continue - contextByLanguage[langCode] = { - site: siteData[langObj.code].site, - currentLanguage: langObj.code, - currentVersion: 'free-pro-team@latest', - } - } - - const changedFilesRelPaths = execSync( - 'git -c diff.renameLimit=10000 diff --name-only origin/main | egrep "^translations/.*/.+(.md|.yml)$"', - { maxBuffer: 1024 * 1024 * 100 } - ) - .toString() - .split('\n') - .filter((path) => path !== '' && !path.endsWith('README.md')) - .sort() - - console.log(`Found ${changedFilesRelPaths.length} translated files.`) - - for (const relPath of changedFilesRelPaths) { - const lang = relPath.split('/')[1] - const context = { - ...contextByLanguage[lang], - pages, - page: pages.find((page) => { - const pageRelPath = `${languages[page.languageCode].dir}/content/${page.relativePath}` - return pageRelPath === relPath - }), - redirects: {}, - } - - // specifically test rendering data/variables files for broken liquid - if (relPath.includes('data/variables')) { - const fileContents = await fs.promises.readFile(relPath, 'utf8') - const { content } = frontmatter(fileContents) - - try { - await renderContent.liquid.parseAndRender(content, context) - } catch (err) { - console.log(chalk.bold(relPath)) - console.log(chalk.red(` error message: ${err.message}`)) - } - } - - if (!context.page && !relPath.includes('data/reusables')) continue - const fileContents = await fs.promises.readFile(relPath, 'utf8') - const { data, content } = frontmatter(fileContents) - const translatableFm = Object.keys(data).filter((key) => fmSchemaProperties[key].translatable) - try { - // test the content - await renderContent.liquid.parseAndRender(content, context) - // test each translatable frontmatter property - for (const key of translatableFm) { - await renderContent.liquid.parseAndRender(data[key], context) - } - } catch (err) { - console.log(chalk.bold(relPath)) - console.log(chalk.red(` error message: ${err.message}`)) - } - } -} - -async function loadAndPatchSiteData(filesWithKnownIssues = {}) { - try { - const siteData = loadSiteData() - return siteData - } catch (error) { - if (error instanceof YAMLException && error.mark) { - const relPath = error.mark.name - if (!filesWithKnownIssues[relPath]) { - // Note the file as problematic - filesWithKnownIssues[relPath] = true - - // This log is important as it will get ${relPath} written to a logfile - console.log(chalk.bold(relPath)) - console.log(chalk.red(` error message: ${error.toString()}`)) - - // Reset the file - console.warn(`resetting file "${relPath}" due to loadSiteData error: ${error.toString()}`) - await exec( - `script/i18n/reset-translated-file.js --prefer-main ${relPath} --reason="loadSiteData error"` - ) - - // Try to load the site data again - return loadAndPatchSiteData(filesWithKnownIssues) - } else { - console.error(`FATAL: Tried to reset file "${relPath}" but still had errors`) - } - } - - throw error - } -} diff --git a/script/server-all-languages b/script/server-all-languages index 5c76ee3f5b5c..4358a18e81e1 100755 --- a/script/server-all-languages +++ b/script/server-all-languages @@ -8,4 +8,6 @@ source script/check-for-node +# TODO would need git clones from the language repos + npm run start-all-languages diff --git a/tests/unit/liquid-tags/tokens-test.js b/tests/unit/liquid-tags/tokens-test.js deleted file mode 100644 index c6e34e1b99f1..000000000000 --- a/tests/unit/liquid-tags/tokens-test.js +++ /dev/null @@ -1,82 +0,0 @@ -import { expect } from '@jest/globals' -import path from 'path' -import { fileURLToPath } from 'url' - -import { getTokensFromFile, Tokens } from '../../../script/i18n/msft-tokens' -const __dirname = path.dirname(fileURLToPath(import.meta.url)) - -function getFixturePath(name) { - return path.join(__dirname, '../..', 'fixtures', name) -} - -describe('getTokensFromFile', () => { - let fixturePath - let tokens - - beforeEach(() => { - fixturePath = getFixturePath('liquid-tags/minimal-conditional.md') - tokens = getTokensFromFile(fixturePath) - }) - - describe('getTokensFromFile', () => { - it('returns all the tokens from a template file', () => { - expect(tokens.length).toEqual(7) - }) - }) - - describe('Tokens', () => { - describe('.rejectType', () => { - it('rejects tokens of a particular type', () => { - const nonHtmlTokens = tokens.rejectType('html') - - expect(nonHtmlTokens.length).toEqual(3) - }) - }) - - describe('.diff', () => { - let tokens - let otherTokens - let reverseTokens - - const addTokens = (collection, elements) => { - elements.forEach((element) => { - collection.push({ getText: () => element }) - }) - } - - beforeEach(() => { - tokens = new Tokens() - otherTokens = new Tokens() - reverseTokens = new Tokens() - addTokens(tokens, ['apples', 'bananas', 'oranges']) - addTokens(otherTokens, ['apples', 'oranges']) - addTokens(reverseTokens, ['oranges', 'bananas', 'apples']) - }) - - it('shows elements that are missing', () => { - const diff = tokens.diff(otherTokens) - - expect(diff.count).toEqual(1) - expect(diff.missing).toEqual(['bananas']) - }) - - it('shows elements that are exceeding', () => { - const diff = otherTokens.diff(tokens) - - expect(diff.count).toEqual(1) - expect(diff.exceeding).toEqual(['bananas']) - }) - - it('shows no difference when collections are the same', () => { - const diff = tokens.diff(tokens) - - expect(diff.count).toEqual(0) - }) - - it('shows no difference when tokens are in different order', () => { - const diff = tokens.diff(reverseTokens) - expect(diff.count).toEqual(0) - }) - }) - }) -})