Perform flex-config env var replacement only during deploy (#501)

.github/workflows/flex_deploy.yaml

@@ -206,7 +206,7 @@ jobs:
       - name: deploy flex config
         working-directory: flex-config
         run: |
-          npm run deploy
+          npm run deploy >> $GITHUB_STEP_SUMMARY
 
   deploy-release-plugin:
     needs:

addons/serverless-schedule-manager/package.json

@@ -6,7 +6,7 @@
     "test": "echo 'Tests to come'; exit 1",
     "start": "twilio serverless:start",
     "deploy": "npm run fetch-config && twilio serverless:deploy --override-existing-project --env \".env${ENVIRONMENT:+.$ENVIRONMENT}\"",
-    "fetch-config": "node scripts/fetch-config.mjs assets/config.private.json $ENVIRONMENT",
+    "fetch-config": "node scripts/fetch-config.mjs assets/config.private.json",
     "lint": "eslint .",
     "lint:fix": "npm run lint -- --fix",
     "lint:report": "npm run lint -- --output-file eslint_report.json --format json"

addons/serverless-schedule-manager/scripts/fetch-config.mjs

@@ -34,21 +34,6 @@ if (!apiKey || !apiSecret) {
 
 let domain = '';
 const outputPath = process.argv[2];
-const environment = process.argv[3];
-
-if (environment) {
-  // First, attempt to get domain via flex-config
-  try {
-    const flexConfig = JSON.parse(await fs.readFile(`../../flex-config/ui_attributes.${environment}.json`, 'utf8'));
-    const configDomain = flexConfig?.custom_data?.features?.schedule_manager?.serverless_domain;
-
-    if (configDomain && configDomain.includes('twil.io')) {
-      domain = configDomain;
-    }
-  } catch (error) {
-    console.log('Unable to read from flex-config, fetching domain via API...', error);
-  }
-}
 
 if (!domain) {
   // Fall back to fetching domain via API

docs/docs/building/deployment/00_ci-deployment.md

@@ -19,6 +19,7 @@ This workflow encapsulates the logic for deploying the entire template. It can b
 1. Deploys Terraform
    - If the option to deploy Terraform was selected, the [Terraform deploy](#terraform-deploy) workflow is executed to deploy resources using Terraform.
    - If this is the first time the template has been deployed, there is a chicken-egg problem with the serverless and Terraform deployments (Terraform wants the serverless service to exist, but the serverless service wants the dependencies deployed by Terraform to exist). To solve this, when the initial release option is selected, the serverless services are deployed twice: Once before Terraform (in a state where some dependencies are missing), then again after Terraform (once the dependencies have been deployed).
+1. Deploys the Flex configuration by running `npm run deploy` from the `flex-config` package
 1. Deploys and releases the Flex plugin using the Twilio CLI
 
 ### Options

docs/docs/building/template-utilities/configuration.md

@@ -28,7 +28,7 @@ The `appConfig.js` file is created for you as part of the initial local environm
 #### The `custom_data` object
 The template maintains the configuration that is deployed to [hosted Flex configuration](https://www.twilio.com/docs/flex/developer/config/flex-configuration-rest-api#ui_attributes) in version control under the flex-config folder. Here, you will find `ui_attributes.common.json` file containing the main configuration set.  
 
-At the time of a GitHub action script deploy of the template, when an environment is provided, a new file is generated from `ui_attributes.example.json` and it is called `ui_attributes.<env-name>.json`.  After creating this file any placeholder values in the file, such as the serverless domain, are replaced as part of the GitHub deployment scripts.  The contents of this file are merged over the top of the `ui_attributes.common.json` file and the output creates the final configuration set that is pushed to the [hosted Flex configuration API](https://www.twilio.com/docs/flex/developer/config/flex-configuration-rest-api#ui_attributes).
+At the time of a GitHub action script deploy of the template, when an environment is provided, a new file is generated from `ui_attributes.example.json` and it is called `ui_attributes.<env-name>.json` (unless it already exists). The contents of this file are merged over the top of the `ui_attributes.common.json` file. After merging the configuration, any placeholder values, such as the serverless domain, are replaced as part of the deployment scripts, and the final configuration set is pushed to the [hosted Flex configuration API](https://www.twilio.com/docs/flex/developer/config/flex-configuration-rest-api#ui_attributes).
 
 If you wish to provide alternate feature configurations per environment, such as IDs or for testing different settings, you may create this file yourself. Simply copy `ui_attributes.example.json` to `ui_attributes.<env-name>.json`, perform the desired changes, and commit the file to the repository. Placeholder values within this file will continue to be automatically replaced as described above during deployment.
 
@@ -208,9 +208,8 @@ We can commit a `.env.<environment name here>` file, for example, `.env.dev`, to
 The setup script when run via `npm install` performs the following operations:
 1. Establish the Twilio account to use
 2. Automatically populate the `.env.<environment name here>` file for each package
-3. Automatically populate the `ui_attributes.<environment name here>.json` file for flex-config deployment if not running locally
-4. Create and populate the `plugin-flex-ts-template-v2/public/appConfig.js` file if running locally
-5. Run `npm install` for each package, so that it is ready to use.
+3. Create and populate the `plugin-flex-ts-template-v2/public/appConfig.js` file if running locally
+4. Run `npm install` for each package, so that it is ready to use.
 
 Several parameters are accepted when the script is run via `npm run postinstall`, which can be used to customize the script's functionality. These parameters can be set as follows:
 

flex-config/index.js → flex-config/index.mjs

@@ -1,7 +1,10 @@
-const axios = require("axios");
-const dotenv = require("dotenv");
-const { promises: fs } = require('fs');
-const _ = require('lodash');
+import axios from 'axios';
+import dotenv from 'dotenv';
+import { promises as fs } from 'fs';
+import merge from 'lodash/merge.js';
+
+import { fillReplacementsForString } from "../scripts/common/fill-replacements.mjs";
+import printReplacements from "../scripts/common/print-replacements.mjs";
 
 async function exists (path) {  
   try {
@@ -32,9 +35,9 @@ async function exists (path) {
 
   deployConfigurationData({
     auth: {
-      TWILIO_ACCOUNT_SID,
-      TWILIO_API_KEY,
-      TWILIO_API_SECRET,
+      accountSid: TWILIO_ACCOUNT_SID,
+      apiKey: TWILIO_API_KEY,
+      apiSecret: TWILIO_API_SECRET,
     },
     environment: ENVIRONMENT,
     overwrite: OVERWRITE_CONFIG,
@@ -54,9 +57,10 @@ Array.prototype.unique = function () {
 
 async function deployConfigurationData({ auth, environment, overwrite }) {
   try {
-
-    defaultEnvFileName = './ui_attributes.example.json';
-    envFileName = `./ui_attributes.${environment}.json`;
+    const commonFileName = './ui_attributes.common.json';
+    const defaultEnvFileName = './ui_attributes.example.json';
+    const envFileName = `./ui_attributes.${environment}.json`;
+    const skillsFileName = './taskrouter_skills.json';
     const envExists = await exists(envFileName);
 
     // first ensure environment specific file exists
@@ -68,9 +72,18 @@ async function deployConfigurationData({ auth, environment, overwrite }) {
       }
     }
 
-    const uiAttributesOverrides = require(envFileName);
-    const uiAttributesCommon = require("./ui_attributes.common.json");
-    const taskrouter_skills = require("./taskrouter_skills.json");
+    const uiAttributesEnvFile = await fs.readFile(new URL(envFileName, import.meta.url), 'utf8');
+    const uiAttributesCommonFile = await fs.readFile(new URL(commonFileName, import.meta.url), 'utf8');
+    const taskrouter_skills = JSON.parse(await fs.readFile(new URL(skillsFileName, import.meta.url), 'utf8'));
+
+    console.log("Populating environmental configuration data...");
+    const uiAttributesEnvReplaced = await fillReplacementsForString(uiAttributesEnvFile, auth, environment);
+    const uiAttributesCommonReplaced = await fillReplacementsForString(uiAttributesCommonFile, auth, environment);
+
+    printReplacements({
+      ...uiAttributesCommonReplaced.envVars,
+      ...uiAttributesEnvReplaced.envVars,
+    });
 
     console.log("Getting current configuration...");
     const {
@@ -80,12 +93,14 @@ async function deployConfigurationData({ auth, environment, overwrite }) {
 
     console.log("Merging current configuraton with new configuration...");
     let uiAttributesMerged;
+    const uiAttributesEnvJson = JSON.parse(uiAttributesEnvReplaced.data);
+    const uiAttributesCommonJson = JSON.parse(uiAttributesCommonReplaced.data);
     if (overwrite && overwrite.toLowerCase() === "true") {
       // when overwriting, clear out the existing custom_data object to remove obsolete values
       delete uiAttributesCurrent.custom_data;
-      uiAttributesMerged = _.merge(uiAttributesCurrent, uiAttributesCommon, uiAttributesOverrides);
+      uiAttributesMerged = merge(uiAttributesCurrent, uiAttributesCommonJson, uiAttributesEnvJson);
     } else {
-      uiAttributesMerged = _.merge({}, uiAttributesCommon, uiAttributesOverrides, uiAttributesCurrent);
+      uiAttributesMerged = merge({}, uiAttributesCommonJson, uiAttributesEnvJson, uiAttributesCurrent);
     }
     const trskillsMerged = tr_current
       ? tr_current.concat(taskrouter_skills).unique()
@@ -101,7 +116,8 @@ async function deployConfigurationData({ auth, environment, overwrite }) {
     });
 
 
-    console.log("Configuration updated: (following output formatted for readability)");
+    console.log("Configuration updated.");
+    console.log("");
 
     var readableFeatures = []
     Object.entries(configurationUpdated.ui_attributes.custom_data.features).forEach( feature => {
@@ -110,14 +126,17 @@ async function deployConfigurationData({ auth, environment, overwrite }) {
     var readableAttributes = configurationUpdated.ui_attributes;
     readableAttributes.custom_data.features = readableFeatures;
 
-    console.log("UI Attributes");
-    console.dir(readableAttributes, { depth: null });
-    console.log("TaskRouter Skills:");
+    console.log("### UI attributes (reduced for readability):");
+    console.log("```");
+    console.dir(readableAttributes.custom_data, { depth: null });
+    console.log("```");
+    console.log("");
+    console.log("### TaskRouter skills:");
     configurationUpdated.taskrouter_skills.forEach(element => {
-      console.log(`\t${element.name}`);
+      console.log(`- ${element.name}`);
     })
   } catch (error) {
-    console.error("error caught", error);
+    console.error("Error caught:", error);
     console.log("Auth", error.config?.auth);
     console.log("Data", error.response?.data);
   }
@@ -128,8 +147,8 @@ async function getConfiguration({ auth }) {
     method: "get",
     url: "https://flex-api.twilio.com/v1/Configuration",
     auth: {
-      username: auth.TWILIO_API_KEY,
-      password: auth.TWILIO_API_SECRET,
+      username: auth.apiKey,
+      password: auth.apiSecret,
     },
   }).then((response) => response.data);
 }
@@ -139,12 +158,12 @@ async function setConfiguration({ auth, configurationChanges }) {
     method: "post",
     url: "https://flex-api.twilio.com/v1/Configuration",
     auth: {
-      username: auth.TWILIO_API_KEY,
-      password: auth.TWILIO_API_SECRET,
+      username: auth.apiKey,
+      password: auth.apiSecret,
     },
     data: {
       ...configurationChanges,
-      account_sid: auth.TWILIO_ACCOUNT_SID,
+      account_sid: auth.accountSid,
     },
   }).then((response) => response.data);
 }

flex-config/package.json

@@ -2,9 +2,9 @@
   "name": "flex-configuration",
   "version": "1.0.0",
   "description": "CLI tool for merging new data into Flex configuration",
-  "main": "index.js",
+  "main": "index.mjs",
   "scripts": {
-    "deploy": "node index.js"
+    "deploy": "node index.mjs"
   },
   "author": "",
   "license": "ISC",

flex-config/ui_attributes.common.json

@@ -5,6 +5,7 @@
     }
   },
   "custom_data": {
+    "serverless_functions_domain": "<YOUR_SERVERLESS_DOMAIN>",
     "common": {
       "log_level": "info",
       "teams": [
@@ -146,7 +147,8 @@
         "enabled": true
       },
       "schedule_manager": {
-        "enabled": true
+        "enabled": true,
+        "serverless_domain": "<YOUR_SCHEDULE_MANAGER_DOMAIN>"
       },
       "multi_call": {
         "enabled": false

flex-config/ui_attributes.example.json

@@ -1,14 +1,6 @@
 {
   "custom_data": {
-    "serverless_functions_domain": "<YOUR_SERVERLESS_DOMAIN>",
     "features": {
-      "activity_skill_filter": {
-        "rules": {
-        }
-      },
-      "schedule_manager": {
-        "serverless_domain": "<YOUR_SCHEDULE_MANAGER_DOMAIN>"
-      }
     }
   }
 }

scripts/common/fill-replacements.mjs

@@ -4,6 +4,14 @@ import shell from 'shelljs';
 import { placeholderPrefix, varNameMapping } from "./constants.mjs";
 import * as fetchCli from "./fetch-cli.mjs";
 
+const parseData = (data) => {
+  let result = {};
+  for (const match of data.matchAll(new RegExp(`<${placeholderPrefix}_(.*)>`, 'g'))) {
+    result[match[1]] = match[0];
+  }
+  return result;
+}
+
 // Initialize env file if necessary, then parse its contents
 const readEnv = async (envFile, exampleFile, overwrite) => {
   if (overwrite || !shell.test('-e', envFile)) {
@@ -18,11 +26,7 @@ const readEnv = async (envFile, exampleFile, overwrite) => {
 
   // read and parse the env file
   const initialEnv = await fs.readFile(envFile, "utf8");
-  let result = {};
-  for (const match of initialEnv.matchAll(new RegExp(`<${placeholderPrefix}_(.*)>`, 'g'))) {
-    result[match[1]] = match[0];
-  }
-  return result;
+  return parseData(initialEnv);
 }
 
 // Fills placeholder variables from process.env if present
@@ -144,6 +148,24 @@ const fillAccountVars = (envVars, account) => {
   return envVars;
 }
 
+const fillAllVars = (envVars, account, environment) => {
+  try {
+    // Fill known env vars from process.env
+    envVars = fillKnownEnvVars(envVars);
+
+    // Fill known account vars
+    envVars = fillAccountVars(envVars, account);
+  } catch (error) {
+    console.error('Error fetching variables', error);
+    return null;
+  }
+
+  // Fetch unknown env vars from the API
+  envVars = fillUnknownEnvVars(envVars, environment);
+
+  return envVars;
+}
+
 const saveReplacements = async (data, path) => {
   try {
     for (const key in data) {
@@ -154,39 +176,62 @@ const saveReplacements = async (data, path) => {
   }
 }
 
-export default async (path, examplePath, account, environment, overwrite) => {
-  console.log(`Setting up ${path}...`);
+// Function to use for generating a populated string from a string containing placeholders
+export const fillReplacementsForString = async (data, account, environment) => {
+  // Parse out the env vars from the string
+  let envVars = parseData(data);
+
+  if (!envVars) {
+    console.error(`Error parsing data`, error);
+    return null;
+  }
+
+  // Fill the envVars with the appropriate replacements
+  envVars = fillAllVars(envVars, account, environment);
+
+  if (!envVars) {
+    return null;
+  }
+
+  // Replace the placeholders with the filled vars
+  let newData = data;
+  for (const key in envVars) {
+    newData = newData.replace(new RegExp(`<${placeholderPrefix}_${key}>`, 'g'), envVars[key]);
+  }
 
+  return {
+    data: newData,
+    envVars,
+  };
+}
+
+// Function to use for generating a populated file based on an example file containing placeholders
+export const fillReplacementsForPath = async (path, examplePath, account, environment, overwrite) => {
   // Check if this package uses environment files
   if (!shell.test('-e', examplePath) && !shell.test('-e', path)) {
     // No environment files, no need to continue
     return null;
   }
 
-  // Initialize the env vars
+  // Parse out the env vars from the file
   let envVars = await readEnv(path, examplePath, overwrite);
 
   if (!envVars) {
     console.error(`Unable to create the file ${path}.`);
     return null;
   }
 
-  try {
-    // Fill known env vars from process.env
-    envVars = fillKnownEnvVars(envVars);
-
-    // Fill known account vars
-    envVars = fillAccountVars(envVars, account);
-  } catch (error) {
-    console.error('Error fetching variables', error);
+  // Fill the envVars with the appropriate replacements
+  envVars = fillAllVars(envVars, account, environment);
+
+  if (!envVars) {
     return null;
   }
 
-  // Fetch unknown env vars from the API
-  envVars = fillUnknownEnvVars(envVars, environment);
-
   // Save!
   await saveReplacements(envVars, path);
 
   return envVars;
-}
+}
+
+export default fillReplacementsForPath;