From 738457f10d6f6f2cd87d0848084eb7c4b3a78183 Mon Sep 17 00:00:00 2001 From: Piotr Szeremeta Date: Tue, 2 Jan 2024 23:30:53 +0100 Subject: [PATCH] [ENG-9500] Allow calling functions with arguments in template files --- .../steps/functions/__tests__/cache.test.ts | 136 ++++++++++++++ packages/steps/build.sh | 1 + packages/steps/package.json | 1 + packages/steps/src/BuildConfig.ts | 166 ++++++++++-------- packages/steps/src/BuildConfigParser.ts | 2 +- packages/steps/src/BuildStep.ts | 18 +- packages/steps/src/BuildStepInput.ts | 60 ++++++- packages/steps/src/BuildWorkflowValidator.ts | 23 +++ .../steps/src/__tests__/BuildConfig-test.ts | 22 ++- .../src/__tests__/BuildConfigParser-test.ts | 24 ++- .../steps/src/__tests__/BuildStep-test.ts | 7 + .../src/__tests__/BuildStepInput-test.ts | 88 +++++++++- .../__tests__/BuildWorkflowValidator-test.ts | 45 ++++- .../steps/src/__tests__/fixtures/build.yml | 5 + .../steps/src/__tests__/fixtures/inputs.yml | 1 + packages/steps/src/inputFunctions.ts | 49 ++++++ .../src/utils/__tests__/template-test.ts | 35 ++++ packages/steps/src/utils/template.ts | 54 ++++++ 18 files changed, 631 insertions(+), 106 deletions(-) create mode 100644 packages/build-tools/src/steps/functions/__tests__/cache.test.ts create mode 100644 packages/steps/src/inputFunctions.ts diff --git a/packages/build-tools/src/steps/functions/__tests__/cache.test.ts b/packages/build-tools/src/steps/functions/__tests__/cache.test.ts new file mode 100644 index 000000000..e57d737df --- /dev/null +++ b/packages/build-tools/src/steps/functions/__tests__/cache.test.ts @@ -0,0 +1,136 @@ +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; + +import { + BuildRuntimePlatform, + BuildStepGlobalContext, + ExternalBuildContextProvider, + CacheManager, +} from '@expo/steps'; +import { anything, capture, instance, mock, reset, verify, when } from 'ts-mockito'; + +import { createLogger } from '../../../__mocks__/@expo/logger'; +import { createRestoreCacheBuildFunction, createSaveCacheBuildFunction } from '../cache'; + +const cacheSaveBuildFunction = createSaveCacheBuildFunction(); +const cacheRestoreBuildFunction = createRestoreCacheBuildFunction(); + +const providerMock = mock(); +const cacheManagerMock = mock(); + +const cacheManager = instance(cacheManagerMock); +const initialCache = { downloadUrls: {} }; + +const provider = instance(providerMock); + +let ctx: BuildStepGlobalContext; + +const existingKey = + 'c7d8e33243968f8675ec0463ad89e11c1e754723695ab9b23dfb8f9ddd389a28-value-8b6e2366e2a2ff8b43556a1dcc5f1cf97ddcf4cdf3c8f9a6d54e0efe2e747922'; + +describe('cache functions', () => { + let key: string; + let paths: string[]; + beforeEach(async () => { + key = '${ hashFiles("./src/*") }-value'; + paths = ['path1', 'path2']; + reset(cacheManagerMock); + reset(providerMock); + + const projectSourceDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'project-')); + when(providerMock.logger).thenReturn(createLogger()); + when(providerMock.runtimePlatform).thenReturn(BuildRuntimePlatform.LINUX); + when(providerMock.staticContext()).thenReturn({ some: 'key', job: { cache: initialCache } }); + when(providerMock.cacheManager).thenReturn(cacheManager); + when(providerMock.projectSourceDirectory).thenReturn(projectSourceDirectory); + when(providerMock.defaultWorkingDirectory).thenReturn(projectSourceDirectory); + when(providerMock.projectTargetDirectory).thenReturn(projectSourceDirectory); + + ctx = new BuildStepGlobalContext(provider, false); + + await fs.mkdir(path.join(projectSourceDirectory, 'src')); + await fs.writeFile(path.join(projectSourceDirectory, 'src', 'path1'), 'placeholder'); + await fs.writeFile(path.join(projectSourceDirectory, 'src', 'path2'), 'placeholder'); + }); + + describe('cacheRestoreBuildFunction', () => { + test('has correct identifiers', () => { + expect(cacheRestoreBuildFunction.id).toBe('restore-cache'); + expect(cacheRestoreBuildFunction.namespace).toBe('eas'); + expect(cacheRestoreBuildFunction.name).toBe('Restore Cache'); + }); + + test('restores cache if it exists', async () => { + when(cacheManagerMock.restoreCache(anything(), anything())); + initialCache.downloadUrls = { [existingKey]: 'url' }; + + const buildStep = cacheRestoreBuildFunction.createBuildStepFromFunctionCall(ctx, { + callInputs: { key, paths }, + }); + + when(providerMock.defaultWorkingDirectory).thenReturn('/tmp'); + + await buildStep.executeAsync(); + + verify(cacheManagerMock.restoreCache(anything(), anything())).once(); + + const [, cache] = capture(cacheManagerMock.restoreCache).first(); + expect(cache.key).toMatch(/^\w+-value/); + expect(cache.paths).toStrictEqual(paths); + }); + + test("doesn't restore cache if it doesn't exist", async () => { + when(cacheManagerMock.restoreCache(anything(), anything())); + initialCache.downloadUrls = { invalidkey: 'url' }; + + const buildStep = cacheRestoreBuildFunction.createBuildStepFromFunctionCall(ctx, { + callInputs: { key, paths }, + }); + + await buildStep.executeAsync(); + + verify(cacheManagerMock.restoreCache(anything(), anything())).never(); + }); + }); + + describe('cacheSaveBuildFunction', () => { + test('has correct identifiers', () => { + expect(cacheSaveBuildFunction.id).toBe('save-cache'); + expect(cacheSaveBuildFunction.namespace).toBe('eas'); + expect(cacheSaveBuildFunction.name).toBe('Save Cache'); + }); + + test('saves cache if it does not exist', async () => { + when(cacheManagerMock.restoreCache(anything(), anything())); + + initialCache.downloadUrls = {}; + + const buildStep = cacheSaveBuildFunction.createBuildStepFromFunctionCall(ctx, { + callInputs: { key, paths }, + }); + + await buildStep.executeAsync(); + + verify(cacheManagerMock.saveCache(anything(), anything())).once(); + + const [, cache] = capture(cacheManagerMock.saveCache).first(); + expect(cache?.key).toMatch(/^\w+-value/); + expect(cache?.paths).toStrictEqual(paths); + }); + + test("doesn't save cache if it exists", async () => { + when(cacheManagerMock.restoreCache(anything(), anything())); + + initialCache.downloadUrls = { [existingKey]: 'url' }; + + const buildStep = cacheSaveBuildFunction.createBuildStepFromFunctionCall(ctx, { + callInputs: { key, paths }, + }); + + await buildStep.executeAsync(); + + verify(cacheManagerMock.saveCache(anything(), anything())).never(); + }); + }); +}); diff --git a/packages/steps/build.sh b/packages/steps/build.sh index 13294e440..eefc4996c 100755 --- a/packages/steps/build.sh +++ b/packages/steps/build.sh @@ -11,6 +11,7 @@ if [[ "$npm_lifecycle_event" == "prepack" ]]; then echo 'Removing "dist_commonjs" and "dist_esm" folders...' rm -rf dist_commonjs dist_esm fi +rm -rf dist_commonjs dist_esm echo 'Compiling TypeScript to JavaScript...' node_modules/.bin/tsc --project tsconfig.build.json diff --git a/packages/steps/package.json b/packages/steps/package.json index 083b63fe4..05a6350f7 100644 --- a/packages/steps/package.json +++ b/packages/steps/package.json @@ -32,6 +32,7 @@ "license": "BUSL-1.1", "devDependencies": { "@jest/globals": "^29.6.2", + "@types/glob": "^8.1.0", "@types/jest": "^29.5.3", "@types/lodash.clonedeep": "^4.5.7", "@types/lodash.get": "^4.4.7", diff --git a/packages/steps/src/BuildConfig.ts b/packages/steps/src/BuildConfig.ts index c478b4dd4..62da66e35 100644 --- a/packages/steps/src/BuildConfig.ts +++ b/packages/steps/src/BuildConfig.ts @@ -5,15 +5,15 @@ import path from 'path'; import Joi from 'joi'; import YAML from 'yaml'; -import { BuildConfigError, BuildWorkflowError } from './errors.js'; -import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; import { BuildFunction } from './BuildFunction.js'; +import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; +import { BuildStepEnv } from './BuildStepEnv.js'; import { - BuildStepInputValueTypeWithRequired, - BuildStepInputValueTypeName, BuildStepInputValueType, + BuildStepInputValueTypeName, + BuildStepInputValueTypeWithRequired, } from './BuildStepInput.js'; -import { BuildStepEnv } from './BuildStepEnv.js'; +import { BuildConfigError, BuildWorkflowError } from './errors.js'; import { BUILD_STEP_IF_CONDITION_EXPRESSION_REGEXP, BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, @@ -96,82 +96,86 @@ const BuildFunctionInputsSchema = Joi.array().items( Joi.alternatives().conditional(Joi.ref('.'), { is: Joi.string(), then: Joi.string().required(), - otherwise: Joi.object({ - name: Joi.string().required(), - defaultValue: Joi.when('allowedValues', { - is: Joi.exist(), - then: Joi.valid(Joi.in('allowedValues')).messages({ - 'any.only': '{{#label}} must be one of allowed values', - }), - }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.STRING, - then: Joi.string().allow(''), - }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.BOOLEAN, - then: Joi.alternatives( - Joi.boolean(), - Joi.string().pattern( - BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, - 'context or output reference regex pattern' - ) - ).messages({ - 'alternatives.types': - '{{#label}} must be a boolean or reference to output or context value', + otherwise: Joi.alternatives().conditional(Joi.ref('.'), { + is: Joi.array(), + then: Joi.array().required(), + otherwise: Joi.object({ + name: Joi.string().required(), + defaultValue: Joi.when('allowedValues', { + is: Joi.exist(), + then: Joi.valid(Joi.in('allowedValues')).messages({ + 'any.only': '{{#label}} must be one of allowed values', }), }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.NUMBER, - then: Joi.alternatives( - Joi.number(), - Joi.string().pattern( - BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, - 'context or output reference regex pattern' - ) - ).messages({ - 'alternatives.types': - '{{#label}} must be a number or reference to output or context value', + .when('allowedValueType', { + is: BuildStepInputValueTypeName.STRING, + then: Joi.string().allow(''), + }) + .when('allowedValueType', { + is: BuildStepInputValueTypeName.BOOLEAN, + then: Joi.alternatives( + Joi.boolean(), + Joi.string().pattern( + BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, + 'context or output reference regex pattern' + ) + ).messages({ + 'alternatives.types': + '{{#label}} must be a boolean or reference to output or context value', + }), + }) + .when('allowedValueType', { + is: BuildStepInputValueTypeName.NUMBER, + then: Joi.alternatives( + Joi.number(), + Joi.string().pattern( + BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, + 'context or output reference regex pattern' + ) + ).messages({ + 'alternatives.types': + '{{#label}} must be a number or reference to output or context value', + }), + }) + .when('allowedValueType', { + is: BuildStepInputValueTypeName.JSON, + then: Joi.alternatives( + Joi.object(), + Joi.string().pattern( + BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, + 'context or output reference regex pattern' + ) + ).messages({ + 'alternatives.types': + '{{#label}} must be a object or reference to output or context value', + }), }), + allowedValues: Joi.when('allowedValueType', { + is: BuildStepInputValueTypeName.STRING, + then: Joi.array().items(Joi.string().allow('')), }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.JSON, - then: Joi.alternatives( - Joi.object(), - Joi.string().pattern( - BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, - 'context or output reference regex pattern' - ) - ).messages({ - 'alternatives.types': - '{{#label}} must be a object or reference to output or context value', + .when('allowedValueType', { + is: BuildStepInputValueTypeName.BOOLEAN, + then: Joi.array().items(Joi.boolean()), + }) + .when('allowedValueType', { + is: BuildStepInputValueTypeName.NUMBER, + then: Joi.array().items(Joi.number()), + }) + .when('allowedValueType', { + is: BuildStepInputValueTypeName.JSON, + then: Joi.array().items(Joi.object()), }), - }), - allowedValues: Joi.when('allowedValueType', { - is: BuildStepInputValueTypeName.STRING, - then: Joi.array().items(Joi.string().allow('')), + allowedValueType: Joi.string() + .valid(...Object.values(BuildStepInputValueTypeName)) + .default(BuildStepInputValueTypeName.STRING), + required: Joi.boolean(), }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.BOOLEAN, - then: Joi.array().items(Joi.boolean()), - }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.NUMBER, - then: Joi.array().items(Joi.number()), - }) - .when('allowedValueType', { - is: BuildStepInputValueTypeName.JSON, - then: Joi.array().items(Joi.object()), - }), - allowedValueType: Joi.string() - .valid(...Object.values(BuildStepInputValueTypeName)) - .default(BuildStepInputValueTypeName.STRING), - required: Joi.boolean(), - }) - .rename('allowed_values', 'allowedValues') - .rename('default_value', 'defaultValue') - .rename('type', 'allowedValueType') - .required(), + .rename('allowed_values', 'allowedValues') + .rename('default_value', 'defaultValue') + .rename('type', 'allowedValueType') + .required(), + }), }) ); @@ -189,7 +193,13 @@ const BuildFunctionCallSchema = Joi.object({ id: Joi.string(), inputs: Joi.object().pattern( Joi.string(), - Joi.alternatives().try(Joi.string().allow(''), Joi.boolean(), Joi.number(), Joi.object()) + Joi.alternatives().try( + Joi.string().allow(''), + Joi.boolean(), + Joi.number(), + Joi.object(), + Joi.array().items(Joi.string().required()) + ) ), name: Joi.string(), workingDirectory: Joi.string(), @@ -399,11 +409,11 @@ export function mergeConfigWithImportedFunctions( } export function isBuildStepCommandRun(step: BuildStepConfig): step is BuildStepCommandRun { - return Boolean(step) && typeof step === 'object' && typeof step.run === 'object'; + return Boolean(step) && typeof step === 'object' && 'run' in step && typeof step.run === 'object'; } export function isBuildStepBareCommandRun(step: BuildStepConfig): step is BuildStepBareCommandRun { - return Boolean(step) && typeof step === 'object' && typeof step.run === 'string'; + return Boolean(step) && typeof step === 'object' && 'run' in step && typeof step.run === 'string'; } export function isBuildStepFunctionCall(step: BuildStepConfig): step is BuildStepFunctionCall { diff --git a/packages/steps/src/BuildConfigParser.ts b/packages/steps/src/BuildConfigParser.ts index 9ec20c0e5..6046ef898 100644 --- a/packages/steps/src/BuildConfigParser.ts +++ b/packages/steps/src/BuildConfigParser.ts @@ -19,12 +19,12 @@ import { } from './BuildConfig.js'; import { BuildFunction, BuildFunctionById } from './BuildFunction.js'; import { BuildStep } from './BuildStep.js'; +import { BuildStepGlobalContext } from './BuildStepContext.js'; import { BuildStepInput, BuildStepInputProvider, BuildStepInputValueTypeName, } from './BuildStepInput.js'; -import { BuildStepGlobalContext } from './BuildStepContext.js'; import { BuildStepOutput, BuildStepOutputProvider } from './BuildStepOutput.js'; import { BuildWorkflow } from './BuildWorkflow.js'; import { BuildWorkflowValidator } from './BuildWorkflowValidator.js'; diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 7a7e35e79..ad95dc779 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -4,7 +4,9 @@ import path from 'path'; import { v4 as uuidv4 } from 'uuid'; +import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; import { BuildStepContext, BuildStepGlobalContext } from './BuildStepContext.js'; +import { BuildStepEnv } from './BuildStepEnv.js'; import { BuildStepInput, BuildStepInputById, makeBuildStepInputByIdMap } from './BuildStepInput.js'; import { BuildStepOutput, @@ -12,22 +14,20 @@ import { SerializedBuildStepOutput, makeBuildStepOutputByIdMap, } from './BuildStepOutput.js'; -import { BIN_PATH } from './utils/shell/bin.js'; -import { getDefaultShell, getShellCommandAndArgs } from './utils/shell/command.js'; import { cleanUpStepTemporaryDirectoriesAsync, createTemporaryEnvsDirectoryAsync, createTemporaryOutputsDirectoryAsync, saveScriptToTemporaryFileAsync, } from './BuildTemporaryFiles.js'; +import { BuildStepRuntimeError } from './errors.js'; +import { BIN_PATH } from './utils/shell/bin.js'; +import { getDefaultShell, getShellCommandAndArgs } from './utils/shell/command.js'; import { spawnAsync } from './utils/shell/spawn.js'; import { getSelectedStatusCheckFromIfStatementTemplate, interpolateWithInputs, } from './utils/template.js'; -import { BuildStepRuntimeError } from './errors.js'; -import { BuildStepEnv } from './BuildStepEnv.js'; -import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; export enum BuildStepStatus { NEW = 'new', @@ -229,6 +229,12 @@ export class BuildStep extends BuildStepOutputAccessor { ctx.registerStep(this); } + private async prepareInputsAsync(): Promise { + if (this.inputs !== undefined) { + await Promise.all(this.inputs.map((input) => input.prepareValueAsync())); + } + } + public async executeAsync(): Promise { try { this.ctx.logger.info( @@ -237,6 +243,8 @@ export class BuildStep extends BuildStepOutputAccessor { ); this.status = BuildStepStatus.IN_PROGRESS; + await this.prepareInputsAsync(); + if (this.command !== undefined) { await this.executeCommandAsync(); } else { diff --git a/packages/steps/src/BuildStepInput.ts b/packages/steps/src/BuildStepInput.ts index ecaec65f0..9ad3ffbf0 100644 --- a/packages/steps/src/BuildStepInput.ts +++ b/packages/steps/src/BuildStepInput.ts @@ -1,9 +1,13 @@ import { bunyan } from '@expo/logger'; import { BuildStepGlobalContext, SerializedBuildStepGlobalContext } from './BuildStepContext.js'; -import { BuildStepRuntimeError } from './errors.js'; +import { BuildConfigError, BuildStepRuntimeError } from './errors.js'; +import callInputFunctionAsync from './inputFunctions.js'; import { + BUILD_STEP_FUNCTION_EXPRESSION_REGEXP, BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX, + iterateWithFunctions, + interpolateWithFunctionsAsync, interpolateWithOutputs, } from './utils/template.js'; @@ -77,6 +81,7 @@ export class BuildStepInput< public readonly required: R; private _value?: BuildStepInputValueType; + private _computedValue?: BuildStepInputValueType; public static createProvider(params: BuildStepInputProviderParams): BuildStepInputProvider { return (ctx, stepDisplayName) => new BuildStepInput(ctx, { ...params, stepDisplayName }); @@ -101,20 +106,59 @@ export class BuildStepInput< this.allowedValueTypeName = allowedValueTypeName; } - public get value(): BuildStepInputValueTypeWithRequired { + public isFunctionCall(): boolean { + return this.rawValue?.toString().match(BUILD_STEP_FUNCTION_EXPRESSION_REGEXP) !== null; + } + + private requiresInterpolation(rawValue: any): rawValue is string { + return typeof rawValue === 'string'; + } + + public validateFunctions( + fn: (error: string | null, f: string | null, args: string[]) => any + ): BuildConfigError[] { + const rawValue = this._value ?? this.defaultValue; + const errors = []; + if (this.requiresInterpolation(rawValue) && this.isFunctionCall()) { + try { + iterateWithFunctions(rawValue, (fun, args) => { + const error = fn(null, fun, args); + if (error) { + errors.push(error); + } + }); + } catch (e) { + if (e instanceof BuildConfigError) { + errors.push(fn(e.message, null, [])); + } else { + throw e; + } + } + } + return errors; + } + + public async prepareValueAsync(): Promise { const rawValue = this._value ?? this.defaultValue; + if (this.requiresInterpolation(rawValue) && this.isFunctionCall()) { + this._computedValue = (await interpolateWithFunctionsAsync( + rawValue, + (fn: string, args: string[]) => { + return callInputFunctionAsync(fn, args, this.ctx); + } + )) as BuildStepInputValueType; + } + } + + public get value(): BuildStepInputValueTypeWithRequired { + const rawValue = this._computedValue ?? this._value ?? this.defaultValue; if (this.required && rawValue === undefined) { throw new BuildStepRuntimeError( `Input parameter "${this.id}" for step "${this.stepDisplayName}" is required but it was not set.` ); } - const valueDoesNotRequireInterpolation = - rawValue === undefined || - typeof rawValue === 'boolean' || - typeof rawValue === 'number' || - typeof rawValue === 'object'; - if (valueDoesNotRequireInterpolation) { + if (!this.requiresInterpolation(rawValue)) { const currentTypeName = typeof rawValue === 'object' ? BuildStepInputValueTypeName.JSON : typeof rawValue; if (currentTypeName !== this.allowedValueTypeName && rawValue !== undefined) { diff --git a/packages/steps/src/BuildWorkflowValidator.ts b/packages/steps/src/BuildWorkflowValidator.ts index 5711cc375..e6fb3bd16 100644 --- a/packages/steps/src/BuildWorkflowValidator.ts +++ b/packages/steps/src/BuildWorkflowValidator.ts @@ -6,6 +6,7 @@ import { BuildStep } from './BuildStep.js'; import { BuildStepInputValueTypeName } from './BuildStepInput.js'; import { BuildWorkflow } from './BuildWorkflow.js'; import { BuildConfigError, BuildWorkflowError } from './errors.js'; +import * as inputFunctions from './inputFunctions.js'; import { duplicates } from './utils/expodash/duplicates.js'; import { nullthrows } from './utils/nullthrows.js'; import { findOutputPaths } from './utils/template.js'; @@ -74,6 +75,28 @@ export class BuildWorkflowValidator { errors.push(error); } + if (currentStepInput.isFunctionCall()) { + errors.push( + ...currentStepInput.validateFunctions((error, fn, args) => { + if (error) { + return new BuildConfigError( + `Input parameter "${currentStepInput.id}" for step "${currentStep.displayName}" ${error}.` + ); + } + if (fn && fn in inputFunctions) { + return null; + } + return new BuildConfigError( + `Input parameter "${currentStepInput.id}" for step "${ + currentStep.displayName + }" is set to "\${ ${fn}(${args + .map((i) => `"${i}"`) + .join(',')}) }" which is not a valid build-in function name.` + ); + }) + ); + } + if (currentStepInput.defaultValue === undefined) { continue; } diff --git a/packages/steps/src/__tests__/BuildConfig-test.ts b/packages/steps/src/__tests__/BuildConfig-test.ts index c166390b9..ee7938634 100644 --- a/packages/steps/src/__tests__/BuildConfig-test.ts +++ b/packages/steps/src/__tests__/BuildConfig-test.ts @@ -3,6 +3,10 @@ import path from 'path'; import url from 'url'; import { + BuildConfig, + BuildConfigSchema, + BuildFunctions, + BuildFunctionsConfigFileSchema, BuildStepBareCommandRun, BuildStepBareFunctionCall, BuildStepCommandRun, @@ -11,16 +15,12 @@ import { isBuildStepBareFunctionCall, isBuildStepCommandRun, isBuildStepFunctionCall, - readRawBuildConfigAsync, - readAndValidateBuildConfigAsync, - validateConfig, - BuildFunctionsConfigFileSchema, - BuildConfigSchema, - validateAllFunctionsExist, - BuildConfig, mergeConfigWithImportedFunctions, - BuildFunctions, + readAndValidateBuildConfigAsync, readAndValidateBuildFunctionsConfigFileAsync, + readRawBuildConfigAsync, + validateAllFunctionsExist, + validateConfig, } from '../BuildConfig.js'; import { BuildConfigError, BuildConfigYAMLError } from '../errors.js'; @@ -33,7 +33,7 @@ describe(readAndValidateBuildConfigAsync, () => { const config = await readAndValidateBuildConfigAsync( path.join(__dirname, './fixtures/build.yml'), { - externalFunctionIds: [], + externalFunctionIds: ['eas/save-cache'], } ); expect(typeof config).toBe('object'); @@ -44,6 +44,10 @@ describe(readAndValidateBuildConfigAsync, () => { expect(config.build.steps[2].run.env).toMatchObject({ FOO: 'bar', BAR: 'baz' }); assert(isBuildStepCommandRun(config.build.steps[5])); expect(config.build.steps[5].run.if).toBe('${ always() }'); + assert(isBuildStepFunctionCall(config.build.steps[6])); + expect(config.build.steps[6]).toMatchObject({ + 'eas/save-cache': { inputs: { key: 'cache-key', paths: ['src'] } }, + }); }); test('valid custom build config with imports', async () => { const config = await readAndValidateBuildConfigAsync( diff --git a/packages/steps/src/__tests__/BuildConfigParser-test.ts b/packages/steps/src/__tests__/BuildConfigParser-test.ts index 3066f82d2..d7fa4e0ca 100644 --- a/packages/steps/src/__tests__/BuildConfigParser-test.ts +++ b/packages/steps/src/__tests__/BuildConfigParser-test.ts @@ -3,12 +3,12 @@ import url from 'url'; import { BuildConfigParser } from '../BuildConfigParser.js'; import { BuildFunction } from '../BuildFunction.js'; +import { BuildRuntimePlatform } from '../BuildRuntimePlatform.js'; import { BuildStepFunction } from '../BuildStep.js'; +import { BuildStepInputValueTypeName } from '../BuildStepInput.js'; import { BuildWorkflow } from '../BuildWorkflow.js'; import { BuildConfigError, BuildStepRuntimeError } from '../errors.js'; import { getDefaultShell } from '../utils/shell/command.js'; -import { BuildRuntimePlatform } from '../BuildRuntimePlatform.js'; -import { BuildStepInputValueTypeName } from '../BuildStepInput.js'; import { createGlobalContextMock } from './utils/context.js'; import { getError, getErrorAsync } from './utils/error.js'; @@ -53,6 +53,14 @@ describe(BuildConfigParser, () => { it('returns a BuildWorkflow object', async () => { const ctx = createGlobalContextMock(); const parser = new BuildConfigParser(ctx, { + externalFunctions: [ + new BuildFunction({ + namespace: 'eas', + id: 'save-cache', + name: 'Cache', + command: 'cache', + }), + ], configPath: path.join(__dirname, './fixtures/build.yml'), }); const result = await parser.parseAsync(); @@ -62,11 +70,19 @@ describe(BuildConfigParser, () => { it('parses steps from the build workflow', async () => { const ctx = createGlobalContextMock(); const parser = new BuildConfigParser(ctx, { + externalFunctions: [ + new BuildFunction({ + namespace: 'eas', + id: 'save-cache', + name: 'Cache', + command: 'cache', + }), + ], configPath: path.join(__dirname, './fixtures/build.yml'), }); const workflow = await parser.parseAsync(); const buildSteps = workflow.buildSteps; - expect(buildSteps.length).toBe(6); + expect(buildSteps.length).toBe(7); // - run: echo "Hi!" const step1 = buildSteps[0]; @@ -199,6 +215,8 @@ describe(BuildConfigParser, () => { property2: ['value2', { value3: { property3: 'value4' } }], }); expect(step1.inputs?.[4].allowedValueTypeName).toBe(BuildStepInputValueTypeName.JSON); + expect(step1.inputs?.[5].id).toBe('function_value'); + expect(step1.inputs?.[5].value).toBe("${ hashFiles('**/*.js') }"); }); it('parses outputs', async () => { diff --git a/packages/steps/src/__tests__/BuildStep-test.ts b/packages/steps/src/__tests__/BuildStep-test.ts index 1f5a29996..aebbf1a3b 100644 --- a/packages/steps/src/__tests__/BuildStep-test.ts +++ b/packages/steps/src/__tests__/BuildStep-test.ts @@ -602,6 +602,13 @@ describe(BuildStep, () => { allowedValueTypeName: BuildStepInputValueTypeName.JSON, required: true, }), + new BuildStepInput(baseStepCtx, { + id: 'foo6', + stepDisplayName: displayName, + defaultValue: "${ hashFiles('src') }-hello", + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + required: true, + }), ]; const outputs: BuildStepOutput[] = [ new BuildStepOutput(baseStepCtx, { diff --git a/packages/steps/src/__tests__/BuildStepInput-test.ts b/packages/steps/src/__tests__/BuildStepInput-test.ts index f06e01f50..4568cb407 100644 --- a/packages/steps/src/__tests__/BuildStepInput-test.ts +++ b/packages/steps/src/__tests__/BuildStepInput-test.ts @@ -1,10 +1,13 @@ -import { BuildStepRuntimeError } from '../errors.js'; +import fs from 'fs/promises'; +import path from 'path'; + import { BuildStep } from '../BuildStep.js'; import { BuildStepInput, BuildStepInputValueTypeName, makeBuildStepInputByIdMap, } from '../BuildStepInput.js'; +import { BuildStepRuntimeError } from '../errors.js'; import { createGlobalContextMock } from './utils/context.js'; import { createMockLogger } from './utils/logger.js'; @@ -255,6 +258,89 @@ describe(BuildStepInput, () => { ); }); + describe('function call', () => { + const ctx = createGlobalContextMock({ + relativeWorkingDirectory: '/tmp/workingDir', + projectSourceDirectory: '/tmp/projectDir', + }); + + beforeEach(async () => { + await fs.mkdir(ctx.defaultWorkingDirectory, { recursive: true }); + await fs.mkdir(ctx.projectSourceDirectory, { recursive: true }); + }); + afterEach(async () => { + await fs.rm(ctx.defaultWorkingDirectory, { recursive: true }); + await fs.rm(ctx.projectSourceDirectory, { recursive: true }); + }); + + test('hashFiles', async () => { + await fs.mkdir(path.join(ctx.defaultWorkingDirectory, 'directoryToHash'), { + recursive: true, + }); + await fs.mkdir(path.join(ctx.defaultWorkingDirectory, 'directoryToSkip'), { + recursive: true, + }); + + await Promise.all([ + fs.writeFile(path.join(ctx.defaultWorkingDirectory, 'filesToHash.ts'), 'lorem ipsum'), + fs.writeFile( + path.join(ctx.defaultWorkingDirectory, 'directoryToHash', 'file1'), + 'lorem ipsum' + ), + fs.writeFile( + path.join(ctx.defaultWorkingDirectory, 'directoryToHash', 'file2'), + 'lorem ipsum' + ), + fs.writeFile( + path.join(ctx.defaultWorkingDirectory, 'directoryToSkip', 'file3'), + 'lorem ipsum' + ), + ]); + + const i = new BuildStepInput(ctx, { + id: 'foo', + stepDisplayName: BuildStep.getDisplayName({ id: 'test1' }), + defaultValue: 'foo-${ hashFiles("./filesToHash.ts", "./directoryToHash/*") }-bar', + required: true, + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + }); + await i.prepareValueAsync(); + + const hash1 = i.value; + + await fs.writeFile(path.join(ctx.defaultWorkingDirectory, 'filesToHash.ts'), 'lorem ipsum 2'); + + await i.prepareValueAsync(); + + const hash2 = i.value; + + await fs.writeFile( + path.join(ctx.defaultWorkingDirectory, 'directoryToHash', 'file1'), + 'lorem ipsum 1' + ); + + await i.prepareValueAsync(); + const hash3 = i.value; + + await fs.writeFile( + path.join(ctx.defaultWorkingDirectory, 'directoryToSkip', 'file2'), + 'lorem ipsum 2' + ); + await i.prepareValueAsync(); + + const hash4 = i.value; + + expect(hash1).toMatch(/^foo-.+-bar$/); + expect(hash2).toMatch(/^foo-.+-bar$/); + expect(hash3).toMatch(/^foo-.+-bar$/); + expect(hash4).toMatch(/^foo-.+-bar$/); + + expect(hash1).not.toBe(hash2); + expect(hash2).not.toBe(hash3); + expect(hash3).toBe(hash4); + }); + }); + test('invalid context value type JSON', () => { const ctx = createGlobalContextMock({ staticContextContent: { diff --git a/packages/steps/src/__tests__/BuildWorkflowValidator-test.ts b/packages/steps/src/__tests__/BuildWorkflowValidator-test.ts index 251512437..5ed5d533f 100644 --- a/packages/steps/src/__tests__/BuildWorkflowValidator-test.ts +++ b/packages/steps/src/__tests__/BuildWorkflowValidator-test.ts @@ -39,7 +39,7 @@ describe(BuildWorkflowValidator, () => { }), new BuildStep(ctx, { id: 'test3', - displayName: BuildStep.getDisplayName({ id: 'test3', command: 'echo 456' }), + displayName: BuildStep.getDisplayName({ id: 'test2', command: 'echo 456' }), command: 'echo 456', }), ], @@ -179,6 +179,16 @@ describe(BuildWorkflowValidator, () => { allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, required: true, }), + BuildStepInput.createProvider({ + id: 'id7', + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + required: true, + }), + BuildStepInput.createProvider({ + id: 'id8', + allowedValueTypeName: BuildStepInputValueTypeName.STRING, + required: true, + }), ], command: 'echo "hi"', }); @@ -197,6 +207,8 @@ describe(BuildWorkflowValidator, () => { id4: '${ steps.step_id.output1 }', id5: '${ eas.job.version.buildNumber }', id6: '${ wrong.aaa }', + id7: '${ invalidFunction("foo") }', + id8: '${ hashFiles("foo) }', }, }), ], @@ -222,6 +234,12 @@ describe(BuildWorkflowValidator, () => { expect((error as BuildWorkflowError).errors[3].message).toBe( 'Input parameter "id6" for step "step_id" is set to "${ wrong.aaa }" which is not of type "number" or is not step or context reference.' ); + expect((error as BuildWorkflowError).errors[4].message).toBe( + 'Input parameter "id7" for step "step_id" is set to "${ invalidFunction("foo") }" which is not a valid build-in function name.' + ); + expect((error as BuildWorkflowError).errors[5].message).toBe( + 'Input parameter "id8" for step "step_id" contains syntax error in "${ hashFiles("foo) }".' + ); }); test('output from future step', async () => { const ctx = createGlobalContextMock(); @@ -519,4 +537,29 @@ describe(BuildWorkflowValidator, () => { `Custom function module path "/non/existent/module" for function "test" does not exist.` ); }); + + test('non-existing function module', async () => { + const ctx = createGlobalContextMock({ runtimePlatform: BuildRuntimePlatform.LINUX }); + const workflow = new BuildWorkflow(ctx, { + buildSteps: [], + buildFunctions: { + test: new BuildFunction({ + id: 'test', + customFunctionModulePath: '/non/existent/module', + }), + }, + }); + + const validator = new BuildWorkflowValidator(workflow); + const error = await getErrorAsync(async () => { + await validator.validateAsync(); + }); + assert(error instanceof BuildWorkflowError); + expect(error).toBeInstanceOf(BuildWorkflowError); + expect(error.errors.length).toBe(1); + expect(error.errors[0]).toBeInstanceOf(BuildConfigError); + expect(error.errors[0].message).toBe( + `Custom function module path "/non/existent/module" for function "test" does not exist.` + ); + }); }); diff --git a/packages/steps/src/__tests__/fixtures/build.yml b/packages/steps/src/__tests__/fixtures/build.yml index bc81b4704..5bae10f5e 100644 --- a/packages/steps/src/__tests__/fixtures/build.yml +++ b/packages/steps/src/__tests__/fixtures/build.yml @@ -29,3 +29,8 @@ build: name: Use non-default shell shell: /nib/hsab command: echo 123 + - eas/save-cache: + inputs: + key: cache-key + paths: + - src \ No newline at end of file diff --git a/packages/steps/src/__tests__/fixtures/inputs.yml b/packages/steps/src/__tests__/fixtures/inputs.yml index 07623b480..213fe5495 100644 --- a/packages/steps/src/__tests__/fixtures/inputs.yml +++ b/packages/steps/src/__tests__/fixtures/inputs.yml @@ -14,4 +14,5 @@ build: - value2 - value3: property3: value4 + function_value: ${ hashFiles('**/*.js') } command: echo "Hi, ${ inputs.name }, ${ inputs.boolean_value }!" diff --git a/packages/steps/src/inputFunctions.ts b/packages/steps/src/inputFunctions.ts new file mode 100644 index 000000000..2121449dd --- /dev/null +++ b/packages/steps/src/inputFunctions.ts @@ -0,0 +1,49 @@ +import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; + +import fg from 'fast-glob'; + +import { BuildStepGlobalContext } from './BuildStepContext.js'; + +export async function hashFilesAsync( + ctx: BuildStepGlobalContext, + files: string[] +): Promise { + const hash = crypto.createHash('sha256'); + + await Promise.all( + files.map(async (file) => { + let filePath = path.join(ctx.defaultWorkingDirectory, file); + if (fs.existsSync(filePath) && fs.lstatSync(filePath).isDirectory()) { + filePath += '/**/*'; + } + const filePaths = await fg(filePath, { onlyFiles: true }); + return Promise.all( + filePaths.map(async (filePath) => { + const content = fs.readFileSync(filePath); + ctx.baseLogger.debug(`Hashing file ${filePath}`); + hash.update(path.relative(ctx.defaultWorkingDirectory, filePath)); + return hash.update(content); + }) + ); + }) + ); + + return hash.digest('hex'); +} + +export const hashFiles = hashFilesAsync; + +export default async function callInputFunctionAsync( + fnName: string, + args: any[], + ctx: BuildStepGlobalContext +): Promise { + switch (fnName) { + case 'hashFiles': + return await hashFilesAsync(ctx, args); + default: + throw new Error(`Unknown input function: ${fnName}`); + } +} diff --git a/packages/steps/src/utils/__tests__/template-test.ts b/packages/steps/src/utils/__tests__/template-test.ts index 20993312d..75810ecf9 100644 --- a/packages/steps/src/utils/__tests__/template-test.ts +++ b/packages/steps/src/utils/__tests__/template-test.ts @@ -6,6 +6,7 @@ import { interpolateWithGlobalContext, interpolateWithInputs, interpolateWithOutputs, + interpolateWithFunctionsAsync, parseOutputPath, } from '../template.js'; @@ -34,6 +35,40 @@ describe(interpolateWithOutputs, () => { }); }); +describe(interpolateWithFunctionsAsync, () => { + test('interpolation', async () => { + const nonArgs = await interpolateWithFunctionsAsync('foo${ noArgs() }bar', async (fn, args) => { + if (fn === 'noArgs' && args.length === 0) { + return 'ok'; + } + return '${fn} | ${args.join(", ")}'; + }); + const oneArg = await interpolateWithFunctionsAsync( + 'foo${ oneArg("src") }bar', + async (fn, args) => { + if (fn === 'oneArg' && args[0] === 'src') { + return 'ok'; + } + return '${fn} | ${args.join(", ")}'; + } + ); + + const manyArgs = await interpolateWithFunctionsAsync( + 'foo${ manyArgs("src", "hello") }bar', + async (fn, args) => { + if (fn === 'manyArgs' && args[0] === 'src' && args[1] === 'hello') { + return 'ok'; + } + return '${fn} | ${args.join(", ")}'; + } + ); + + expect(nonArgs).toBe('foookbar'); + expect(oneArg).toBe('foookbar'); + expect(manyArgs).toBe('foookbar'); + }); +}); + describe(interpolateWithGlobalContext, () => { test('interpolation', () => { const result = interpolateWithGlobalContext( diff --git a/packages/steps/src/utils/template.ts b/packages/steps/src/utils/template.ts index c4c7a9376..afc3d2369 100644 --- a/packages/steps/src/utils/template.ts +++ b/packages/steps/src/utils/template.ts @@ -7,6 +7,7 @@ import { nullthrows } from './nullthrows.js'; export const BUILD_STEP_INPUT_EXPRESSION_REGEXP = /\${\s*(inputs\.[\S]+)\s*}/; export const BUILD_STEP_OUTPUT_EXPRESSION_REGEXP = /\${\s*(steps\.[\S]+)\s*}/; +export const BUILD_STEP_FUNCTION_EXPRESSION_REGEXP = /\${\s*(?\w+)\((?.*)\)\s*}/; export const BUILD_GLOBAL_CONTEXT_EXPRESSION_REGEXP = /\${\s*(eas\.[\S]+)\s*}/; export const BUILD_STEP_OR_BUILD_GLOBAL_CONTEXT_REFERENCE_REGEX = /\${\s*((steps|eas)\.[\S]+)\s*}/; export const BUILD_STEP_IF_CONDITION_EXPRESSION_REGEXP = /\${\s*(always|success|failure)\(\)\s*}/; @@ -18,6 +19,59 @@ export function interpolateWithInputs( return interpolate(templateString, BUILD_STEP_INPUT_EXPRESSION_REGEXP, inputs); } +export function templateFunctionsAndArgsIterator(templateString: string): Iterable { + return { + [Symbol.iterator]() { + const regex = new RegExp(BUILD_STEP_FUNCTION_EXPRESSION_REGEXP, 'g'); + let functionCallMatch; + return { + next() { + while ((functionCallMatch = regex.exec(templateString))) { + if (functionCallMatch?.groups) { + const templateFunction = functionCallMatch.groups['fun']; + try { + const args = JSON.parse(`[${functionCallMatch.groups['args']}]`.replace(/'/g, '"')); + return { done: false, value: { templateFunction, args, functionCallMatch } }; + } catch (e) { + if (e instanceof SyntaxError) { + throw new BuildConfigError(`contains syntax error in "${templateString}"`); + } + throw e; + } + } + } + return { done: true, value: null }; + }, + }; + }, + }; +} + +export function iterateWithFunctions( + templateString: string, + fn: (fn: string, args: string[]) => any +): void { + const iterator = templateFunctionsAndArgsIterator(templateString); + for (const { templateFunction, args } of iterator) { + fn(templateFunction, args); + } +} + +export async function interpolateWithFunctionsAsync( + templateString: string, + fn: (fn: string, args: string[]) => Promise +): Promise { + let result = templateString; + + const iterator = templateFunctionsAndArgsIterator(templateString); + + for (const { templateFunction, args, functionCallMatch } of iterator) { + const value = await fn(templateFunction, args); + result = result.replace(functionCallMatch[0], value); + } + return result; +} + export function interpolateWithOutputs( templateString: string, fn: (path: string) => string