[ENG-9500] Add save and restore cache functions

packages/build-tools/src/context.ts

@@ -1,26 +1,28 @@
 import path from 'path';
 
-import fs from 'fs-extra';
+import { ExpoConfig } from '@expo/config';
 import {
   BuildPhase,
   BuildPhaseResult,
   BuildPhaseStats,
+  Env,
+  EnvironmentSecretType,
   Job,
   LogMarker,
-  Env,
-  errors,
   Metadata,
-  EnvironmentSecretType,
+  errors,
 } from '@expo/eas-build-job';
-import { ExpoConfig } from '@expo/config';
-import { bunyan } from '@expo/logger';
-import { SpawnPromise, SpawnOptions, SpawnResult } from '@expo/turtle-spawn';
 import { BuildTrigger } from '@expo/eas-build-job/dist/common';
+import { bunyan } from '@expo/logger';
+import { CacheManager } from '@expo/steps';
+import { SpawnOptions, SpawnPromise, SpawnResult } from '@expo/turtle-spawn';
+import fs from 'fs-extra';
 
-import { PackageManager, resolvePackageManager } from './utils/packageManager';
 import { resolveBuildPhaseErrorAsync } from './buildErrors/detectError';
 import { readAppConfig } from './utils/appConfig';
 import { createTemporaryEnvironmentSecretFile } from './utils/environmentSecrets';
+import { PackageManager, resolvePackageManager } from './utils/packageManager';
+export { CacheManager } from '@expo/steps';
 
 export enum ArtifactType {
   APPLICATION_ARCHIVE = 'APPLICATION_ARCHIVE',
@@ -33,11 +35,6 @@ export enum ArtifactType {
 
 export type Artifacts = Partial<Record<ArtifactType, string>>;
 
-export interface CacheManager {
-  saveCache(ctx: BuildContext<Job>): Promise<void>;
-  restoreCache(ctx: BuildContext<Job>): Promise<void>;
-}
-
 export interface LogBuffer {
   getLogs(): string[];
   getPhaseLogs(buildPhase: string): string[];
@@ -132,6 +129,10 @@ export class BuildContext<TJob extends Job> {
       : this.buildExecutablesDirectory;
   }
 
+  get projectRootDirectory(): string | undefined {
+    return this.job.projectRootDirectory;
+  }
+
   public get job(): TJob {
     return this._job;
   }

packages/build-tools/src/customBuildContext.ts

@@ -2,7 +2,7 @@ import path from 'path';
 
 import { BuildPhase, Env, Job, Metadata, Platform } from '@expo/eas-build-job';
 import { bunyan } from '@expo/logger';
-import { ExternalBuildContextProvider, BuildRuntimePlatform } from '@expo/steps';
+import { ExternalBuildContextProvider, BuildRuntimePlatform, CacheManager } from '@expo/steps';
 
 import { ArtifactType, BuildContext } from './context';
 
@@ -40,6 +40,9 @@ export class CustomBuildContext implements ExternalBuildContextProvider {
   public readonly runtimeApi: BuilderRuntimeApi;
   public readonly job: Job;
   public readonly metadata?: Metadata;
+  public readonly cacheManager?: CacheManager;
+  public readonly buildDirectory: string;
+  public readonly projectRootDirectory: string;
 
   private _env: Env;
 
@@ -56,6 +59,9 @@ export class CustomBuildContext implements ExternalBuildContextProvider {
     this.runtimeApi = {
       uploadArtifacts: (...args) => buildCtx['uploadArtifacts'](...args),
     };
+    this.cacheManager = buildCtx.cacheManager;
+    this.buildDirectory = buildCtx.buildDirectory;
+    this.projectRootDirectory = buildCtx.projectRootDirectory ?? '.';
   }
 
   public get runtimePlatform(): BuildRuntimePlatform {

packages/build-tools/src/index.ts

@@ -2,17 +2,17 @@ import * as Builders from './builders';
 
 export { Builders };
 
+export { PackageManager } from './utils/packageManager';
+
 export {
   Artifacts,
   ArtifactType,
   BuildContext,
-  CacheManager,
   LogBuffer,
   SkipNativeBuildError,
+  CacheManager,
 } from './context';
 
-export { PackageManager } from './utils/packageManager';
-
 export { findAndUploadXcodeBuildLogsAsync } from './ios/xcodeBuildLogs';
 
 export { Hook, runHookIfPresent } from './utils/hooks';

packages/build-tools/src/steps/easFunctions.ts

@@ -8,6 +8,7 @@ import { createSetUpNpmrcBuildFunction } from './functions/useNpmToken';
 import { createInstallNodeModulesBuildFunction } from './functions/installNodeModules';
 import { createPrebuildBuildFunction } from './functions/prebuild';
 import { createFindAndUploadBuildArtifactsBuildFunction } from './functions/findAndUploadBuildArtifacts';
+import { createSaveCacheBuildFunction, createRestoreCacheBuildFunction } from './functions/cache';
 import { configureEASUpdateIfInstalledFunction } from './functions/configureEASUpdateIfInstalled';
 import { injectAndroidCredentialsFunction } from './functions/injectAndroidCredentials';
 import { configureAndroidVersionFunction } from './functions/configureAndroidVersion';
@@ -26,6 +27,8 @@ export function getEasFunctions(ctx: CustomBuildContext): BuildFunction[] {
     createInstallNodeModulesBuildFunction(),
     createPrebuildBuildFunction(),
     createFindAndUploadBuildArtifactsBuildFunction(ctx),
+    createSaveCacheBuildFunction(),
+    createRestoreCacheBuildFunction(),
     configureEASUpdateIfInstalledFunction(),
     injectAndroidCredentialsFunction(),
     configureAndroidVersionFunction(),

packages/build-tools/src/steps/functions/cache.ts

@@ -0,0 +1,122 @@
+import crypto from 'crypto';
+
+import {
+  BuildFunction,
+  BuildStepInput,
+  BuildStepInputValueTypeName,
+  BuildStepOutput,
+} from '@expo/steps';
+
+function createCacheKey(cacheKey: string, paths: string[]): string {
+  const hash = crypto.createHash('sha256');
+  hash.update(paths.sort().join(''));
+
+  const pathsHash = hash.digest('hex');
+
+  return `${cacheKey}-${pathsHash}`;
+}
+
+export function createRestoreCacheBuildFunction(): BuildFunction {
+  return new BuildFunction({
+    namespace: 'eas',
+    id: 'restore-cache',
+    name: 'Restore Cache',
+    inputProviders: [
+      BuildStepInput.createProvider({
+        id: 'key',
+        required: true,
+        allowedValueTypeName: BuildStepInputValueTypeName.STRING,
+      }),
+      BuildStepInput.createProvider({
+        id: 'paths',
+        required: true,
+        allowedValueTypeName: BuildStepInputValueTypeName.JSON,
+      }),
+    ],
+    outputProviders: [
+      BuildStepOutput.createProvider({
+        id: 'cache_key',
+        required: true,
+      }),
+    ],
+    fn: async (stepsCtx, { inputs, outputs }) => {
+      const cacheManager = stepsCtx.global.cacheManager;
+
+      if (!cacheManager) {
+        return;
+      }
+
+      outputs.cache_key.set(inputs.key.value as string);
+      const paths = inputs.paths.value as [string];
+      const key = createCacheKey(inputs.key.value as string, paths);
+
+      cacheManager.generateUrls = true;
+      const {
+        job: { cache },
+      } = stepsCtx.global.provider.staticContext();
+
+      if (!(cache.downloadUrls && key in cache.downloadUrls)) {
+        stepsCtx.logger.info(`Cache ${key} does not exist, skipping restoring`);
+        return;
+      }
+      stepsCtx.logger.info(`Restoring cache ${key} in:\n ${paths.join('\n')}`);
+
+      await cacheManager.restoreCache(stepsCtx, {
+        ...cache,
+        disabled: false,
+        clear: false,
+        key,
+        paths,
+      });
+    },
+  });
+}
+
+export function createSaveCacheBuildFunction(): BuildFunction {
+  return new BuildFunction({
+    namespace: 'eas',
+    id: 'save-cache',
+    name: 'Save Cache',
+    inputProviders: [
+      BuildStepInput.createProvider({
+        id: 'key',
+        required: true,
+        allowedValueTypeName: BuildStepInputValueTypeName.STRING,
+      }),
+      BuildStepInput.createProvider({
+        id: 'paths',
+        required: true,
+        allowedValueTypeName: BuildStepInputValueTypeName.JSON,
+      }),
+    ],
+    fn: async (stepsCtx, { inputs }) => {
+      const cacheManager = stepsCtx.global.cacheManager;
+
+      if (!cacheManager) {
+        return;
+      }
+      const paths = inputs.paths.value as [string];
+      const key = createCacheKey(inputs.key.value as string, paths);
+
+      const {
+        job: { cache },
+      } = stepsCtx.global.provider.staticContext();
+      if (cache.downloadUrls && key in cache.downloadUrls) {
+        stepsCtx.logger.info(`Cache ${key} already exists, skipping saving`);
+        return;
+      }
+
+      stepsCtx.logger.info(`Saving cache from:\n ${paths.join('\n')}`);
+
+      cacheManager.generateUrls = true;
+
+      await cacheManager.saveCache(stepsCtx, {
+        disabled: false,
+        clear: false,
+        key,
+        paths,
+      });
+      stepsCtx.logger.info('Cache saved');
+    },
+  });
+}

packages/eas-build-job/src/common.ts

@@ -124,6 +124,7 @@ export interface Cache {
    */
   customPaths?: string[];
   paths: string[];
+  downloadUrls?: Record<string, string>;
 }
 
 export const CacheSchema = Joi.object({

packages/steps/src/BuildStepContext.ts

@@ -4,21 +4,24 @@ import path from 'path';
 import { bunyan } from '@expo/logger';
 import { v4 as uuidv4 } from 'uuid';
 
+import { BuildRuntimePlatform } from './BuildRuntimePlatform.js';
 import {
   BuildStep,
   BuildStepOutputAccessor,
   SerializedBuildStepOutputAccessor,
 } from './BuildStep.js';
+import { BuildStepEnv } from './BuildStepEnv.js';
+import { CacheManager } from './cacheUtils.js';
+import { BuildStepRuntimeError } from './errors.js';
 import {
   getObjectValueForInterpolation,
   interpolateWithGlobalContext,
   parseOutputPath,
 } from './utils/template.js';
-import { BuildStepRuntimeError } from './errors.js';
-import { BuildRuntimePlatform } from './BuildRuntimePlatform.js';
-import { BuildStepEnv } from './BuildStepEnv.js';
 
 interface SerializedExternalBuildContextProvider {
+  buildDirectory: string;
+  projectRootDirectory: string;
   projectSourceDirectory: string;
   projectTargetDirectory: string;
   defaultWorkingDirectory: string;
@@ -29,12 +32,15 @@ interface SerializedExternalBuildContextProvider {
 }
 
 export interface ExternalBuildContextProvider {
+  buildDirectory: string;
+  projectRootDirectory: string;
   readonly projectSourceDirectory: string;
   readonly projectTargetDirectory: string;
   readonly defaultWorkingDirectory: string;
   readonly buildLogsDirectory: string;
   readonly runtimePlatform: BuildRuntimePlatform;
   readonly logger: bunyan;
+  readonly cacheManager?: CacheManager;
 
   readonly staticContext: () => Record<string, any>;
 
@@ -54,16 +60,25 @@ export class BuildStepGlobalContext {
   public readonly runtimePlatform: BuildRuntimePlatform;
   public readonly baseLogger: bunyan;
   private didCheckOut = false;
+  public readonly cacheManager?: CacheManager;
 
   private stepById: Record<string, BuildStepOutputAccessor> = {};
 
   constructor(
-    private readonly provider: ExternalBuildContextProvider,
+    public readonly provider: ExternalBuildContextProvider,
     public readonly skipCleanup: boolean
   ) {
     this.stepsInternalBuildDirectory = path.join(os.tmpdir(), 'eas-build', uuidv4());
     this.runtimePlatform = provider.runtimePlatform;
     this.baseLogger = provider.logger;
+    this.cacheManager = provider.cacheManager;
+  }
+
+  public get buildDirectory(): string {
+    return this.provider.buildDirectory;
+  }
+  public get projectRootDirectory(): string {
+    return this.provider.projectRootDirectory;
   }
 
   public get projectSourceDirectory(): string {
@@ -142,6 +157,8 @@ export class BuildStepGlobalContext {
         defaultWorkingDirectory: this.provider.defaultWorkingDirectory,
         buildLogsDirectory: this.provider.buildLogsDirectory,
         runtimePlatform: this.provider.runtimePlatform,
+        buildDirectory: this.provider.buildDirectory,
+        projectRootDirectory: this.provider.projectRootDirectory,
         staticContext: this.provider.staticContext(),
         env: this.provider.env,
       },
@@ -159,6 +176,8 @@ export class BuildStepGlobalContext {
       defaultWorkingDirectory: serialized.provider.defaultWorkingDirectory,
       buildLogsDirectory: serialized.provider.buildLogsDirectory,
       runtimePlatform: serialized.provider.runtimePlatform,
+      buildDirectory: serialized.provider.buildDirectory,
+      projectRootDirectory: serialized.provider.projectRootDirectory,
       logger,
       staticContext: () => serialized.provider.staticContext,
       env: serialized.provider.env,
@@ -197,6 +216,10 @@ export class BuildStepContext {
     this.relativeWorkingDirectory = relativeWorkingDirectory;
   }
 
+  public get workingdir(): string {
+    return this.workingDirectory;
+  }
+
   public get global(): BuildStepGlobalContext {
     return this.ctx;
   }