From 066dcc1ded735b795204d03c1672fb98ec84c22b Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Wed, 20 Dec 2023 14:25:10 +0000 Subject: [PATCH 1/9] implement custom cache handler logic this includes: - refactoring existing logic so that it uses the incrementalCache class concept that Next.js also uses (so that we alight the two implementations) - reading the user next.config.(m)js file and create a custom cache handler and collecting the cache handler referenced in the incrementalCacheHandlerPath config - building the custom cache handler instead of the built-in ones in case one was indeed provided by the user --- packages/next-on-pages/build-metadata.d.ts | 10 + packages/next-on-pages/env.d.ts | 1 + .../src/buildApplication/buildCacheFiles.ts | 91 +++++++ .../src/buildApplication/buildWorkerFile.ts | 22 +- .../src/buildApplication/nextConfig.ts | 213 +++++++++++++++++ .../templates/_worker.js/index.ts | 9 +- .../templates/_worker.js/utils/cache.ts | 91 +++++-- .../templates/_worker.js/utils/fetch.ts | 5 +- .../cache/{kv.ts => KVCacheHandler.ts} | 8 +- .../{adaptor.ts => builtInCacheHandler.ts} | 96 +++----- .../templates/cache/incrementalCache.ts | 226 ++++++++++++++++++ .../next-on-pages/templates/cache/index.ts | 2 +- ...-api.ts => workersCacheApiCacheHandler.ts} | 8 +- .../src/buildApplication/nextConfig.test.ts | 88 +++++++ 14 files changed, 765 insertions(+), 105 deletions(-) create mode 100644 packages/next-on-pages/src/buildApplication/buildCacheFiles.ts create mode 100644 packages/next-on-pages/src/buildApplication/nextConfig.ts rename packages/next-on-pages/templates/cache/{kv.ts => KVCacheHandler.ts} (61%) rename packages/next-on-pages/templates/cache/{adaptor.ts => builtInCacheHandler.ts} (71%) create mode 100644 packages/next-on-pages/templates/cache/incrementalCache.ts rename packages/next-on-pages/templates/cache/{cache-api.ts => workersCacheApiCacheHandler.ts} (73%) create mode 100644 packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts diff --git a/packages/next-on-pages/build-metadata.d.ts b/packages/next-on-pages/build-metadata.d.ts index 16b7d8713..5e8c69fb7 100644 --- a/packages/next-on-pages/build-metadata.d.ts +++ b/packages/next-on-pages/build-metadata.d.ts @@ -4,4 +4,14 @@ type NextOnPagesBuildMetadata = { /** Locales used by the application (collected from the Vercel output) */ collectedLocales: string[]; + /** (subset of) values obtained from the user's next.config.js (if any was found) */ + config?: { + experimental?: Pick< + NonNullable< + // eslint-disable-next-line @typescript-eslint/consistent-type-imports -- the import needs to be dynamic since the nextConfig file itself uses this type + import('./src/buildApplication/nextConfig').NextConfig['experimental'] + >, + 'allowedRevalidateHeaderKeys' | 'fetchCacheKeyPrefix' + >; + }; }; diff --git a/packages/next-on-pages/env.d.ts b/packages/next-on-pages/env.d.ts index dcc62b1b4..583fbeb80 100644 --- a/packages/next-on-pages/env.d.ts +++ b/packages/next-on-pages/env.d.ts @@ -6,6 +6,7 @@ declare global { CF_PAGES?: string; SHELL?: string; __NEXT_ON_PAGES__KV_SUSPENSE_CACHE?: KVNamespace; + __BUILD_METADATA__: NextOnPagesBuildMetadata; [key: string]: string | Fetcher; } } diff --git a/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts new file mode 100644 index 000000000..cf0f99a85 --- /dev/null +++ b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts @@ -0,0 +1,91 @@ +import { build } from 'esbuild'; +import { join } from 'path'; +import { getNextConfig } from './nextConfig'; + +/** + * Builds files needed by the application in order to implement the Next.js suspense caching, these can + * be either for the custom cache handler provided by the user or the builtin cache handlers. + * + * @param nopDistDir path to the dist directory in which the build process saves the output files + * @param minify flag indicating wether minification should be applied to the cache files + * @param templatesDir path to the templates directory + */ +export async function buildCacheFiles( + nopDistDir: string, + minify: boolean, + templatesDir: string, +): Promise { + const outputCacheDir = join(nopDistDir, 'cache'); + + const customCacheHandlerBuilt = await buildCustomIncrementalCacheHandler( + outputCacheDir, + minify, + ); + + if (!customCacheHandlerBuilt) { + await buildBuiltInCacheHandlers(templatesDir, outputCacheDir, minify); + } +} + +/** + * Builds the file implementing the custom cache handler provided by the user, if one was provided. + * + * @param outputCacheDir path to the directory in which to write the file + * @param minify flag indicating wether minification should be applied to the cache files + * @returns true if the file was built, false otherwise (meaning that the user has not provided a custom cache handler) + */ +async function buildCustomIncrementalCacheHandler( + outputCacheDir: string, + minify: boolean, +): Promise { + const nextConfig = await getNextConfig(); + + const incrementalCacheHandlerPath = + nextConfig?.experimental?.incrementalCacheHandlerPath; + + if (!incrementalCacheHandlerPath) { + return false; + } + + try { + await build({ + entryPoints: [incrementalCacheHandlerPath], + bundle: true, + target: 'es2022', + platform: 'neutral', + outfile: join(outputCacheDir, 'custom.js'), + minify, + }); + } catch { + throw new Error( + `Failed to build custom incremental cache handler from the following provided path: ${incrementalCacheHandlerPath}`, + ); + } + return true; +} + +/** + * Builds the files implementing the builtin cache handlers. + * + * @param templatesDir path to the templates directory (from which the builtin cache files are taken) + * @param outputCacheDir path to the directory in which to write the files + * @param minify flag indicating wether minification should be applied to the cache files + */ +async function buildBuiltInCacheHandlers( + templatesDir: string, + outputCacheDir: string, + minify: boolean, +): Promise { + await build({ + entryPoints: [ + 'builtInCacheHandler.ts', + 'workersCacheApiCacheHandler.ts', + 'KVCacheHandler.ts', + ].map(fileName => join(templatesDir, 'cache', fileName)), + bundle: false, + target: 'es2022', + platform: 'neutral', + outdir: outputCacheDir, + minify, + }); +} diff --git a/packages/next-on-pages/src/buildApplication/buildWorkerFile.ts b/packages/next-on-pages/src/buildApplication/buildWorkerFile.ts index d84e54619..f985e26ac 100644 --- a/packages/next-on-pages/src/buildApplication/buildWorkerFile.ts +++ b/packages/next-on-pages/src/buildApplication/buildWorkerFile.ts @@ -6,6 +6,8 @@ import { generateGlobalJs } from './generateGlobalJs'; import type { ProcessedVercelOutput } from './processVercelOutput'; import { getNodeEnv } from '../utils/getNodeEnv'; import { normalizePath } from '../utils'; +import { buildCacheFiles } from './buildCacheFiles'; +import { extractBuildMetadataConfig, getNextConfig } from './nextConfig'; /** * Construct a record for the build output map. @@ -61,6 +63,14 @@ export async function buildWorkerFile( const outputFile = join(workerJsDir, 'index.js'); + const nextConfig = await getNextConfig(); + + const buildMetadataConfig = nextConfig + ? { + config: extractBuildMetadataConfig(nextConfig), + } + : {}; + await build({ entryPoints: [join(templatesDir, '_worker.js')], banner: { @@ -76,22 +86,14 @@ export async function buildWorkerFile( __NODE_ENV__: JSON.stringify(getNodeEnv()), __BUILD_METADATA__: JSON.stringify({ collectedLocales: collectLocales(vercelConfig.routes), + ...buildMetadataConfig, }), }, outfile: outputFile, minify, }); - await build({ - entryPoints: ['adaptor.ts', 'cache-api.ts', 'kv.ts'].map(fileName => - join(templatesDir, 'cache', fileName), - ), - bundle: false, - target: 'es2022', - platform: 'neutral', - outdir: join(nopDistDir, 'cache'), - minify, - }); + await buildCacheFiles(nopDistDir, minify, templatesDir); return relative('.', outputFile); } diff --git a/packages/next-on-pages/src/buildApplication/nextConfig.ts b/packages/next-on-pages/src/buildApplication/nextConfig.ts new file mode 100644 index 000000000..ee1e3be0b --- /dev/null +++ b/packages/next-on-pages/src/buildApplication/nextConfig.ts @@ -0,0 +1,213 @@ +import { resolve } from 'path'; +import { validateFile } from '../utils'; +import * as os from 'os'; + +/** + * The type of object a next.config.js file yields. + * + * Note: in the Next.js codebase they have a more complex/proper type for this, here we have a very simplified + * version of it which includes just what we need in next-on-pages. + */ +export type NextConfig = Record & { + experimental?: { + incrementalCacheHandlerPath?: string; + allowedRevalidateHeaderKeys?: string[]; + fetchCacheKeyPrefix?: string; + }; +}; + +/** + * Gets the user defined next config object (from their next.config.(m)js file). + * + * Note: If the user defined their config file by exporting a factory function, the + * function is appropriately used and the resulting config object is returned. + * + * @returns the user defined next config object or null if such object could not be obtained (meaning that no next.config.(m)js file was found) + */ +export async function getNextConfig(): Promise { + const configFilePath = + (await getConfigFilePath('js')) || (await getConfigFilePath('mjs')); + + if (!configFilePath) { + return null; + } + + const configObjOrFn = await import(configFilePath).then(m => m.default); + + const configObj = + typeof configObjOrFn === 'function' + ? configObjOrFn(PHASE_PRODUCTION_BUILD, { defaultConfig }) + : configObjOrFn; + + return configObj; +} + +/** + * Gets the path of a next.config file present in the current directory if present. + * + * @param extension the config file extension (either 'js' or 'mjs') + * @returns the path of the file if it was found, null otherwise + */ +async function getConfigFilePath( + extension: 'js' | 'mjs', +): Promise { + const nextConfigJsPath = resolve(`next.config.${extension}`); + const nextConfigJsFound = await validateFile(nextConfigJsPath); + if (nextConfigJsFound) { + return nextConfigJsPath; + } + return null; +} + +// https://github.com/vercel/next.js/blob/0fc1d9e9/packages/next/src/shared/lib/constants.ts#L37 +const PHASE_PRODUCTION_BUILD = 'phase-production-build'; + +// https://github.com/vercel/next.js/blob/0fc1d9e9/packages/next/src/server/config-shared.ts#L701-L815 +const defaultConfig = { + env: {}, + webpack: null, + eslint: { + ignoreDuringBuilds: false, + }, + typescript: { + ignoreBuildErrors: false, + tsconfigPath: 'tsconfig.json', + }, + distDir: '.next', + cleanDistDir: true, + assetPrefix: '', + configOrigin: 'default', + useFileSystemPublicRoutes: true, + generateBuildId: () => null, + generateEtags: true, + pageExtensions: ['tsx', 'ts', 'jsx', 'js'], + poweredByHeader: true, + compress: true, + analyticsId: process.env['VERCEL_ANALYTICS_ID'] || '', + images: { + deviceSizes: [640, 750, 828, 1080, 1200, 1920, 2048, 3840], + imageSizes: [16, 32, 48, 64, 96, 128, 256, 384], + path: '/_next/image', + loader: 'default', + loaderFile: '', + domains: [], + disableStaticImages: false, + minimumCacheTTL: 60, + formats: ['image/webp'], + dangerouslyAllowSVG: false, + contentSecurityPolicy: `script-src 'none'; frame-src 'none'; sandbox;`, + contentDispositionType: 'inline', + remotePatterns: [], + unoptimized: false, + }, + devIndicators: { + buildActivity: true, + buildActivityPosition: 'bottom-right', + }, + onDemandEntries: { + maxInactiveAge: 60 * 1000, + pagesBufferLength: 5, + }, + amp: { + canonicalBase: '', + }, + basePath: '', + sassOptions: {}, + trailingSlash: false, + i18n: null, + productionBrowserSourceMaps: false, + optimizeFonts: true, + excludeDefaultMomentLocales: true, + serverRuntimeConfig: {}, + publicRuntimeConfig: {}, + reactProductionProfiling: false, + reactStrictMode: null, + httpAgentOptions: { + keepAlive: true, + }, + outputFileTracing: true, + staticPageGenerationTimeout: 60, + swcMinify: true, + output: process.env['NEXT_PRIVATE_STANDALONE'] ? 'standalone' : undefined, + modularizeImports: undefined, + experimental: { + windowHistorySupport: false, + serverMinification: true, + serverSourceMaps: false, + caseSensitiveRoutes: false, + useDeploymentId: false, + deploymentId: undefined, + useDeploymentIdServerActions: false, + appDocumentPreloading: undefined, + clientRouterFilter: true, + clientRouterFilterRedirects: false, + fetchCacheKeyPrefix: '', + middlewarePrefetch: 'flexible', + optimisticClientCache: true, + manualClientBasePath: false, + cpus: Math.max( + 1, + (Number(process.env['CIRCLE_NODE_TOTAL']) || + (os.cpus() || { length: 1 }).length) - 1, + ), + memoryBasedWorkersCount: false, + isrFlushToDisk: true, + workerThreads: false, + proxyTimeout: undefined, + optimizeCss: false, + nextScriptWorkers: false, + scrollRestoration: false, + externalDir: false, + disableOptimizedLoading: false, + gzipSize: true, + craCompat: false, + esmExternals: true, + isrMemoryCacheSize: 50 * 1024 * 1024, + incrementalCacheHandlerPath: undefined, + fullySpecified: false, + outputFileTracingRoot: process.env['NEXT_PRIVATE_OUTPUT_TRACE_ROOT'] || '', + swcTraceProfiling: false, + forceSwcTransforms: false, + swcPlugins: undefined, + largePageDataBytes: 128 * 1000, + disablePostcssPresetEnv: undefined, + amp: undefined, + urlImports: undefined, + adjustFontFallbacks: false, + adjustFontFallbacksWithSizeAdjust: false, + turbo: undefined, + turbotrace: undefined, + typedRoutes: false, + instrumentationHook: false, + bundlePagesExternals: false, + ppr: + process.env['__NEXT_TEST_MODE'] && + process.env['__NEXT_EXPERIMENTAL_PPR'] === 'true' + ? true + : false, + webpackBuildWorker: undefined, + }, +}; + +/** + * Given a raw nextConfig object it extracts the data from it that we'd want to save as build metadata + * (for later runtime usage). + * + * @param nextConfig the raw config object obtained from a next.config.js file + * @returns the extracted config build metadata + */ +export function extractBuildMetadataConfig( + nextConfig: NextConfig, +): NonNullable { + const config: NonNullable = {}; + + if (nextConfig.experimental) { + config.experimental = {}; + config.experimental.allowedRevalidateHeaderKeys = + nextConfig.experimental.allowedRevalidateHeaderKeys; + config.experimental.fetchCacheKeyPrefix = + nextConfig.experimental.fetchCacheKeyPrefix; + } + + return config; +} diff --git a/packages/next-on-pages/templates/_worker.js/index.ts b/packages/next-on-pages/templates/_worker.js/index.ts index 929f6cc47..5c2d1fa2d 100644 --- a/packages/next-on-pages/templates/_worker.js/index.ts +++ b/packages/next-on-pages/templates/_worker.js/index.ts @@ -34,8 +34,13 @@ export default { } return envAsyncLocalStorage.run( - // NOTE: The `SUSPENSE_CACHE_URL` is used to tell the Next.js Fetch Cache where to send requests. - { ...env, NODE_ENV: __NODE_ENV__, SUSPENSE_CACHE_URL }, + { + ...env, + NODE_ENV: __NODE_ENV__, + // NOTE: The `SUSPENSE_CACHE_URL` is used to tell the Next.js Fetch Cache where to send requests. + SUSPENSE_CACHE_URL, + __BUILD_METADATA__, + }, async () => { const url = new URL(request.url); if (url.pathname.startsWith('/_next/image')) { diff --git a/packages/next-on-pages/templates/_worker.js/utils/cache.ts b/packages/next-on-pages/templates/_worker.js/utils/cache.ts index 5030fa70d..06ef82332 100644 --- a/packages/next-on-pages/templates/_worker.js/utils/cache.ts +++ b/packages/next-on-pages/templates/_worker.js/utils/cache.ts @@ -1,31 +1,42 @@ -import type { CacheAdaptor, IncrementalCacheValue } from '../../cache'; import { SUSPENSE_CACHE_URL } from '../../cache'; +import { + IncrementalCache, + type CacheHandler, + type IncrementalCacheValue, +} from '../../cache/incrementalCache'; // https://github.com/vercel/next.js/blob/48a566bc/packages/next/src/server/lib/incremental-cache/fetch-cache.ts#L19 const CACHE_TAGS_HEADER = 'x-vercel-cache-tags'; // https://github.com/vercel/next.js/blob/ba23d986/packages/next/src/lib/constants.ts#L18 const NEXT_CACHE_SOFT_TAGS_HEADER = 'x-next-cache-soft-tags'; +// https://github.com/vercel/next.js/blob/fc25fcef/packages/next/src/server/lib/incremental-cache/fetch-cache.ts#L21 +const CACHE_STATE_HEADER = 'x-vercel-cache-state'; + /** * Handles an internal request to the suspense cache. * * @param request Incoming request to handle. + * @param buildMetadata Metadata collected during the build process. * @returns Response to the request, or null if the request is not for the suspense cache. */ -export async function handleSuspenseCacheRequest(request: Request) { +export async function handleSuspenseCacheRequest( + request: Request, + buildMetadata: NextOnPagesBuildMetadata, +) { const baseUrl = `https://${SUSPENSE_CACHE_URL}/v1/suspense-cache/`; if (!request.url.startsWith(baseUrl)) return null; try { const url = new URL(request.url); - const cache = await getSuspenseCacheAdaptor(); + const incrementalCache = await getIncrementalCache(request, buildMetadata); if (url.pathname === '/v1/suspense-cache/revalidate') { // Update the revalidated timestamp for the tags in the tags manifest. const tags = url.searchParams.get('tags')?.split(',') ?? []; for (const tag of tags) { - await cache.revalidateTag(tag); + await incrementalCache.revalidateTag(tag); } return new Response(null, { status: 200 }); @@ -44,16 +55,15 @@ export async function handleSuspenseCacheRequest(request: Request) { NEXT_CACHE_SOFT_TAGS_HEADER, ); - // Retrieve the value from the cache. - const data = await cache.get(cacheKey, { softTags }); + const data = await incrementalCache.get(cacheKey, { softTags }); if (!data) return new Response(null, { status: 404 }); return new Response(JSON.stringify(data.value), { status: 200, headers: { 'Content-Type': 'application/json', - 'x-vercel-cache-state': 'fresh', - age: `${(Date.now() - (data.lastModified ?? Date.now())) / 1000}`, + [CACHE_STATE_HEADER]: 'fresh', + age: `${data.age}`, }, }); } @@ -65,7 +75,7 @@ export async function handleSuspenseCacheRequest(request: Request) { body.tags ??= getTagsFromHeader(request, CACHE_TAGS_HEADER) ?? []; } - await cache.set(cacheKey, body); + await incrementalCache.set(cacheKey, body, { tags: body.tags }); return new Response(null, { status: 200 }); } @@ -80,29 +90,58 @@ export async function handleSuspenseCacheRequest(request: Request) { } /** - * Gets the cache adaptor to use for the suspense cache. + * Gets an IncrementalCache instance to be used to implement the suspense caching. * - * @returns Adaptor for the suspense cache. + * @param request Incoming request to handle. + * @param buildMetadata Metadata collected during the build process. + * @returns An IncrementalCache instance */ -export async function getSuspenseCacheAdaptor(): Promise { - if (process.env.__NEXT_ON_PAGES__KV_SUSPENSE_CACHE) { - return getInternalCacheAdaptor('kv'); +export async function getIncrementalCache( + request: Request, + buildMetadata: NextOnPagesBuildMetadata, +): Promise { + let curCacheHandler: typeof CacheHandler | undefined = undefined; + try { + const customAdaptorFileName = 'custom.js'; + curCacheHandler = ( + await import(`./__next-on-pages-dist__/cache/${customAdaptorFileName}`) + ).default; + } catch (e) { + /**/ + } + + if (!curCacheHandler) { + if (process.env.__NEXT_ON_PAGES__KV_SUSPENSE_CACHE) { + curCacheHandler = await getBuiltInCacheHandler('kv'); + } else { + curCacheHandler = await getBuiltInCacheHandler('workers-cache-api'); + } } - return getInternalCacheAdaptor('cache-api'); + const requestHeaders = Object.fromEntries(request.headers.entries()); + + const { allowedRevalidateHeaderKeys, fetchCacheKeyPrefix } = + buildMetadata.config?.experimental ?? {}; + + return new IncrementalCache({ + curCacheHandler, + allowedRevalidateHeaderKeys, + requestHeaders, + fetchCacheKeyPrefix, + }); } -/** - * Gets an internal cache adaptor. - * - * @param type The type of adaptor to get. - * @returns A new instance of the adaptor. - */ -async function getInternalCacheAdaptor( - type: 'kv' | 'cache-api', -): Promise { - const adaptor = await import(`./__next-on-pages-dist__/cache/${type}.js`); - return new adaptor.default(); +async function getBuiltInCacheHandler( + type: 'kv' | 'workers-cache-api', +): Promise { + const fileName = { + kv: 'KVCacheHandler', + 'workers-cache-api': 'workersCacheApiCacheHandler', + }[type]; + const cacheHandlerModule = await import( + `./__next-on-pages-dist__/cache/${fileName}.js` + ); + return cacheHandlerModule.default; } function getTagsFromHeader(req: Request, key: string): string[] | undefined { diff --git a/packages/next-on-pages/templates/_worker.js/utils/fetch.ts b/packages/next-on-pages/templates/_worker.js/utils/fetch.ts index da353ceb7..56b8a5790 100644 --- a/packages/next-on-pages/templates/_worker.js/utils/fetch.ts +++ b/packages/next-on-pages/templates/_worker.js/utils/fetch.ts @@ -23,7 +23,10 @@ function applyPatch() { let response = await handleInlineAssetRequest(request); if (response) return response; - response = await handleSuspenseCacheRequest(request); + response = await handleSuspenseCacheRequest( + request, + process.env.__BUILD_METADATA__, + ); if (response) return response; setRequestUserAgentIfNeeded(request); diff --git a/packages/next-on-pages/templates/cache/kv.ts b/packages/next-on-pages/templates/cache/KVCacheHandler.ts similarity index 61% rename from packages/next-on-pages/templates/cache/kv.ts rename to packages/next-on-pages/templates/cache/KVCacheHandler.ts index c9e95684f..1092edba1 100644 --- a/packages/next-on-pages/templates/cache/kv.ts +++ b/packages/next-on-pages/templates/cache/KVCacheHandler.ts @@ -1,8 +1,8 @@ -import { CacheAdaptor } from './adaptor.js'; +import { BuiltInCacheHandler } from './builtInCacheHandler.js'; +import type { CacheHandlerContext } from './incrementalCache.js'; -/** Suspense Cache adaptor for Workers KV. */ -export default class KVAdaptor extends CacheAdaptor { - constructor(ctx: Record = {}) { +export default class KVCacheHandler extends BuiltInCacheHandler { + constructor(ctx: CacheHandlerContext) { super(ctx); } diff --git a/packages/next-on-pages/templates/cache/adaptor.ts b/packages/next-on-pages/templates/cache/builtInCacheHandler.ts similarity index 71% rename from packages/next-on-pages/templates/cache/adaptor.ts rename to packages/next-on-pages/templates/cache/builtInCacheHandler.ts index 608877882..98562ffd2 100644 --- a/packages/next-on-pages/templates/cache/adaptor.ts +++ b/packages/next-on-pages/templates/cache/builtInCacheHandler.ts @@ -1,23 +1,33 @@ +import type { + CacheHandler, + CacheHandlerContext, + CacheHandlerValue, + IncrementalCache, + TagsManifest, +} from './incrementalCache'; + // NOTE: This is given the same name that the environment variable has in the Next.js source code. export const SUSPENSE_CACHE_URL = 'INTERNAL_SUSPENSE_CACHE_HOSTNAME.local'; // https://github.com/vercel/next.js/blob/f6babb4/packages/next/src/lib/constants.ts#23 const NEXT_CACHE_IMPLICIT_TAG_ID = '_N_T_'; -// Set to track the revalidated tags in requests. -const revalidatedTags = new Set(); - -/** Generic adaptor for the Suspense Cache. */ -export class CacheAdaptor { +/** A Shared base for built-in cache handlers. */ +export class BuiltInCacheHandler implements CacheHandler { /** The tags manifest for fetch calls. */ public tagsManifest: TagsManifest | undefined; /** The key used for the tags manifest in the cache. */ public tagsManifestKey = 'tags-manifest'; + // Set to track the revalidated tags in requests. + private revalidatedTags: Set; + /** - * @param ctx The incremental cache context from Next.js. NOTE: This is not currently utilised in NOP. + * @param ctx The incremental cache context. */ - constructor(protected ctx: Record = {}) {} + constructor(protected ctx: CacheHandlerContext) { + this.revalidatedTags = new Set(ctx.revalidatedTags); + } /** * Retrieves an entry from the storage mechanism. @@ -42,23 +52,26 @@ export class CacheAdaptor { /** * Puts a new entry in the suspense cache. * - * @param key Key for the item in the suspense cache. + * @param cacheKey Key for the item in the suspense cache. * @param value The cached value to add to the suspense cache. + * @param ctx The cache handler context. */ - public async set(key: string, value: IncrementalCacheValue): Promise { + public async set( + ...[cacheKey, value, ctx]: Parameters + ): Promise { const newEntry: CacheHandlerValue = { lastModified: Date.now(), value, }; // Update the cache entry. - await this.update(key, JSON.stringify(newEntry)); + await this.update(cacheKey, JSON.stringify(newEntry)); switch (newEntry.value?.kind) { case 'FETCH': { // Update the tags with the cache key. - const tags = getTagsFromEntry(newEntry); - await this.setTags(tags, { cacheKey: key }); + const tags = getTagsFromEntry(newEntry) ?? ctx.tags ?? []; + await this.setTags(tags, { cacheKey: cacheKey }); const derivedTags = getDerivedTags(tags); const implicitTags = derivedTags.map( @@ -66,7 +79,7 @@ export class CacheAdaptor { ); [...derivedTags, ...implicitTags].forEach(tag => - revalidatedTags.delete(tag), + this.revalidatedTags.delete(tag), ); } } @@ -76,15 +89,14 @@ export class CacheAdaptor { * Retrieves an entry from the suspense cache. * * @param key Key for the item in the suspense cache. - * @param opts Soft cache tags used when checking if an entry is stale. + * @param ctx The cache handler context. * @returns The cached value, or null if no entry exists. */ public async get( - key: string, - { softTags }: { softTags?: string[] }, + ...[cacheKey, ctx]: Parameters ): Promise { // Get entry from the cache. - const entry = await this.retrieve(key); + const entry = await this.retrieve(cacheKey); if (!entry) return null; let data: CacheHandlerValue; @@ -102,13 +114,13 @@ export class CacheAdaptor { // Check if the cache entry is stale or fresh based on the tags. const tags = getTagsFromEntry(data); - const combinedTags = softTags - ? [...tags, ...softTags] - : getDerivedTags(tags); + const combinedTags = ctx?.softTags + ? [...(tags ?? []), ...ctx.softTags] + : getDerivedTags(tags ?? []); const isStale = combinedTags.some(tag => { // If a revalidation has been triggered, the current entry is stale. - if (revalidatedTags.has(tag)) return true; + if (this.revalidatedTags.has(tag)) return true; const tagEntry = this.tagsManifest?.items?.[tag]; return ( @@ -135,7 +147,7 @@ export class CacheAdaptor { // Update the revalidated timestamp for the tags in the tags manifest. await this.setTags([tag], { revalidatedAt: Date.now() }); - revalidatedTags.add(tag); + this.revalidatedTags.add(tag); } /** @@ -155,7 +167,7 @@ export class CacheAdaptor { } /** - * Saves the local tags manifest in the suspence cache. + * Saves the local tags manifest in the suspense cache. */ public async saveTagsManifest(): Promise { if (this.tagsManifest) { @@ -207,38 +219,6 @@ export class CacheAdaptor { } } -// https://github.com/vercel/next.js/blob/261db49/packages/next/src/server/lib/incremental-cache/file-system-cache.ts#L17 -export type TagsManifest = { - version: 1; - items: { [tag: string]: TagsManifestItem }; -}; -export type TagsManifestItem = { keys: string[]; revalidatedAt?: number }; - -// https://github.com/vercel/next.js/blob/df4c2aa8/packages/next/src/server/response-cache/types.ts#L24 -export type CachedFetchValue = { - kind: 'FETCH'; - data: { - headers: { [k: string]: string }; - body: string; - url: string; - status?: number; - // field used by older versions of Next.js (see: https://github.com/vercel/next.js/blob/fda1ecc/packages/next/src/server/response-cache/types.ts#L23) - tags?: string[]; - }; - // tags are only present with file-system-cache - // fetch cache stores tags outside of cache entry - tags?: string[]; - revalidate: number; -}; - -export type CacheHandlerValue = { - lastModified?: number; - age?: number; - cacheState?: string; - value: IncrementalCacheValue | null; -}; -export type IncrementalCacheValue = CachedFetchValue; - /** * Derives a list of tags from the given tags. This is taken from the Next.js source code. * @@ -274,6 +254,8 @@ export function getDerivedTags(tags: string[]): string[] { return derivedTags; } -export function getTagsFromEntry(entry: CacheHandlerValue): string[] { - return entry.value?.tags ?? entry.value?.data?.tags ?? []; +export function getTagsFromEntry( + entry: CacheHandlerValue, +): string[] | undefined { + return entry.value?.tags ?? entry.value?.data?.tags; } diff --git a/packages/next-on-pages/templates/cache/incrementalCache.ts b/packages/next-on-pages/templates/cache/incrementalCache.ts new file mode 100644 index 000000000..fa91f5a6f --- /dev/null +++ b/packages/next-on-pages/templates/cache/incrementalCache.ts @@ -0,0 +1,226 @@ +// NOTE: this file is mostly a simplified version of an equivalent file in the Next.js codebase, most logic here +// is a simplified and/or tweaked version aimed for next-on-pages usage +// (the Next.js file: https://github.com/vercel/next.js/blob/c4adae8/packages/next/src/server/lib/incremental-cache/index.ts) + +// source:https://github.com/vercel/next.js/blob/0fc1d9e98/packages/next/src/lib/constants.ts#L17 +export const NEXT_CACHE_REVALIDATED_TAGS_HEADER = 'x-next-revalidated-tags'; + +/** + * Simplified version of the interface of the same name from Next.js + * source: https://github.com/vercel/next.js/blob/0fc1d9e982/packages/next/src/server/lib/incremental-cache/index.ts#L26-L39 + */ +export interface CacheHandlerContext { + revalidatedTags: string[]; + _requestHeaders: IncrementalCache['requestHeaders']; + fetchCacheKeyPrefix?: string; +} + +/* eslint-disable -- + the following class comes from the Next.js source code: + https://github.com/vercel/next.js/blob/3b64a53e59/packages/next/src/server/lib/incremental-cache/index.ts#L48-L63 + eslint is disabled for it just so that we can keep the code as is without modifications +*/ +export class CacheHandler { + // eslint-disable-next-line + constructor(_ctx: CacheHandlerContext) {} + + public async get( + ..._args: Parameters + ): Promise { + return {} as any; + } + + public async set( + ..._args: Parameters + ): Promise {} + + public async revalidateTag(_tag: string): Promise {} +} +/* eslint-enable */ + +/** + * Simplified (and tweaked) version of the IncrementalCache from Next.js + * source: https://github.com/vercel/next.js/blob/c4adae89b/packages/next/src/server/lib/incremental-cache/index.ts#L65 + */ +export class IncrementalCache { + cacheHandler: CacheHandler; + requestHeaders: Record; + revalidatedTags?: string[]; + + constructor({ + curCacheHandler, + requestHeaders, + fetchCacheKeyPrefix, + }: { + allowedRevalidateHeaderKeys?: string[]; + requestHeaders: IncrementalCache['requestHeaders']; + fetchCacheKeyPrefix?: string; + curCacheHandler: typeof CacheHandler; + }) { + this.requestHeaders = requestHeaders; + + let revalidatedTags: string[] = []; + + if ( + typeof requestHeaders[NEXT_CACHE_REVALIDATED_TAGS_HEADER] === 'string' + ) { + revalidatedTags = + requestHeaders[NEXT_CACHE_REVALIDATED_TAGS_HEADER].split(','); + } + + this.cacheHandler = new curCacheHandler({ + revalidatedTags, + _requestHeaders: requestHeaders, + fetchCacheKeyPrefix, + }); + } + + async revalidateTag(tag: string) { + return this.cacheHandler?.revalidateTag(tag); + } + + async get( + cacheKey: string, + ctx: { + kindHint?: IncrementalCacheKindHint; + revalidate?: number | false; + fetchUrl?: string; + fetchIdx?: number; + tags?: string[]; + softTags?: string[]; + } = {}, + ): Promise<(IncrementalCacheEntry & { age?: number }) | null> { + let entry: (IncrementalCacheEntry & { age?: number }) | null = null; + let revalidate = ctx.revalidate; + + const cacheData = await this.cacheHandler.get(cacheKey, ctx); + + const age = cacheData + ? (Date.now() - (cacheData.lastModified || 0)) / 1000 + : undefined; + + if (cacheData?.value?.kind === 'FETCH') { + const combinedTags = [...(ctx.tags || []), ...(ctx.softTags || [])]; + // if a tag was revalidated we don't return stale data + if ( + combinedTags.some(tag => { + return this.revalidatedTags?.includes(tag); + }) + ) { + return null; + } + + revalidate = revalidate || cacheData.value.revalidate; + const isStale = + typeof revalidate === 'number' && + typeof age === 'number' && + age > revalidate; + const data = cacheData.value.data; + + return { + isStale: isStale, + value: { + kind: 'FETCH', + data, + revalidate: revalidate, + }, + age, + revalidateAfter: + typeof revalidate === 'number' && Date.now() + revalidate * 1000, + }; + } + + let isStale: boolean | -1 | undefined; + let revalidateAfter: false | number; + + if (cacheData?.lastModified === -1) { + revalidateAfter = -1 * CACHE_ONE_YEAR; + isStale = -1; + } else { + revalidateAfter = 1 * 1000 + (cacheData?.lastModified || Date.now()); + isStale = revalidateAfter < Date.now() ? true : undefined; + } + + entry = { + isStale, + revalidateAfter, + value: null, + }; + + if (cacheData) { + entry.value = cacheData.value; + entry.age = age; + } else { + await this.set(cacheKey, entry.value, ctx); + } + + return { + ...entry, + age, + }; + } + + async set( + pathname: string, + data: IncrementalCacheValue | null, + ctx: { + revalidate?: number | false; + fetchUrl?: string; + fetchIdx?: number; + tags?: string[]; + }, + ) { + await this.cacheHandler.set(pathname, data, ctx); + } +} + +// https://github.com/vercel/next.js/blob/0fc1d9e982/packages/next/src/server/response-cache/types.ts#L131 +type IncrementalCacheKindHint = 'app' | 'pages' | 'fetch'; + +// https://github.com/vercel/next.js/blob/0fc1d9e982/packages/next/src/server/response-cache/types.ts#L83-L90 +export type IncrementalCacheEntry = { + curRevalidate?: number | false; + // milliseconds to revalidate after + revalidateAfter: number | false; + // -1 here dictates a blocking revalidate should be used + isStale?: boolean | -1; + value: IncrementalCacheValue | null; +}; + +const CACHE_ONE_YEAR = 31536000; + +// https://github.com/vercel/next.js/blob/261db49/packages/next/src/server/lib/incremental-cache/file-system-cache.ts#L17 +export type TagsManifest = { + version: 1; + items: { [tag: string]: TagsManifestItem }; +}; +export type TagsManifestItem = { keys: string[]; revalidatedAt?: number }; + +// https://github.com/vercel/next.js/blob/df4c2aa8/packages/next/src/server/response-cache/types.ts#L24 +export type CachedFetchValue = { + kind: 'FETCH'; + data: { + headers: { [k: string]: string }; + body: string; + url: string; + status?: number; + // field used by older versions of Next.js (see: https://github.com/vercel/next.js/blob/fda1ecc/packages/next/src/server/response-cache/types.ts#L23) + tags?: string[]; + }; + // tags are only present with file-system-cache + // fetch cache stores tags outside of the cache entry's data + tags?: string[]; + revalidate: number; +}; + +// https://github.com/vercel/next.js/blob/0fc1d9e982/packages/next/src/server/lib/incremental-cache/index.ts#L41-L46 +export type CacheHandlerValue = { + lastModified?: number; + age?: number; + cacheState?: string; + value: IncrementalCacheValue | null; +}; + +// source: https://github.com/vercel/next.js/blob/0fc1d9e982c/packages/next/src/server/response-cache/types.ts#L92 +// Note: the type is much simplified here +export type IncrementalCacheValue = CachedFetchValue; diff --git a/packages/next-on-pages/templates/cache/index.ts b/packages/next-on-pages/templates/cache/index.ts index 41c162740..78b7c2785 100644 --- a/packages/next-on-pages/templates/cache/index.ts +++ b/packages/next-on-pages/templates/cache/index.ts @@ -1 +1 @@ -export * from './adaptor'; +export * from './builtInCacheHandler'; diff --git a/packages/next-on-pages/templates/cache/cache-api.ts b/packages/next-on-pages/templates/cache/workersCacheApiCacheHandler.ts similarity index 73% rename from packages/next-on-pages/templates/cache/cache-api.ts rename to packages/next-on-pages/templates/cache/workersCacheApiCacheHandler.ts index 1c1af697d..d5d229225 100644 --- a/packages/next-on-pages/templates/cache/cache-api.ts +++ b/packages/next-on-pages/templates/cache/workersCacheApiCacheHandler.ts @@ -1,11 +1,11 @@ -import { CacheAdaptor } from './adaptor.js'; +import { BuiltInCacheHandler } from './builtInCacheHandler.js'; +import type { CacheHandlerContext } from './incrementalCache.js'; -/** Suspense Cache adaptor for the Cache API. */ -export default class CacheApiAdaptor extends CacheAdaptor { +export default class WorkersCacheAPICacheHandler extends BuiltInCacheHandler { /** Name of the cache to open in the Cache API. */ public cacheName = 'suspense-cache'; - constructor(ctx: Record = {}) { + constructor(ctx: CacheHandlerContext) { super(ctx); } diff --git a/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts new file mode 100644 index 000000000..5f22d4e8b --- /dev/null +++ b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts @@ -0,0 +1,88 @@ +import { vi, describe, test, expect } from 'vitest'; +import { getNextConfig } from '../../../src/buildApplication/nextConfig'; + +const mocks = vi.hoisted(() => ({ + nextConfigJsFileExists: true, + nextConfigMjsFileExists: false, +})); + +vi.mock('path', () => ({ + resolve: (str: string) => str, +})); + +vi.mock('../../../src/utils/fs', () => ({ + validateFile: (file: string) => { + if (file === 'next.config.js') return mocks.nextConfigJsFileExists; + if (file === 'next.config.mjs') return mocks.nextConfigMjsFileExists; + return false; + }, +})); + +describe('getNextConfigJs', () => { + test('neither next.config.js nor next.config.mjs file exists', async () => { + mocks.nextConfigJsFileExists = false; + mocks.nextConfigMjsFileExists = false; + const config = await getNextConfig(); + expect(config).toBeNull(); + }); + + test('processing a standard next.config.js file', async () => { + mocks.nextConfigJsFileExists = true; + mocks.nextConfigMjsFileExists = false; + vi.doMock('next.config.js', () => ({ + default: { + experimental: { + incrementalCacheHandlerPath: 'my/incrementalCacheHandler/path', + }, + }, + })); + + const config = await getNextConfig(); + + expect(config).toEqual({ + experimental: { + incrementalCacheHandlerPath: 'my/incrementalCacheHandler/path', + }, + }); + }); + + test('processing a standard next.config.mjs file', async () => { + mocks.nextConfigJsFileExists = false; + mocks.nextConfigMjsFileExists = true; + + vi.doMock('next.config.mjs', () => ({ + default: { + trailingSlash: true, + }, + })); + + const config = await getNextConfig(); + + expect(config).toEqual({ + trailingSlash: true, + }); + }); + + test('processing a next.config.js file exporting a function', async () => { + mocks.nextConfigJsFileExists = true; + mocks.nextConfigMjsFileExists = false; + + vi.doMock('next.config.js', () => ({ + default: ( + _phase: string, + { defaultConfig }: { defaultConfig: { distDir: string } }, + ) => { + const nextConfig = { + distDir: `default___${defaultConfig.distDir}`, + }; + return nextConfig; + }, + })); + + const config = await getNextConfig(); + + expect(config).toEqual({ + distDir: 'default___.next', + }); + }); +}); From a539601f42740488315948a050befc5d38190b2a Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Wed, 27 Dec 2023 13:01:13 +0100 Subject: [PATCH 2/9] add unit tests for extractBuildMetadataConfig --- .../src/buildApplication/nextConfig.test.ts | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts index 5f22d4e8b..d2b884dbc 100644 --- a/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts +++ b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts @@ -1,5 +1,8 @@ import { vi, describe, test, expect } from 'vitest'; -import { getNextConfig } from '../../../src/buildApplication/nextConfig'; +import { + extractBuildMetadataConfig, + getNextConfig, +} from '../../../src/buildApplication/nextConfig'; const mocks = vi.hoisted(() => ({ nextConfigJsFileExists: true, @@ -86,3 +89,44 @@ describe('getNextConfigJs', () => { }); }); }); + +describe('extractBuildMetadataConfig', () => { + test('handles an empty object correctly', async () => { + expect(extractBuildMetadataConfig({})).toEqual({}); + }); + + test('extracts the desired metadata', async () => { + expect( + extractBuildMetadataConfig({ + experimental: { + allowedRevalidateHeaderKeys: ['a', 'b', 'c'], + fetchCacheKeyPrefix: 'my-prefix', + }, + }), + ).toEqual({ + experimental: { + allowedRevalidateHeaderKeys: ['a', 'b', 'c'], + fetchCacheKeyPrefix: 'my-prefix', + }, + }); + }); + + test('extract only the desired data', async () => { + expect( + extractBuildMetadataConfig({ + experimental: { + allowedRevalidateHeaderKeys: ['123'], + incrementalCacheHandlerPath: '../../../test', + }, + trailingSlash: true, + eslint: { + ignoreDuringBuilds: true, + }, + }), + ).toEqual({ + experimental: { + allowedRevalidateHeaderKeys: ['123'], + }, + }); + }); +}); From 482812690e75f408a9eaccdd5eef7e5a497fe676 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Wed, 27 Dec 2023 16:06:09 +0100 Subject: [PATCH 3/9] update caching doc --- packages/next-on-pages/docs/caching.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/next-on-pages/docs/caching.md b/packages/next-on-pages/docs/caching.md index 25d70f0eb..2100e199c 100644 --- a/packages/next-on-pages/docs/caching.md +++ b/packages/next-on-pages/docs/caching.md @@ -2,20 +2,24 @@ `@cloudflare/next-on-pages` comes with support for data revalidation and caching for fetch requests. This is done in our router and acts as an extension to Next.js' built-in functionality. -## Storage Options +## Zero Configuration Options -There are various different bindings and storage options that one could use for caching. At the moment, `@cloudflare/next-on-pages` supports the Cache API and Workers KV out-of-the-box. +The following are two different caching implementation options that don't require any code nor configuration change. -In the future, support will be available for creating custom cache interfaces and using different bindings. - -### Cache API +### Workers Cache API The [Cache API](https://developers.cloudflare.com/workers/runtime-apis/cache/) is a per data-center cache that is ideal for storing data that is not required to be accessible globally. It is worth noting that Vercel's Data Cache is regional, like with the Cache API, so there is no difference in terms of data availability. Due to how the Cache API works, you need to be using a domain for your deployment for it to take effect. +Besides the above requirement, no other action is needed to use this option. + ### Workers KV -[Workers KV](https://developers.cloudflare.com/kv/) is a low-latency key-value store that is ideal for storing data that should be globally distributed. KV is eventually consistent, which means that it will take up to 60 seconds for updates to be reflected globally. +[Workers KV](https://developers.cloudflare.com/kv/) is a low-latency key-value store that is ideal for storing data that should be globally distributed. KV is eventually consistent, which means that it can take up to 60 seconds for updates to be reflected globally. + +To use this option all you need to add a binding to your Pages project with the name `__NEXT_ON_PAGES__KV_SUSPENSE_CACHE`, and map it to a KV namespace. + +## Custom Cache Handler -To use Workers KV for caching, you need to add a binding to your Pages project with the name `__NEXT_ON_PAGES__KV_SUSPENSE_CACHE`, and map it to a KV namespace. +In case a custom solution is needed (for example in order to integrate with a third party storage solution or integrate with different [Cloudflare Bindings](https://developers.cloudflare.com/pages/functions/bindings/)) you will need to implement your own logic and register it via the Next.js [incrementalCacheHandlerPath](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) option. From cfdc265bc01cb30883f8947524df13a9ef3e1b7c Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Fri, 22 Dec 2023 18:31:23 +0100 Subject: [PATCH 4/9] update support (in docs and eslint) for incrementalCacheHandlerPath --- .../src/rules/no-unsupported-configs.ts | 2 +- .../tests/rules/no-unsupported-configs.test.ts | 4 ++-- packages/next-on-pages/docs/supported.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts b/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts index fcb0ba9cb..50b504c4c 100644 --- a/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts +++ b/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts @@ -25,7 +25,7 @@ const configs: Config[] = [ { name: 'headers', support: '✅' }, { name: 'httpAgentOptions', support: 'N/A' }, { name: 'images', support: '✅' }, - { name: 'incrementalCacheHandlerPath', support: '🔄' }, + { name: 'incrementalCacheHandlerPath', support: '✅' }, { name: 'logging', support: 'N/A' }, { name: 'experimental/mdxRs', support: '✅' }, { name: 'onDemandEntries', support: 'N/A' }, diff --git a/packages/eslint-plugin-next-on-pages/tests/rules/no-unsupported-configs.test.ts b/packages/eslint-plugin-next-on-pages/tests/rules/no-unsupported-configs.test.ts index d17d3e4f4..ef668959c 100644 --- a/packages/eslint-plugin-next-on-pages/tests/rules/no-unsupported-configs.test.ts +++ b/packages/eslint-plugin-next-on-pages/tests/rules/no-unsupported-configs.test.ts @@ -200,7 +200,7 @@ describe('no-unsupported-configs', () => { /** @type {import('next').NextConfig} */ const nextConfig = { assetPrefix: 'test', - incrementalCacheHandlerPath: true, + generateEtags: true, } module.exports = nextConfig @@ -225,7 +225,7 @@ describe('no-unsupported-configs', () => { }, { message: - 'The "incrementalCacheHandlerPath" configuration is not currently supported by next-on-pages.', + 'The "generateEtags" configuration is not currently supported by next-on-pages.', }, ], }, diff --git a/packages/next-on-pages/docs/supported.md b/packages/next-on-pages/docs/supported.md index 40a9acd66..b7c875e97 100644 --- a/packages/next-on-pages/docs/supported.md +++ b/packages/next-on-pages/docs/supported.md @@ -93,7 +93,7 @@ To check the latest state of the routers and possible missing features you can c | headers | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/headers), [app](https://nextjs.org/docs/app/api-reference/next-config-js/headers) | ✅ | | httpAgentOptions | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/httpAgentOptions), [app](https://nextjs.org/docs/app/api-reference/next-config-js/httpAgentOptions) | `N/A` | | images | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/images), [app](https://nextjs.org/docs/app/api-reference/next-config-js/images) | ✅ | -| incrementalCacheHandlerPath | [app](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) | 🔄 | +| incrementalCacheHandlerPath | [app](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) | ✅ | | logging | [app](https://nextjs.org/docs/app/api-reference/next-config-js/logging) | `N/A`5 | | mdxRs | [app](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs) | ✅ | | onDemandEntries | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/onDemandEntries), [app](https://nextjs.org/docs/app/api-reference/next-config-js/onDemandEntries) | `N/A`6 | From ede37395a8bf74eda8175f9298eb70c5465cce96 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 16 Jan 2024 16:46:23 +0000 Subject: [PATCH 5/9] fixup! update caching doc --- packages/next-on-pages/docs/caching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next-on-pages/docs/caching.md b/packages/next-on-pages/docs/caching.md index 2100e199c..c7b511683 100644 --- a/packages/next-on-pages/docs/caching.md +++ b/packages/next-on-pages/docs/caching.md @@ -18,7 +18,7 @@ Besides the above requirement, no other action is needed to use this option. [Workers KV](https://developers.cloudflare.com/kv/) is a low-latency key-value store that is ideal for storing data that should be globally distributed. KV is eventually consistent, which means that it can take up to 60 seconds for updates to be reflected globally. -To use this option all you need to add a binding to your Pages project with the name `__NEXT_ON_PAGES__KV_SUSPENSE_CACHE`, and map it to a KV namespace. +To use this option all you need to to is to add a binding to your Pages project with the name `__NEXT_ON_PAGES__KV_SUSPENSE_CACHE`, and map it to a KV namespace. ## Custom Cache Handler From 5e629dd905313a2cea330eb76f1fc7ac8184161e Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 16 Jan 2024 19:00:21 +0000 Subject: [PATCH 6/9] simplify config.experimental creation --- .../next-on-pages/src/buildApplication/nextConfig.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/next-on-pages/src/buildApplication/nextConfig.ts b/packages/next-on-pages/src/buildApplication/nextConfig.ts index ee1e3be0b..52bfd181f 100644 --- a/packages/next-on-pages/src/buildApplication/nextConfig.ts +++ b/packages/next-on-pages/src/buildApplication/nextConfig.ts @@ -202,11 +202,11 @@ export function extractBuildMetadataConfig( const config: NonNullable = {}; if (nextConfig.experimental) { - config.experimental = {}; - config.experimental.allowedRevalidateHeaderKeys = - nextConfig.experimental.allowedRevalidateHeaderKeys; - config.experimental.fetchCacheKeyPrefix = - nextConfig.experimental.fetchCacheKeyPrefix; + config.experimental = { + allowedRevalidateHeaderKeys: + nextConfig.experimental.allowedRevalidateHeaderKeys, + fetchCacheKeyPrefix: nextConfig.experimental.fetchCacheKeyPrefix, + }; } return config; From 689bad84a871f56e535afe08869c9ff707633615 Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 22 Jan 2024 10:31:53 +0000 Subject: [PATCH 7/9] move incrementalCacheHandlerPath check as suggested in PR review --- .../src/buildApplication/buildCacheFiles.ts | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts index cf0f99a85..4a31b34e0 100644 --- a/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts +++ b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts @@ -17,36 +17,34 @@ export async function buildCacheFiles( ): Promise { const outputCacheDir = join(nopDistDir, 'cache'); - const customCacheHandlerBuilt = await buildCustomIncrementalCacheHandler( - outputCacheDir, - minify, - ); + const nextConfig = await getNextConfig(); + + const incrementalCacheHandlerPath = + nextConfig?.experimental?.incrementalCacheHandlerPath; - if (!customCacheHandlerBuilt) { + if (incrementalCacheHandlerPath) { + await buildCustomIncrementalCacheHandler( + incrementalCacheHandlerPath, + outputCacheDir, + minify, + ); + } else { await buildBuiltInCacheHandlers(templatesDir, outputCacheDir, minify); } } /** - * Builds the file implementing the custom cache handler provided by the user, if one was provided. + * Builds the file implementing the custom cache handler provided by the user. * + * @param incrementalCacheHandlerPath path to the user defined incremental cache handler * @param outputCacheDir path to the directory in which to write the file - * @param minify flag indicating wether minification should be applied to the cache files - * @returns true if the file was built, false otherwise (meaning that the user has not provided a custom cache handler) + * @param minify flag indicating wether minification should be applied to the output file */ async function buildCustomIncrementalCacheHandler( + incrementalCacheHandlerPath: string, outputCacheDir: string, minify: boolean, -): Promise { - const nextConfig = await getNextConfig(); - - const incrementalCacheHandlerPath = - nextConfig?.experimental?.incrementalCacheHandlerPath; - - if (!incrementalCacheHandlerPath) { - return false; - } - +): Promise { try { await build({ entryPoints: [incrementalCacheHandlerPath], @@ -61,7 +59,6 @@ async function buildCustomIncrementalCacheHandler( `Failed to build custom incremental cache handler from the following provided path: ${incrementalCacheHandlerPath}`, ); } - return true; } /** From de839e042dcd0d060cdabefd181364cdfcc5828f Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 22 Jan 2024 10:44:59 +0000 Subject: [PATCH 8/9] introduce NextConfigExperimental as suggested in PR review --- packages/next-on-pages/build-metadata.d.ts | 6 ++---- .../next-on-pages/src/buildApplication/nextConfig.ts | 12 +++++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/next-on-pages/build-metadata.d.ts b/packages/next-on-pages/build-metadata.d.ts index 5e8c69fb7..4416e8236 100644 --- a/packages/next-on-pages/build-metadata.d.ts +++ b/packages/next-on-pages/build-metadata.d.ts @@ -7,10 +7,8 @@ type NextOnPagesBuildMetadata = { /** (subset of) values obtained from the user's next.config.js (if any was found) */ config?: { experimental?: Pick< - NonNullable< - // eslint-disable-next-line @typescript-eslint/consistent-type-imports -- the import needs to be dynamic since the nextConfig file itself uses this type - import('./src/buildApplication/nextConfig').NextConfig['experimental'] - >, + // eslint-disable-next-line @typescript-eslint/consistent-type-imports -- the import needs to be dynamic since the nextConfig file itself uses this type + import('./src/buildApplication/nextConfig').NextConfigExperimental, 'allowedRevalidateHeaderKeys' | 'fetchCacheKeyPrefix' >; }; diff --git a/packages/next-on-pages/src/buildApplication/nextConfig.ts b/packages/next-on-pages/src/buildApplication/nextConfig.ts index 52bfd181f..d24d530bd 100644 --- a/packages/next-on-pages/src/buildApplication/nextConfig.ts +++ b/packages/next-on-pages/src/buildApplication/nextConfig.ts @@ -9,11 +9,13 @@ import * as os from 'os'; * version of it which includes just what we need in next-on-pages. */ export type NextConfig = Record & { - experimental?: { - incrementalCacheHandlerPath?: string; - allowedRevalidateHeaderKeys?: string[]; - fetchCacheKeyPrefix?: string; - }; + experimental?: NextConfigExperimental; +}; + +export type NextConfigExperimental = { + incrementalCacheHandlerPath?: string; + allowedRevalidateHeaderKeys?: string[]; + fetchCacheKeyPrefix?: string; }; /** From dc6770c87d84c62547ede306d389b9cce9ca4e0a Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Tue, 23 Jan 2024 00:00:51 +0000 Subject: [PATCH 9/9] update experimental.incrementalCacheHandlerPath to cacheHandler --- .../src/rules/no-unsupported-configs.ts | 2 +- packages/next-on-pages/docs/caching.md | 2 +- packages/next-on-pages/docs/supported.md | 2 +- .../src/buildApplication/buildCacheFiles.ts | 15 +++++++-------- .../src/buildApplication/nextConfig.ts | 4 ++-- .../tests/src/buildApplication/nextConfig.test.ts | 10 +++------- 6 files changed, 15 insertions(+), 20 deletions(-) diff --git a/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts b/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts index 50b504c4c..60b8dcbb7 100644 --- a/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts +++ b/packages/eslint-plugin-next-on-pages/src/rules/no-unsupported-configs.ts @@ -25,7 +25,7 @@ const configs: Config[] = [ { name: 'headers', support: '✅' }, { name: 'httpAgentOptions', support: 'N/A' }, { name: 'images', support: '✅' }, - { name: 'incrementalCacheHandlerPath', support: '✅' }, + { name: 'cacheHandler', support: '✅' }, { name: 'logging', support: 'N/A' }, { name: 'experimental/mdxRs', support: '✅' }, { name: 'onDemandEntries', support: 'N/A' }, diff --git a/packages/next-on-pages/docs/caching.md b/packages/next-on-pages/docs/caching.md index c7b511683..56e7feb10 100644 --- a/packages/next-on-pages/docs/caching.md +++ b/packages/next-on-pages/docs/caching.md @@ -22,4 +22,4 @@ To use this option all you need to to is to add a binding to your Pages project ## Custom Cache Handler -In case a custom solution is needed (for example in order to integrate with a third party storage solution or integrate with different [Cloudflare Bindings](https://developers.cloudflare.com/pages/functions/bindings/)) you will need to implement your own logic and register it via the Next.js [incrementalCacheHandlerPath](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) option. +In case a custom solution is needed (for example in order to integrate with a third party storage solution or integrate with different [Cloudflare Bindings](https://developers.cloudflare.com/pages/functions/bindings/)) you will need to implement your own logic and register it via the Next.js [cacheHandler](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) option. diff --git a/packages/next-on-pages/docs/supported.md b/packages/next-on-pages/docs/supported.md index b7c875e97..fbfd46bbf 100644 --- a/packages/next-on-pages/docs/supported.md +++ b/packages/next-on-pages/docs/supported.md @@ -93,7 +93,7 @@ To check the latest state of the routers and possible missing features you can c | headers | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/headers), [app](https://nextjs.org/docs/app/api-reference/next-config-js/headers) | ✅ | | httpAgentOptions | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/httpAgentOptions), [app](https://nextjs.org/docs/app/api-reference/next-config-js/httpAgentOptions) | `N/A` | | images | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/images), [app](https://nextjs.org/docs/app/api-reference/next-config-js/images) | ✅ | -| incrementalCacheHandlerPath | [app](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) | ✅ | +| cacheHandler | [app](https://nextjs.org/docs/app/api-reference/next-config-js/incrementalCacheHandlerPath) | ✅ | | logging | [app](https://nextjs.org/docs/app/api-reference/next-config-js/logging) | `N/A`5 | | mdxRs | [app](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs) | ✅ | | onDemandEntries | [pages](https://nextjs.org/docs/pages/api-reference/next-config-js/onDemandEntries), [app](https://nextjs.org/docs/app/api-reference/next-config-js/onDemandEntries) | `N/A`6 | diff --git a/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts index 4a31b34e0..1bb8612a8 100644 --- a/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts +++ b/packages/next-on-pages/src/buildApplication/buildCacheFiles.ts @@ -19,12 +19,11 @@ export async function buildCacheFiles( const nextConfig = await getNextConfig(); - const incrementalCacheHandlerPath = - nextConfig?.experimental?.incrementalCacheHandlerPath; + const cacheHandlerPath = nextConfig?.cacheHandler; - if (incrementalCacheHandlerPath) { + if (cacheHandlerPath) { await buildCustomIncrementalCacheHandler( - incrementalCacheHandlerPath, + cacheHandlerPath, outputCacheDir, minify, ); @@ -36,18 +35,18 @@ export async function buildCacheFiles( /** * Builds the file implementing the custom cache handler provided by the user. * - * @param incrementalCacheHandlerPath path to the user defined incremental cache handler + * @param cacheHandlerPath path to the user defined incremental cache handler * @param outputCacheDir path to the directory in which to write the file * @param minify flag indicating wether minification should be applied to the output file */ async function buildCustomIncrementalCacheHandler( - incrementalCacheHandlerPath: string, + cacheHandlerPath: string, outputCacheDir: string, minify: boolean, ): Promise { try { await build({ - entryPoints: [incrementalCacheHandlerPath], + entryPoints: [cacheHandlerPath], bundle: true, target: 'es2022', platform: 'neutral', @@ -56,7 +55,7 @@ async function buildCustomIncrementalCacheHandler( }); } catch { throw new Error( - `Failed to build custom incremental cache handler from the following provided path: ${incrementalCacheHandlerPath}`, + `Failed to build custom incremental cache handler from the following provided path: ${cacheHandlerPath}`, ); } } diff --git a/packages/next-on-pages/src/buildApplication/nextConfig.ts b/packages/next-on-pages/src/buildApplication/nextConfig.ts index d24d530bd..f0d4e5d1b 100644 --- a/packages/next-on-pages/src/buildApplication/nextConfig.ts +++ b/packages/next-on-pages/src/buildApplication/nextConfig.ts @@ -9,11 +9,11 @@ import * as os from 'os'; * version of it which includes just what we need in next-on-pages. */ export type NextConfig = Record & { + cacheHandler?: string; experimental?: NextConfigExperimental; }; export type NextConfigExperimental = { - incrementalCacheHandlerPath?: string; allowedRevalidateHeaderKeys?: string[]; fetchCacheKeyPrefix?: string; }; @@ -165,7 +165,7 @@ const defaultConfig = { craCompat: false, esmExternals: true, isrMemoryCacheSize: 50 * 1024 * 1024, - incrementalCacheHandlerPath: undefined, + cacheHandlerPath: undefined, fullySpecified: false, outputFileTracingRoot: process.env['NEXT_PRIVATE_OUTPUT_TRACE_ROOT'] || '', swcTraceProfiling: false, diff --git a/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts index d2b884dbc..5b2427241 100644 --- a/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts +++ b/packages/next-on-pages/tests/src/buildApplication/nextConfig.test.ts @@ -34,18 +34,14 @@ describe('getNextConfigJs', () => { mocks.nextConfigMjsFileExists = false; vi.doMock('next.config.js', () => ({ default: { - experimental: { - incrementalCacheHandlerPath: 'my/incrementalCacheHandler/path', - }, + cacheHandlerPath: 'my/incrementalCacheHandler/path', }, })); const config = await getNextConfig(); expect(config).toEqual({ - experimental: { - incrementalCacheHandlerPath: 'my/incrementalCacheHandler/path', - }, + cacheHandlerPath: 'my/incrementalCacheHandler/path', }); }); @@ -114,9 +110,9 @@ describe('extractBuildMetadataConfig', () => { test('extract only the desired data', async () => { expect( extractBuildMetadataConfig({ + cacheHandlerPath: '../../../test', experimental: { allowedRevalidateHeaderKeys: ['123'], - incrementalCacheHandlerPath: '../../../test', }, trailingSlash: true, eslint: {