diff --git a/cypress/e2e/default/dynamic-routes.cy.ts b/cypress/e2e/default/dynamic-routes.cy.ts index 266f6faa43..4d7fcd1716 100644 --- a/cypress/e2e/default/dynamic-routes.cy.ts +++ b/cypress/e2e/default/dynamic-routes.cy.ts @@ -61,12 +61,13 @@ describe('Dynamic Routing', () => { expect(res.body).to.contain('Under the Dome') }) }) - it('renders fallback page via ODB on a non-prerendered dynamic route with fallback: true', () => { + it('does not render fallback page via ODB on a non-prerendered dynamic route with fallback: true', () => { + // unfortunately there is a problem with `fallback: true` in ODB context - the fallback would be cached indefinitely + // so visits to those pages would always render the fallback first and browser client would later re-render with correct + // content. As this is not ideal the `fallback: true` is treated as `fallback: blocking`. cy.request({ url: '/getStaticProps/withFallback/3/', headers: { 'x-nf-debug-logging': '1' } }).then((res) => { expect(res.status).to.eq(200) - // expect 'odb' until https://github.com/netlify/pillar-runtime/issues/438 is fixed expect(res.headers).to.have.property('x-nf-render-mode', 'odb') - // expect 'Bitten' until the above is fixed and we can test for fallback 'Loading...' message expect(res.body).to.contain('Bitten') }) }) @@ -115,12 +116,14 @@ describe('Dynamic Routing', () => { }, ) }) - it('renders fallback page via ODB on a non-prerendered dynamic route with revalidate and fallback: true', () => { + it('does not render fallback page via ODB on a non-prerendered dynamic route with revalidate and fallback: true', () => { + // unfortunately there is a problem with `fallback: true` in ODB context - the fallback would be cached indefinitely + // so visits to those pages would always render the fallback first and browser client would later re-render with correct + // content. As this is not ideal the `fallback: true` is treated as `fallback: blocking`. cy.request({ url: '/getStaticProps/withRevalidate/withFallback/3/', headers: { 'x-nf-debug-logging': '1' } }).then( (res) => { expect(res.status).to.eq(200) expect(res.headers).to.have.property('x-nf-render-mode', 'odb ttl=60') - // expect 'Bitten' until https://github.com/netlify/pillar-runtime/issues/438 is fixed expect(res.body).to.contain('Bitten') }, ) diff --git a/package-lock.json b/package-lock.json index 77c36bda3d..1fef96ba3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3887,6 +3887,14 @@ "integrity": "sha512-4wMPu9iN3/HL97QblBsBay3E1etIciR84izI3U+4iALY+JHCrI+a2jO0qbAZ/nxKoegypYEaiiqWXylm+/zfrw==", "dev": true }, + "node_modules/@netlify/blobs": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-2.2.0.tgz", + "integrity": "sha512-j2C0+IvWj9CLNGPoiA7ETquMFDExZTrv4CarjfE6Au0eY3zlinnnTVae7DE+VQFK+U0CDM/O0VvelNy1QbsdwQ==", + "engines": { + "node": "^14.16.0 || >=16.0.0" + } + }, "node_modules/@netlify/build": { "version": "29.22.5", "resolved": "https://registry.npmjs.org/@netlify/build/-/build-29.22.5.tgz", @@ -6307,7 +6315,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=8.0.0" } @@ -8862,13 +8870,13 @@ "version": "15.7.5", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", - "devOptional": true + "dev": true }, "node_modules/@types/react": { "version": "18.0.38", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.38.tgz", "integrity": "sha512-ExsidLLSzYj4cvaQjGnQCk4HFfVT9+EZ9XZsQ8Hsrcn8QNgXtpZ3m9vSIC2MWtx7jHictK6wYhQgGh6ic58oOw==", - "devOptional": true, + "dev": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -8894,7 +8902,7 @@ "version": "0.16.2", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", - "devOptional": true + "dev": true }, "node_modules/@types/semver": { "version": "7.3.13", @@ -12483,7 +12491,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==", - "devOptional": true + "dev": true }, "node_modules/custom-routes": { "resolved": "demos/custom-routes", @@ -17038,7 +17046,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.1.0.tgz", "integrity": "sha512-oNkuqVTA8jqG1Q6c+UglTOD1xhC1BtjKI7XkCXRkZHrN5m18/XsnUp8Q89GkQO/z+0WjonSvl0FLhDYftp46nQ==", - "devOptional": true + "dev": true }, "node_modules/import-fresh": { "version": "3.3.0", @@ -24376,7 +24384,7 @@ "version": "1.56.1", "resolved": "https://registry.npmjs.org/sass/-/sass-1.56.1.tgz", "integrity": "sha512-VpEyKpyBPCxE7qGDtOcdJ6fFbcpOM+Emu7uZLxVrkX8KVU/Dp5UF7WLvzqRuUhB6mqqQt1xffLoG+AndxTZrCQ==", - "devOptional": true, + "dev": true, "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -27456,6 +27464,7 @@ "version": "4.40.2", "license": "MIT", "dependencies": { + "@netlify/blobs": "^2.2.0", "@netlify/esbuild": "0.14.39", "@netlify/functions": "^1.6.0", "@netlify/ipx": "^1.4.5", @@ -30114,6 +30123,11 @@ "integrity": "sha512-4wMPu9iN3/HL97QblBsBay3E1etIciR84izI3U+4iALY+JHCrI+a2jO0qbAZ/nxKoegypYEaiiqWXylm+/zfrw==", "dev": true }, + "@netlify/blobs": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@netlify/blobs/-/blobs-2.2.0.tgz", + "integrity": "sha512-j2C0+IvWj9CLNGPoiA7ETquMFDExZTrv4CarjfE6Au0eY3zlinnnTVae7DE+VQFK+U0CDM/O0VvelNy1QbsdwQ==" + }, "@netlify/build": { "version": "29.22.5", "resolved": "https://registry.npmjs.org/@netlify/build/-/build-29.22.5.tgz", @@ -31291,6 +31305,7 @@ "version": "file:packages/runtime", "requires": { "@delucis/if-env": "^1.1.2", + "@netlify/blobs": "^2.2.0", "@netlify/build": "^29.22.5", "@netlify/esbuild": "0.14.39", "@netlify/functions": "^1.6.0", @@ -31712,7 +31727,7 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", - "devOptional": true + "dev": true }, "@opentelemetry/api-logs": { "version": "0.41.2", @@ -31727,8 +31742,7 @@ "version": "1.13.0", "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.13.0.tgz", "integrity": "sha512-pS5fU4lrRjOIPZQqA2V1SUM9QUFXbO+8flubAiy6ntLjnAjJJUdRFOUOxK6v86ZHI2p2S8A0vD0BTu95FZYvjA==", - "dev": true, - "requires": {} + "dev": true }, "@opentelemetry/core": { "version": "1.17.0", @@ -33465,13 +33479,13 @@ "version": "15.7.5", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", - "devOptional": true + "dev": true }, "@types/react": { "version": "18.0.38", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.0.38.tgz", "integrity": "sha512-ExsidLLSzYj4cvaQjGnQCk4HFfVT9+EZ9XZsQ8Hsrcn8QNgXtpZ3m9vSIC2MWtx7jHictK6wYhQgGh6ic58oOw==", - "devOptional": true, + "dev": true, "requires": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -33497,7 +33511,7 @@ "version": "0.16.2", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", - "devOptional": true + "dev": true }, "@types/semver": { "version": "7.3.13", @@ -33811,8 +33825,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} + "dev": true }, "acorn-walk": { "version": "7.2.0", @@ -33869,8 +33882,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-3.0.0.tgz", "integrity": "sha512-V3wD15YHfHz6y0KdhYFjyy9vWtEVALT9UrxfN3zqlI6dMioHnJrqOYfyPKol3oqrnCM9uwkcdCwkJ0WUcbLMTQ==", - "dev": true, - "requires": {} + "dev": true }, "ansi-color": { "version": "0.2.1", @@ -35656,8 +35668,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.2.0.tgz", "integrity": "sha512-NkANeMnaHrlaSSlpKGyvn2R4rqUDeE/9E5YHx+b4nwo0R8dZyAqcih8/gxpCZvqWP9Vf6xuLpMSzSgdVEIM78g==", - "dev": true, - "requires": {} + "dev": true }, "cp-file": { "version": "10.0.0", @@ -36221,7 +36232,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", "integrity": "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw==", - "devOptional": true + "dev": true }, "custom-routes": { "version": "file:demos/custom-routes", @@ -37463,15 +37474,13 @@ "version": "8.5.0", "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-8.5.0.tgz", "integrity": "sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q==", - "dev": true, - "requires": {} + "dev": true }, "eslint-config-standard": { "version": "17.0.0", "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-17.0.0.tgz", "integrity": "sha512-/2ks1GKyqSOkH7JFvXJicu0iMpoojkwB+f5Du/1SC0PtBL+s8v30k9njRZ21pm2drKYm2342jFnGWzttxPmZVg==", - "dev": true, - "requires": {} + "dev": true }, "eslint-formatter-codeframe": { "version": "7.32.1", @@ -37926,8 +37935,7 @@ "version": "6.1.1", "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.1.1.tgz", "integrity": "sha512-tjqWDwVZQo7UIPMeDReOpUgHCmCiH+ePnVT+5zVapL0uuHnegBUs2smM13CzOs2Xb5+MHMRFTs9v24yjba4Oig==", - "dev": true, - "requires": {} + "dev": true }, "eslint-plugin-react": { "version": "7.31.10", @@ -37985,8 +37993,7 @@ "version": "4.6.0", "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz", "integrity": "sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g==", - "dev": true, - "requires": {} + "dev": true }, "eslint-plugin-unicorn": { "version": "43.0.2", @@ -38613,8 +38620,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.0.1.tgz", "integrity": "sha512-bdrUUb0eYQrPRlaAtlSRoLs7sp6yKEwbMQuUgwvi/14TnaqhM/deSZUrC5ic+yjm5nEPPWE61oWpTTxQFQMmLA==", - "dev": true, - "requires": {} + "dev": true }, "fetch-blob": { "version": "3.2.0", @@ -39620,7 +39626,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.1.0.tgz", "integrity": "sha512-oNkuqVTA8jqG1Q6c+UglTOD1xhC1BtjKI7XkCXRkZHrN5m18/XsnUp8Q89GkQO/z+0WjonSvl0FLhDYftp46nQ==", - "devOptional": true + "dev": true }, "import-fresh": { "version": "3.3.0", @@ -40793,8 +40799,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "requires": {} + "dev": true }, "jest-regex-util": { "version": "27.5.1", @@ -44337,8 +44342,7 @@ "version": "8.9.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.9.0.tgz", "integrity": "sha512-Ja7nszREasGaYUYCI2k4lCKIRTt+y7XuqVoHR44YpI49TtryyqbqvDMn5eqfW7e6HzTukDRIsXqzVHScqRcafg==", - "dev": true, - "requires": {} + "dev": true } } }, @@ -45158,7 +45162,7 @@ "version": "1.56.1", "resolved": "https://registry.npmjs.org/sass/-/sass-1.56.1.tgz", "integrity": "sha512-VpEyKpyBPCxE7qGDtOcdJ6fFbcpOM+Emu7uZLxVrkX8KVU/Dp5UF7WLvzqRuUhB6mqqQt1xffLoG+AndxTZrCQ==", - "devOptional": true, + "dev": true, "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -46933,8 +46937,7 @@ "ws": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", - "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", - "requires": {} + "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==" } } }, @@ -47372,8 +47375,7 @@ "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", - "dev": true, - "requires": {} + "dev": true }, "xml": { "version": "1.0.1", diff --git a/packages/runtime/package.json b/packages/runtime/package.json index aa56ace2ae..08e6a35f06 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -12,6 +12,7 @@ "manifest.yml" ], "dependencies": { + "@netlify/blobs": "^2.2.0", "@netlify/esbuild": "0.14.39", "@netlify/functions": "^1.6.0", "@netlify/ipx": "^1.4.5", diff --git a/packages/runtime/src/constants.ts b/packages/runtime/src/constants.ts index 2a9fd1d9c6..853da51327 100644 --- a/packages/runtime/src/constants.ts +++ b/packages/runtime/src/constants.ts @@ -33,6 +33,7 @@ export const HIDDEN_PATHS = destr(process.env.NEXT_KEEP_METADATA_FILES) '/prerender-manifest.js', '/required-server-files.json', '/static-manifest.json', + '/blobs-manifest.json', ] export const ODB_FUNCTION_PATH = `/.netlify/builders/${ODB_FUNCTION_NAME}` diff --git a/packages/runtime/src/helpers/config.ts b/packages/runtime/src/helpers/config.ts index dc34fb3ed8..4bb98eb7b7 100644 --- a/packages/runtime/src/helpers/config.ts +++ b/packages/runtime/src/helpers/config.ts @@ -37,6 +37,9 @@ const defaultFailBuild = (message: string, { error }): never => { export const getNextConfig = async function getNextConfig({ publish, failBuild = defaultFailBuild, +}: { + publish: string + failBuild?: (message: string, { error }) => never }): Promise { try { const { config, appDir, ignore }: RequiredServerFiles = await readJSON(join(publish, 'required-server-files.json')) diff --git a/packages/runtime/src/helpers/files.ts b/packages/runtime/src/helpers/files.ts index 7449d47773..403732c2c4 100644 --- a/packages/runtime/src/helpers/files.ts +++ b/packages/runtime/src/helpers/files.ts @@ -1,8 +1,9 @@ import { cpus } from 'os' +import type { Blobs } from '@netlify/blobs' import type { NetlifyConfig } from '@netlify/build/types' import { yellowBright } from 'chalk' -import { existsSync, readJson, move, copy, writeJson, ensureDir, readFileSync, remove } from 'fs-extra' +import { existsSync, readJson, move, copy, writeJson, ensureDir, readFileSync, remove, readFile } from 'fs-extra' import globby from 'globby' import { PrerenderManifest } from 'next/dist/build' import { outdent } from 'outdent' @@ -11,6 +12,7 @@ import { join, resolve, dirname } from 'pathe' import slash from 'slash' import { MINIMUM_REVALIDATE_SECONDS, DIVIDER, HIDDEN_PATHS } from '../constants' +import { getNormalizedBlobKey } from '../templates/blobStorage' import { NextConfig } from './config' import { loadPrerenderManifest } from './edge' @@ -75,14 +77,15 @@ export const getMiddleware = async (publish: string): Promise> => export const moveStaticPages = async ({ netlifyConfig, target, - i18n, - basePath, + nextConfig, + netliBlob, }: { netlifyConfig: NetlifyConfig target: 'server' | 'serverless' | 'experimental-serverless-trace' - i18n: NextConfig['i18n'] - basePath?: string + nextConfig: Pick + netliBlob?: Blobs }): Promise => { + const { i18n, basePath } = nextConfig console.log('Moving static page files to serve from CDN...') const outputDir = join(netlifyConfig.build.publish, target === 'serverless' ? 'serverless' : 'server') const buildId = readFileSync(join(netlifyConfig.build.publish, 'BUILD_ID'), 'utf8').trim() @@ -117,16 +120,28 @@ export const moveStaticPages = async ({ }) let fileCount = 0 + let blobCount = 0 const filesManifest: Record = {} - const moveFile = async (file: string) => { + const blobsManifest = new Set() + + const getSourceAndTargetPath = (file: string): { targetPath: string; source: string } => { + const source = join(outputDir, file) // Strip the initial 'app' or 'pages' directory from the output path const pathname = file.split('/').slice(1).join('/') // .rsc data files go next to the html file const isData = file.endsWith('.json') - const source = join(outputDir, file) const targetFile = isData ? join(dataDir, pathname) : pathname const targetPath = basePath ? join(basePath, targetFile) : targetFile + return { + targetPath, + source, + } + } + + const moveFile = async (file: string) => { + const { source, targetPath } = getSourceAndTargetPath(file) + fileCount += 1 filesManifest[file] = targetPath @@ -138,6 +153,18 @@ export const moveStaticPages = async ({ console.warn('Error moving file', source, error) } } + const uploadFileToBlobStorageAndDelete = async (file: string) => { + const { source } = getSourceAndTargetPath(file) + + blobsManifest.add(file) + + const content = await readFile(source, 'utf8') + + blobCount += 1 + + await netliBlob.set(getNormalizedBlobKey(file), content) + await remove(source) + } // Move all static files, except nft manifests const pages = await globby(['{app,pages}/**/*.{html,json,rsc}', '!**/*.js.nft.{html,json}'], { cwd: outputDir, @@ -156,8 +183,14 @@ export const moveStaticPages = async ({ const filePath = slash(rawPath) // Remove the initial 'app' or 'pages' directory from the output path const pagePath = filePath.split('/').slice(1).join('/') - // Don't move ISR files, as they're used for the first request + if (isrFiles.has(pagePath)) { + if (netliBlob) { + // if netliblob is enabled, we will move ISR assets to blob store and delete files after that + // to minimize the number of files needed in lambda + return limit(uploadFileToBlobStorageAndDelete, filePath) + } + // Don't move ISR files, as they're used for the first request return } if (isDynamicRoute(pagePath)) { @@ -183,7 +216,10 @@ export const moveStaticPages = async ({ return limit(moveFile, filePath) }) await Promise.all(promises) - console.log(`Moved ${fileCount} files`) + console.log(`Moved ${fileCount} files to CDN`) + if (netliBlob) { + console.log(`Moved ${blobCount} files to Blob Storage`) + } if (matchedPages.size !== 0) { console.log( @@ -246,8 +282,9 @@ export const moveStaticPages = async ({ } } - // Write the manifest for use in the serverless functions + // Write the manifests for use in the serverless functions await writeJson(join(netlifyConfig.build.publish, 'static-manifest.json'), Object.entries(filesManifest)) + await writeJson(join(netlifyConfig.build.publish, 'blobs-manifest.json'), [...blobsManifest]) if (i18n?.defaultLocale) { const rootPath = basePath ? join(netlifyConfig.build.publish, basePath) : netlifyConfig.build.publish diff --git a/packages/runtime/src/helpers/flags.ts b/packages/runtime/src/helpers/flags.ts index 368895f61f..cf044cb2b0 100644 --- a/packages/runtime/src/helpers/flags.ts +++ b/packages/runtime/src/helpers/flags.ts @@ -36,3 +36,10 @@ export const bundleBasedOnNftFiles = (featureFlags: Record): bo return isEnabled } + +export const useBlobsForISRAssets = (featureFlags: Record): boolean => { + const isEnabled = + destr(process.env.NEXT_USE_BLOBS_FOR_ISR) ?? featureFlags['next-runtime-use-blobs-for-isr-assets'] ?? false + + return isEnabled +} diff --git a/packages/runtime/src/helpers/functions.ts b/packages/runtime/src/helpers/functions.ts index f7ded83ed1..ed38c59556 100644 --- a/packages/runtime/src/helpers/functions.ts +++ b/packages/runtime/src/helpers/functions.ts @@ -159,6 +159,10 @@ export const generateFunctions = async ( join(__dirname, '..', '..', 'lib', 'templates', 'handlerUtils.js'), join(functionsDir, functionName, 'handlerUtils.js'), ) + await copyFile( + join(__dirname, '..', '..', 'lib', 'templates', 'blobStorage.js'), + join(functionsDir, functionName, 'blobStorage.js'), + ) await writeFunctionConfiguration({ functionName, functionTitle, functionsDir }) const nfInternalFiles = await glob(join(functionsDir, functionName, '**')) @@ -333,6 +337,8 @@ export const getCommonDependencies = async (publish: string) => { // using package.json because otherwise, we'd find some /dist/... path traceNPMPackage('@netlify/functions/package.json', publish), + traceNPMPackage('@netlify/blobs/package.json', publish), + traceNPMPackage('is-promise', publish), ]) @@ -390,6 +396,7 @@ const getSSRDependencies = async (publish: string): Promise => { }), join(publish, '**', '*.html'), join(publish, 'static-manifest.json'), + join(publish, 'blobs-manifest.json'), ] } diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index a69628fbee..7dca0b8c9e 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -1,6 +1,6 @@ import { join, relative } from 'path' -import type { NetlifyPlugin, NetlifyPluginOptions } from '@netlify/build/types' +import type { NetlifyPlugin, NetlifyPluginConstants, NetlifyPluginOptions } from '@netlify/build/types' import { bold, redBright } from 'chalk' import destr from 'destr' import { existsSync, readFileSync } from 'fs-extra' @@ -18,7 +18,7 @@ import { import { onPreDev } from './helpers/dev' import { writeEdgeFunctions, loadMiddlewareManifest, cleanupEdgeFunctions } from './helpers/edge' import { moveStaticPages, movePublicFiles, removeMetadataFiles } from './helpers/files' -import { bundleBasedOnNftFiles, splitApiRoutes } from './helpers/flags' +import { bundleBasedOnNftFiles, splitApiRoutes, useBlobsForISRAssets } from './helpers/flags' import { generateFunctions, setupImageFunction, @@ -47,6 +47,16 @@ import { warnForProblematicUserRewrites, warnForRootRedirects, } from './helpers/verification' +import { Blobs, isBlobStorageAvailable } from './templates/blobStorage' + +type EnhancedNetlifyPluginConstants = NetlifyPluginConstants & { + NETLIFY_API_HOST?: string + NETLIFY_API_TOKEN?: string +} + +type EnhancedNetlifyPluginOptions = NetlifyPluginOptions & { constants: EnhancedNetlifyPluginConstants } & { + featureFlags?: Record +} const plugin: NetlifyPlugin = { async onPreBuild({ @@ -84,7 +94,7 @@ const plugin: NetlifyPlugin = { build: { failBuild }, }, featureFlags = {}, - }: NetlifyPluginOptions & { featureFlags?: Record }) { + }: EnhancedNetlifyPluginOptions) { if (shouldSkip()) { return } @@ -195,7 +205,24 @@ const plugin: NetlifyPlugin = { await movePublicFiles({ appDir, outdir, publish, basePath }) if (!destr(process.env.SERVE_STATIC_FILES_FROM_ORIGIN)) { - await moveStaticPages({ target, netlifyConfig, i18n, basePath }) + const useBlobs = useBlobsForISRAssets(featureFlags) + + const { NETLIFY_API_HOST, NETLIFY_API_TOKEN, SITE_ID } = constants + + const testBlobStorage = useBlobs + ? new Blobs({ + authentication: { + apiURL: `https://${NETLIFY_API_HOST}`, + token: NETLIFY_API_TOKEN, + }, + context: `deploy:${process.env.DEPLOY_ID}`, + siteID: SITE_ID, + }) + : undefined + + const netliBlob = testBlobStorage && (await isBlobStorageAvailable(testBlobStorage)) ? testBlobStorage : undefined + + await moveStaticPages({ target, netlifyConfig, nextConfig: { basePath, i18n }, netliBlob }) } await generateStaticRedirects({ diff --git a/packages/runtime/src/templates/blobStorage.ts b/packages/runtime/src/templates/blobStorage.ts new file mode 100644 index 0000000000..0481234e14 --- /dev/null +++ b/packages/runtime/src/templates/blobStorage.ts @@ -0,0 +1,30 @@ +import { Buffer } from 'buffer' + +import { Blobs } from '@netlify/blobs' + +export const isBlobStorageAvailable = async (netliBlob: Blobs) => { + try { + // request a key that is not present. If it returns `null` then the blob storage is available + // if it throws it's not available. + await netliBlob.get('any-key') + return true + } catch { + return false + } +} + +type BlobsInit = ConstructorParameters[0] + +let blobInit: BlobsInit +export const setBlobInit = (init: BlobsInit): void => { + blobInit = init +} +export const getBlobInit = (): BlobsInit => blobInit + +/** + * @netlify/blobs ATM has some limitation to keys, so we need to normalize it for now (they will be resolved so we will be able to remove this code) + */ +export const getNormalizedBlobKey = (key: string): string => Buffer.from(key).toString('base64url') + +// eslint-disable-next-line unicorn/prefer-export-from -- we are both using and re-exporting Blobs here for simplicity of importing Blobs and our helpers from same module +export { Blobs } diff --git a/packages/runtime/src/templates/getHandler.ts b/packages/runtime/src/templates/getHandler.ts index f04be6bae5..5e59ce22b5 100644 --- a/packages/runtime/src/templates/getHandler.ts +++ b/packages/runtime/src/templates/getHandler.ts @@ -1,4 +1,4 @@ -import { HandlerContext, HandlerEvent } from '@netlify/functions' +import type { HandlerContext, HandlerEvent } from '@netlify/functions' import type { Bridge as NodeBridge } from '@vercel/node-bridge/bridge' // Aliasing like this means the editor may be able to syntax-highlight the string import { outdent as javascript } from 'outdent' @@ -10,6 +10,7 @@ import type { NextServerType } from './handlerUtils' import type { NetlifyNextServerType } from './server' /* eslint-disable @typescript-eslint/no-var-requires */ +const { Buffer } = require('buffer') const { promises } = require('fs') const { Server } = require('http') const path = require('path') @@ -18,13 +19,9 @@ const { URLSearchParams, URL } = require('url') const { Bridge } = require('@vercel/node-bridge/bridge') -const { - augmentFsModule, - getMaxAge, - getMultiValueHeaders, - getPrefetchResponse, - normalizePath, -} = require('./handlerUtils') +const { setBlobInit } = require('./blobStorage') as typeof import('./blobStorage') +const { augmentFsModule, getMaxAge, getMultiValueHeaders, getPrefetchResponse, normalizePath } = + require('./handlerUtils') as typeof import('./handlerUtils') const { overrideRequireHooks, applyRequireHooks } = require('./requireHooks') const { getNetlifyNextServer } = require('./server') /* eslint-enable @typescript-eslint/no-var-requires */ @@ -39,6 +36,7 @@ type MakeHandlerParams = { pageRoot: string NextServer: NextServerType staticManifest: Array<[string, string]> + blobsManifest: Set mode: 'ssr' | 'odb' useHooks: boolean } @@ -51,6 +49,7 @@ const makeHandler = ({ pageRoot, NextServer, staticManifest = [], + blobsManifest = new Set(), mode = 'ssr', useHooks, }: MakeHandlerParams) => { @@ -88,7 +87,7 @@ const makeHandler = ({ // Set during the request as it needs to get it from the request URL. Defaults to the URL env var let base = process.env.URL - augmentFsModule({ promises, staticManifest, pageRoot, getBase: () => base }) + augmentFsModule({ promises, staticManifest, blobsManifest, pageRoot, getBase: () => base }) // We memoize this because it can be shared between requests, but don't instantiate it until // the first request because we need the host and port. @@ -153,6 +152,19 @@ const makeHandler = ({ event.headers['accept-language'] = event.headers['accept-language'].replace(/\s*,.*$/, '') } + if (context?.clientContext?.custom?.blobs) { + const rawData = Buffer.from(context.clientContext.custom.blobs, 'base64') + const data = JSON.parse(rawData.toString('ascii')) + setBlobInit({ + authentication: { + contextURL: data.url, + token: data.token, + }, + context: `deploy:${event.headers['x-nf-deploy-id']}`, + siteID: event.headers['x-nf-site-id'], + }) + } + const { headers, ...result } = await getBridge(event, context).launcher(event, context) // Convert all headers to multiValueHeaders @@ -228,8 +240,10 @@ export const getHandler = ({ process.env.NODE_ENV = 'production'; + const { Buffer } = require('buffer') const { Server } = require("http"); const { promises } = require("fs"); + const { setBlobInit } = require('./blobStorage') // We copy the file here rather than requiring from the node module const { Bridge } = require("./bridge"); const { augmentFsModule, getMaxAge, getMultiValueHeaders, getPrefetchResponse, normalizePath, nextVersionNum } = require('./handlerUtils') @@ -242,11 +256,15 @@ export const getHandler = ({ try { staticManifest = require("${publishDir}/static-manifest.json") } catch {} + let blobsManifest + try { + blobsManifest = new Set(require("${publishDir}/blobs-manifest.json")) + } catch {} const path = require("path"); const pageRoot = path.resolve(path.join(__dirname, "${publishDir}", "server")); exports.handler = ${ isODB - ? `builder((${makeHandler.toString()})({ conf: config, app: "${appDir}", pageRoot, NextServer, staticManifest, mode: 'odb', useHooks: ${useHooks}}));` - : `(${makeHandler.toString()})({ conf: config, app: "${appDir}", pageRoot, NextServer, staticManifest, mode: 'ssr', useHooks: ${useHooks}});` + ? `builder((${makeHandler.toString()})({ conf: config, app: "${appDir}", pageRoot, NextServer, staticManifest, blobsManifest, mode: 'odb', useHooks: ${useHooks}}));` + : `(${makeHandler.toString()})({ conf: config, app: "${appDir}", pageRoot, NextServer, staticManifest, blobsManifest, mode: 'ssr', useHooks: ${useHooks}});` } ` diff --git a/packages/runtime/src/templates/handlerUtils.ts b/packages/runtime/src/templates/handlerUtils.ts index b905293220..babcd0bf5d 100644 --- a/packages/runtime/src/templates/handlerUtils.ts +++ b/packages/runtime/src/templates/handlerUtils.ts @@ -1,4 +1,4 @@ -import fs, { createWriteStream, existsSync } from 'fs' +import fs, { createWriteStream, existsSync, writeFile } from 'fs' import { ServerResponse } from 'http' import { tmpdir } from 'os' import path from 'path' @@ -14,13 +14,14 @@ import type { StaticRoute } from '../helpers/types' export type NextServerType = typeof NextNodeServer const streamPipeline = promisify(pipeline) +const writeFilePromisified = promisify(writeFile) /** * Downloads a file from the CDN to the local aliased filesystem. This is a fallback, because in most cases we'd expect * files required at runtime to not be sent to the CDN. */ -export const downloadFile = async (url: string, destination: string): Promise => { - console.log(`Downloading ${url} to ${destination}`) +export const downloadFileFromCDN = async (url: string, destination: string): Promise => { + console.log(`Downloading ${url} from CDN to ${destination}`) const httpx = url.startsWith('https') ? https : http @@ -45,6 +46,22 @@ export const downloadFile = async (url: string, destination: string): Promise => { + console.log(`Downloading ${filePath} from Blobs Storage to ${destination}`) + + const { Blobs, getNormalizedBlobKey, getBlobInit } = + // eslint-disable-next-line @typescript-eslint/no-var-requires + require('./blobStorage') as typeof import('./blobStorage') + + const netliBlob = new Blobs(getBlobInit()) + + const blobKey = getNormalizedBlobKey(filePath) + + const fileContent = await netliBlob.get(blobKey, { type: 'text' }) + + await writeFilePromisified(destination, fileContent) +} + /** * Parse maxage from a cache-control header */ @@ -86,11 +103,13 @@ export const getMultiValueHeaders = ( export const augmentFsModule = ({ promises, staticManifest, + blobsManifest, pageRoot, getBase, }: { promises: typeof fs.promises staticManifest: Array<[string, string]> + blobsManifest: Set pageRoot: string getBase: () => string }) => { @@ -109,6 +128,11 @@ export const augmentFsModule = ({ // Grab the real fs.promises.readFile... const readfileOrig = promises.readFile const statsOrig = promises.stat + + const { getBlobInit } = + // eslint-disable-next-line @typescript-eslint/no-var-requires + require('./blobStorage') as typeof import('./blobStorage') + // ...then monkey-patch it to see if it's requesting a CDN file promises.readFile = (async (file, options) => { const baseUrl = getBase() @@ -118,31 +142,37 @@ export const augmentFsModule = ({ // We only want the part after `.next/server/` const filePath = file.slice(pageRoot.length + 1) - // Is it in the CDN and not local? - if (staticFiles.has(filePath) && !existsSync(file)) { - // This name is safe to use, because it's one that was already created by Next - const cacheFile = path.join(cacheDir, filePath) - const url = `${baseUrl}/${staticFiles.get(filePath)}` + if (!existsSync(file)) { + // Is it in the CDN or Blobs Storage and not local? + const isStatic = staticFiles.has(filePath) + const isBlob = getBlobInit() ? blobsManifest.has(filePath) : false + if (isStatic || isBlob) { + // This name is safe to use, because it's one that was already created by Next + const cacheFile = path.join(cacheDir, filePath) + const url = `${baseUrl}/${staticFiles.get(filePath)}` - // If it's already downloading we can wait for it to finish - if (downloadPromises.has(url)) { - await downloadPromises.get(url) - } - // Have we already cached it? We download every time if running locally to avoid staleness - if ((!existsSync(cacheFile) || process.env.NETLIFY_DEV) && baseUrl) { - await promises.mkdir(path.dirname(cacheFile), { recursive: true }) - - try { - // Append the path to our host and we can load it like a regular page - const downloadPromise = downloadFile(url, cacheFile) - downloadPromises.set(url, downloadPromise) - await downloadPromise - } finally { - downloadPromises.delete(url) + // If it's already downloading we can wait for it to finish + if (downloadPromises.has(url)) { + await downloadPromises.get(url) + } + // Have we already cached it? We download every time if running locally to avoid staleness + if ((!existsSync(cacheFile) || process.env.NETLIFY_DEV) && baseUrl) { + await promises.mkdir(path.dirname(cacheFile), { recursive: true }) + + try { + // Append the path to our host and we can load it like a regular page + const downloadPromise = isStatic + ? downloadFileFromCDN(url, cacheFile) + : downloadFileFromBlobs(filePath, cacheFile) + downloadPromises.set(url, downloadPromise) + await downloadPromise + } finally { + downloadPromises.delete(url) + } } + // Return the cache file + return readfileOrig(cacheFile, options) } - // Return the cache file - return readfileOrig(cacheFile, options) } } diff --git a/packages/runtime/src/templates/server.ts b/packages/runtime/src/templates/server.ts index dc137afcc1..057117c5fe 100644 --- a/packages/runtime/src/templates/server.ts +++ b/packages/runtime/src/templates/server.ts @@ -47,6 +47,28 @@ const getNetlifyNextServer = (NextServer: NextServerType) => { } } + protected getPrerenderManifest(): PrerenderManifest { + const manifest = super.getPrerenderManifest() + + if (typeof manifest?.dynamicRoutes === 'object') { + for (const route of Object.values(manifest.dynamicRoutes)) { + // 'fallback' property is: + // - a string when fallback: true is used + // - `null` when fallback: blocking is used + // - `false` when fallback: false is used + // `fallback: true` is not working correctly with ODBs + // as we will cache fallback html forever, so + // we are treating those as `fallback: blocking` + // by editing the manifest + if (typeof route.fallback === 'string') { + route.fallback = null + } + } + } + + return manifest + } + public getRequestHandler(): NodeRequestHandler { const handler = super.getRequestHandler() return async (req, res, parsedUrl) => { @@ -79,9 +101,13 @@ const getNetlifyNextServer = (NextServer: NextServerType) => { // but ignore in preview mode (prerender_bypass is set to true in preview mode) // because otherwise revalidate will override preview mode if (!headers.cookie?.includes('__prerender_bypass')) { - // this header controls whether Next.js will revalidate the page - // and needs to be set to the preview mode id to enable it - headers['x-prerender-revalidate'] = this.renderOpts.previewProps.previewModeId + const isFirstODBRequest = headers['x-nf-builder-cache'] === 'miss' + // first ODB request should NOT be revalidated and instead it should try to serve cached version + if (!isFirstODBRequest) { + // this header controls whether Next.js will revalidate the page + // and needs to be set to the preview mode id to enable it + headers['x-prerender-revalidate'] = this.renderOpts.previewProps.previewModeId + } } return handler(req, res, parsedUrl) diff --git a/test/e2e/next-test-lib/fetch-polyfill.js b/test/e2e/next-test-lib/fetch-polyfill.js new file mode 100644 index 0000000000..c7fd4c5b94 --- /dev/null +++ b/test/e2e/next-test-lib/fetch-polyfill.js @@ -0,0 +1,8 @@ +if (!globalThis.fetch) { + const fetch = require('node-fetch') + + globalThis.fetch = fetch + globalThis.Headers = fetch.Headers + globalThis.Request = fetch.Request + globalThis.Response = fetch.Response +} diff --git a/test/e2e/next-test-lib/next-modes/next-deploy.ts b/test/e2e/next-test-lib/next-modes/next-deploy.ts index 1463cbc1af..3a2fd0aee1 100644 --- a/test/e2e/next-test-lib/next-modes/next-deploy.ts +++ b/test/e2e/next-test-lib/next-modes/next-deploy.ts @@ -75,6 +75,7 @@ export class NextDeployInstance extends NextInstance { NODE_ENV: 'production', DISABLE_IPX: platform() === 'linux' ? undefined : '1', NEXT_KEEP_METADATA_FILES: 'true', + NODE_OPTIONS: `--require ${require.resolve('../fetch-polyfill.js')}` }, }) diff --git a/test/index.spec.ts b/test/index.spec.ts index d095709c76..763fb3ab6d 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -559,10 +559,10 @@ describe('onBuild()', () => { expect(existsSync(odbHandlerFile)).toBeTruthy() expect(readFileSync(handlerFile, 'utf8')).toMatch( - `({ conf: config, app: "../../..", pageRoot, NextServer, staticManifest, mode: 'ssr', useHooks: false})`, + `({ conf: config, app: "../../..", pageRoot, NextServer, staticManifest, blobsManifest, mode: 'ssr', useHooks: false})`, ) expect(readFileSync(odbHandlerFile, 'utf8')).toMatch( - `({ conf: config, app: "../../..", pageRoot, NextServer, staticManifest, mode: 'odb', useHooks: false})`, + `({ conf: config, app: "../../..", pageRoot, NextServer, staticManifest, blobsManifest, mode: 'odb', useHooks: false})`, ) expect(readFileSync(handlerFile, 'utf8')).toMatch(`require("../../../.next/required-server-files.json")`) expect(readFileSync(odbHandlerFile, 'utf8')).toMatch(`require("../../../.next/required-server-files.json")`) diff --git a/test/templates/handlerUtils.spec.ts b/test/templates/handlerUtils.spec.ts index 26516ba2da..6a7ec3125c 100644 --- a/test/templates/handlerUtils.spec.ts +++ b/test/templates/handlerUtils.spec.ts @@ -9,7 +9,7 @@ import { unlocalizeRoute, localizeRoute, localizeDataRoute, - downloadFile, + downloadFileFromCDN, } from '../../packages/runtime/src/templates/handlerUtils' describe('normalizeRoute', () => { @@ -99,7 +99,7 @@ describe('downloadFile', () => { 'https://raw.githubusercontent.com/netlify/next-runtime/c2668af24a78eb69b33222913f44c1900a3bce23/manifest.yml' const tmpFile = join(os.tmpdir(), 'next-test', 'downloadfile.txt') await ensureDir(path.dirname(tmpFile)) - await downloadFile(url, tmpFile) + await downloadFileFromCDN(url, tmpFile) expect(existsSync(tmpFile)).toBeTruthy() expect(readFileSync(tmpFile, 'utf8')).toMatchInlineSnapshot(` "name: netlify-plugin-nextjs-experimental @@ -112,7 +112,7 @@ describe('downloadFile', () => { const url = 'https://nonexistentdomain.example' const tmpFile = join(os.tmpdir(), 'next-test', 'downloadfile.txt') await ensureDir(path.dirname(tmpFile)) - await expect(downloadFile(url, tmpFile)).rejects.toThrowErrorMatchingInlineSnapshot( + await expect(downloadFileFromCDN(url, tmpFile)).rejects.toThrowErrorMatchingInlineSnapshot( `"getaddrinfo ENOTFOUND nonexistentdomain.example"`, ) }) @@ -121,7 +121,7 @@ describe('downloadFile', () => { const url = 'https://example.com/nonexistentfile' const tmpFile = join(os.tmpdir(), 'next-test', 'downloadfile.txt') await ensureDir(path.dirname(tmpFile)) - await expect(downloadFile(url, tmpFile)).rejects.toThrow( + await expect(downloadFileFromCDN(url, tmpFile)).rejects.toThrow( 'Failed to download https://example.com/nonexistentfile: 404 Not Found', ) })