add support parallel builds

.changeset/ten-emus-heal.md

@@ -0,0 +1,22 @@
+---
+'astro': minor
+---
+
+Adds the `build.concurreny` to specify the number of pages to build in parallel. In most cases, you should stick with the default value of `1`,
+and batch or cache long running tasks like fetch calls or data access to improve the overall rendering time.
+
+Use this option only if the refactors are not possible. If the number is set too high, the page rendering
+may slow down due to insufficient memory resources and because JS is single-threaded.
+
+In the future, Astro may reuse this option to render pages concurrently with multiple threads.
+
+```js
+// astro.config.mjs
+import { defineConfig } from 'astro';
+
+export default defineConfig({
+  build: {
+    concurrency: 2,
+  },
+});
+```

packages/astro/src/@types/astro.ts

@@ -1094,6 +1094,29 @@ export interface AstroUserConfig {
 		 * ```
 		 */
 		inlineStylesheets?: 'always' | 'auto' | 'never';
+		/**
+		 * @docs
+		 * @name build.concurrency
+		 * @type {boolean}
+		 * @default `1`
+		 * @description
+		 * The number of pages to build in parallel. In most cases, you should stick with the default value of `1`,
+		 * and batch or cache long running tasks like fetch calls or data access to improve the overall rendering time.
+		 *
+		 * Use this option only if the refactors are not possible. If the number is set too high, the page rendering
+		 * may slow down due to insufficient memory resources and because JS is single-threaded.
+		 *
+		 * In the future, Astro may reuse this option to render pages concurrently with multiple threads.
+		 *
+		 * ```js
+		 * {
+		 *   build: {
+		 *     concurrency: 1
+		 *   }
+		 * }
+		 * ```
+		 */
+		concurrency?: number;
 	};
 
 	/**

packages/astro/src/core/build/generate.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import { bgGreen, black, blue, bold, dim, green, magenta, red } from 'kleur/colors';
+import PLimit from 'p-limit';
 import PQueue from 'p-queue';
 import type {
 	AstroConfig,
@@ -198,35 +199,65 @@ async function generatePage(
 		styles,
 		mod: pageModule,
 	};
+
+	async function generatePathWithLogs(
+		path: string,
+		route: RouteData,
+		index: number,
+		paths: string[],
+		isConcurrent: boolean,
+	) {
+		const timeStart = performance.now();
+		pipeline.logger.debug('build', `Generating: ${path}`);
+
+		const filePath = getOutputFilename(config, path, pageData.route.type);
+		const lineIcon =
+			(index === paths.length - 1 && !isConcurrent) || paths.length === 1 ? '└─' : '├─';
+
+		// Log the rendering path first if not concurrent. We'll later append the time taken to render.
+		// We skip if it's concurrent as the logs may overlap
+		if (!isConcurrent) {
+			logger.info(null, `  ${blue(lineIcon)} ${dim(filePath)}`, false);
+		}
+
+		await generatePath(path, pipeline, generationOptions, route);
+
+		const timeEnd = performance.now();
+		const isSlow = timeEnd - timeStart > THRESHOLD_SLOW_RENDER_TIME_MS;
+		const timeIncrease = (isSlow ? red : dim)(`(+${getTimeStat(timeStart, timeEnd)})`);
+
+		if (isConcurrent) {
+			logger.info(null, `  ${blue(lineIcon)} ${dim(filePath)} ${timeIncrease}`);
+		} else {
+			logger.info('SKIP_FORMAT', ` ${timeIncrease}`);
+		}
+	}
+
 	// Now we explode the routes. A route render itself, and it can render its fallbacks (i18n routing)
 	for (const route of eachRouteInRouteData(pageData)) {
 		const icon =
 			route.type === 'page' || route.type === 'redirect' || route.type === 'fallback'
 				? green('▶')
 				: magenta('λ');
 		logger.info(null, `${icon} ${getPrettyRouteName(route)}`);
+
 		// Get paths for the route, calling getStaticPaths if needed.
 		const paths = await getPathsForRoute(route, pageModule, pipeline, builtPaths);
-		let timeStart = performance.now();
-		let prevTimeEnd = timeStart;
-		for (let i = 0; i < paths.length; i++) {
-			const path = paths[i];
-			pipeline.logger.debug('build', `Generating: ${path}`);
-			const filePath = getOutputFilename(config, path, pageData.route.type);
-			const lineIcon = i === paths.length - 1 ? '└─' : '├─';
-			logger.info(null, `  ${blue(lineIcon)} ${dim(filePath)}`, false);
-			await generatePath(path, pipeline, generationOptions, route);
-			const timeEnd = performance.now();
-			const timeChange = getTimeStat(prevTimeEnd, timeEnd);
-			const timeIncrease = `(+${timeChange})`;
-			let timeIncreaseLabel;
-			if (timeEnd - prevTimeEnd > THRESHOLD_SLOW_RENDER_TIME_MS) {
-				timeIncreaseLabel = red(timeIncrease);
-			} else {
-				timeIncreaseLabel = dim(timeIncrease);
+
+		// Generate each paths
+		if (config.build.concurrency > 1) {
+			const limit = PLimit(config.build.concurrency);
+			const promises: Promise<void>[] = [];
+			for (let i = 0; i < paths.length; i++) {
+				const path = paths[i];
+				promises.push(limit(() => generatePathWithLogs(path, route, i, paths, true)));
+			}
+			await Promise.allSettled(promises);
+		} else {
+			for (let i = 0; i < paths.length; i++) {
+				const path = paths[i];
+				await generatePathWithLogs(path, route, i, paths, false);
 			}
-			logger.info('SKIP_FORMAT', ` ${timeIncreaseLabel}`);
-			prevTimeEnd = timeEnd;
 		}
 	}
 }

packages/astro/src/core/config/schema.ts

@@ -63,6 +63,7 @@ export const ASTRO_CONFIG_DEFAULTS = {
 		serverEntry: 'entry.mjs',
 		redirects: true,
 		inlineStylesheets: 'auto',
+		concurrency: 1,
 	},
 	image: {
 		service: { entrypoint: 'astro/assets/services/sharp', config: {} },
@@ -186,6 +187,7 @@ export const AstroConfigSchema = z.object({
 				.enum(['always', 'auto', 'never'])
 				.optional()
 				.default(ASTRO_CONFIG_DEFAULTS.build.inlineStylesheets),
+			concurrency: z.number().min(1).optional().default(ASTRO_CONFIG_DEFAULTS.build.concurrency),
 		})
 		.default({}),
 	server: z.preprocess(
@@ -619,6 +621,7 @@ export function createRelativeSchema(cmd: string, fileProtocolRoot: string) {
 					.enum(['always', 'auto', 'never'])
 					.optional()
 					.default(ASTRO_CONFIG_DEFAULTS.build.inlineStylesheets),
+				concurrency: z.number().optional().default(ASTRO_CONFIG_DEFAULTS.build.concurrency),
 			})
 			.optional()
 			.default({}),