Skip to content

Commit

Permalink
limit concurrency
Browse files Browse the repository at this point in the history
  • Loading branch information
@ztanner
ztanner committed Aug 6, 2024
1 parent c015fb5 commit 4341d84
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 105 deletions.
201 changes: 109 additions & 92 deletions packages/next/src/export/worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,106 +351,123 @@ export async function exportPages(

renderOpts.incrementalCache = incrementalCache

let maxAttempts = nextConfig.experimental.staticGenerationRetryCount ?? 1

return Promise.all(
paths.map(async (path) => {
const pathMap = exportPathMap[path]

const { page } = exportPathMap[path]
const pageKey = page !== path ? `${page}: ${path}` : path

let result

for (let attempt = 0; attempt < maxAttempts; attempt++) {
try {
result = await Promise.race<ExportPageResult | undefined>([
exportPage({
path,
pathMap,
distDir,
outDir,
pagesDataDir,
renderOpts,
ampValidatorPath:
nextConfig.experimental.amp?.validator || undefined,
trailingSlash: nextConfig.trailingSlash,
serverRuntimeConfig: nextConfig.serverRuntimeConfig,
subFolders: nextConfig.trailingSlash && !options.buildExport,
buildExport: options.buildExport,
optimizeFonts: nextConfig.optimizeFonts,
optimizeCss: nextConfig.experimental.optimizeCss,
disableOptimizedLoading:
nextConfig.experimental.disableOptimizedLoading,
parentSpanId: input.parentSpanId,
httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput,
enableExperimentalReact: needsExperimentalReact(nextConfig),
}),
// If exporting the page takes longer than the timeout, reject the promise.
new Promise((_, reject) => {
setTimeout(() => {
reject(new TimeoutError())
}, nextConfig.staticPageGenerationTimeout * 1000)
}),
])

// If there was an error in the export, throw it immediately. In the catch block, we might retry the export,
// or immediately fail the build, depending on user configuration. We might also continue on and attempt other pages.
if (result && 'error' in result) {
throw new ExportError()
}
const maxConcurrency = 2
const results: ExportPagesResult = []

const exportPageWithRetry = async (path: string, maxAttempts: number) => {
const pathMap = exportPathMap[path]
const { page } = exportPathMap[path]
const pageKey = page !== path ? `${page}: ${path}` : path
let attempt = 0
let result

while (attempt < maxAttempts) {
try {
result = await Promise.race<ExportPageResult | undefined>([
exportPage({
path,
pathMap,
distDir,
outDir,
pagesDataDir,
renderOpts,
ampValidatorPath:
nextConfig.experimental.amp?.validator || undefined,
trailingSlash: nextConfig.trailingSlash,
serverRuntimeConfig: nextConfig.serverRuntimeConfig,
subFolders: nextConfig.trailingSlash && !options.buildExport,
buildExport: options.buildExport,
optimizeFonts: nextConfig.optimizeFonts,
optimizeCss: nextConfig.experimental.optimizeCss,
disableOptimizedLoading:
nextConfig.experimental.disableOptimizedLoading,
parentSpanId: input.parentSpanId,
httpAgentOptions: nextConfig.httpAgentOptions,
debugOutput: options.debugOutput,
enableExperimentalReact: needsExperimentalReact(nextConfig),
}),
// If exporting the page takes longer than the timeout, reject the promise.
new Promise((_, reject) => {
setTimeout(() => {
reject(new TimeoutError())
}, nextConfig.staticPageGenerationTimeout * 1000)
}),
])

// If there was an error in the export, throw it immediately. In the catch block, we might retry the export,
// or immediately fail the build, depending on user configuration. We might also continue on and attempt other pages.
if (result && 'error' in result) {
throw new ExportError()
}

// If the export succeeds, break out of the retry loop
break
} catch (err) {
// The only error that should be caught here is an ExportError, as `exportPage` doesn't throw and instead returns an object with an `error` property.
// This is an overly cautious check to ensure that we don't accidentally catch an unexpected error.
if (!(err instanceof ExportError || err instanceof TimeoutError)) {
throw err
}
// If the export succeeds, break out of the retry loop
break
} catch (err) {
// The only error that should be caught here is an ExportError, as `exportPage` doesn't throw and instead returns an object with an `error` property.
// This is an overly cautious check to ensure that we don't accidentally catch an unexpected error.
if (!(err instanceof ExportError || err instanceof TimeoutError)) {
throw err
}

if (err instanceof TimeoutError) {
// If the export times out, we will restart the worker up to 3 times.
maxAttempts = 3
}
if (err instanceof TimeoutError) {
// If the export times out, we will restart the worker up to 3 times.
maxAttempts = 3
}

// We've reached the maximum number of attempts
if (attempt >= maxAttempts - 1) {
// Log a message if we've reached the maximum number of attempts.
// We only care to do this if maxAttempts was configured.
if (maxAttempts > 0) {
console.info(
`Failed to build ${pageKey} after ${maxAttempts} attempts.`
)
}
// If prerenderEarlyExit is enabled, we'll exit the build immediately.
if (nextConfig.experimental.prerenderEarlyExit) {
throw new ExportError(
`Export encountered an error on ${pageKey}, exiting the build.`
)
} else {
// Otherwise, this is a no-op. The build will continue, and a summary of failed pages will be displayed at the end.
}
// We've reached the maximum number of attempts
if (attempt >= maxAttempts - 1) {
// Log a message if we've reached the maximum number of attempts.
// We only care to do this if maxAttempts was configured.
if (maxAttempts > 0) {
console.info(
`Failed to build ${pageKey} after ${maxAttempts} attempts.`
)
}
// If prerenderEarlyExit is enabled, we'll exit the build immediately.
if (nextConfig.experimental.prerenderEarlyExit) {
throw new ExportError(
`Export encountered an error on ${pageKey}, exiting the build.`
)
} else {
// Otherwise, this is a no-op. The build will continue, and a summary of failed pages will be displayed at the end.
}
} else {
// Otherwise, we have more attempts to make. Wait before retrying
if (err instanceof TimeoutError) {
console.info(
`Failed to build ${pageKey} (attempt ${attempt + 1} of ${maxAttempts}) because it took more than ${nextConfig.staticPageGenerationTimeout} seconds. Retrying again shortly.`
)
} else {
// Otherwise, we have more attempts to make. Wait before retrying
if (err instanceof TimeoutError) {
console.info(
`Failed to build ${pageKey} (attempt ${attempt + 1} of ${maxAttempts}) because it took more than ${nextConfig.staticPageGenerationTimeout} seconds. Retrying again shortly.`
)
} else {
console.info(
`Failed to build ${pageKey} (attempt ${attempt + 1} of ${maxAttempts}). Retrying again shortly.`
)
}
await new Promise((r) => setTimeout(r, Math.random() * 500))
console.info(
`Failed to build ${pageKey} (attempt ${attempt + 1} of ${maxAttempts}). Retrying again shortly.`
)
}
await new Promise((r) => setTimeout(r, Math.random() * 500))
}
}

return { result, path, pageKey }
})
)
attempt++
}

return { result, path, pageKey }
}

for (let i = 0; i < paths.length; i += maxConcurrency) {
const subset = paths.slice(i, i + maxConcurrency)

const subsetResults = await Promise.all(
subset.map((path) =>
exportPageWithRetry(
path,
nextConfig.experimental.staticGenerationRetryCount ?? 1
)
)
)

results.push(...subsetResults)
}

return results
}

async function exportPage(
Expand Down
30 changes: 17 additions & 13 deletions test/integration/build-output/test/index.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,19 +168,23 @@ describe('Build Output', () => {
/ \/slow-static\/.+\/.+(?: \(\d+ ms\))?| \[\+\d+ more paths\]/g
)

// Check that there are some paths printed
expect(matches.length).toBeGreaterThan(0)

// Check that each match (except the last one) contains a duration
// This explicitly doesn't check for specific paths as workers
// process paths in a non-deterministic order
matches.slice(0, -1).forEach((match) => {
expect(match).toMatch(/ \(\d+ ms\)/)
})

// Check that the last match is "[+2 more paths]"
const lastMatch = matches[matches.length - 1]
expect(lastMatch).toBe(' [+2 more paths]')
for (const check of [
// summary
expect.stringMatching(
/\/\[propsDuration\]\/\[renderDuration\] \(\d+ ms\)/
),
// ordered by duration, includes duration
expect.stringMatching(/\/2000\/10 \(\d+ ms\)$/),
expect.stringMatching(/\/10\/1000 \(\d+ ms\)$/),
expect.stringMatching(/\/300\/10 \(\d+ ms\)$/),
// max of 7 preview paths
' [+2 more paths]',
]) {
// the order isn't guaranteed on the timing tests as while() is being
// used in the render so can block the thread of other renders sharing
// the same worker
expect(matches).toContainEqual(check)
}
})

it('should not emit extracted comments', async () => {
Expand Down

0 comments on commit 4341d84

Please # to comment.