diff --git a/BENCHMARK.md b/BENCHMARK.md index dd79da38e..be5bfbbf2 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -22,6 +22,7 @@ Some solutions are not included in the automated benchmark runs, either because - [Running a benchmark of all solutions for a particular language](#running-a-benchmark-of-all-solutions-for-a-particular-language) - [Running in unconfined mode](#running-in-unconfined-mode) - [Output formats](#output-formats) +- [Setting the solution timeout](#setting-the-solution-timeout) ## What operating system to use? @@ -375,3 +376,13 @@ The output format can be controlled via the `FORMATTER` variable like this: make FORMATTER=json make DIRECTORY=PrimeCrystal/solution_1 FORMATTER=csv ``` + +## Setting the solution timeout + +The run of each solution is limited to a certain duration, which is 10 minutes by default. +You can change this setting through the `TIMEOUT` variable like this: + +```shell +make TIMEOUT=15 +make DIRECTORY=PrimeCPP/solution_2 TIMEOUT=15 +``` diff --git a/Makefile b/Makefile index 55fdfb371..548d27050 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ SHELL := /bin/bash DIRECTORY := $(shell pwd) FORMATTER := "table" +TIMEOUT := "10" .PHONY: all all: benchmark @@ -14,6 +15,7 @@ benchmark: check-env ARGS=("-d $${REALPATH}" "-f $(FORMATTER)"); \ [ ! -z $${OUTPUT_FILE} ] && ARGS+=( "-o $${OUTPUT_FILE}" ); \ [ ! -z $${UNCONFINED} ] && ARGS+=( "--unconfined" ); \ + [ ! -z $${TIMEOUT} ] && ARGS+=( "-t $${TIMEOUT}" ); \ cd ./tools; npm ci --silent && npm start --silent -- benchmark $${ARGS[@]} .PHONY: check-env diff --git a/tools/src/commands/benchmark.ts b/tools/src/commands/benchmark.ts index 0f34acb74..5dc2a8331 100644 --- a/tools/src/commands/benchmark.ts +++ b/tools/src/commands/benchmark.ts @@ -30,9 +30,11 @@ export const command = new Command('benchmark') .option('-f, --formatter ', 'Output formatter', 'table') .option('-o, --output-file ', 'Write output to given file') .option('-u, --unconfined', 'Run with seccomp:unconfined (native performance for interpreted languages)') + .option('-t, --timeout ', 'Timeout for each benchmark in minutes', '10') .action(async (args) => { const directory = path.resolve(args.directory as string); const unconfined = args.unconfined === true; + const timeout = parseInt(args.timeout as string); logger.info(`Unconfined mode: ${unconfined}`); @@ -106,11 +108,18 @@ export const command = new Command('benchmark') let output = ''; try { logger.info(`[${implementation}][${solution}] Running...`); - output = dockerService.runContainer(imageName, options); + output = dockerService.runContainer(imageName, timeout, options); } catch (err) { - logger.warn( - `[${implementation}][${solution}] Exited with abnormal code: ${err.status}. Results might be partial...` - ); + if (err.signal) { + logger.warn( + `[${implementation}][${solution}] Killed after ${timeout} minutes with signal: ${err.signal}. Results are likely partial...` + ); + } + else { + logger.warn( + `[${implementation}][${solution}] Exited with abnormal code: ${err.status}. Results might be partial...` + ); + } output = err.output .filter((block: Buffer | null) => block !== null) .map((block: Buffer) => block.toString('utf8')) diff --git a/tools/src/services/docker.ts b/tools/src/services/docker.ts index cf891c158..04aafc5fb 100644 --- a/tools/src/services/docker.ts +++ b/tools/src/services/docker.ts @@ -7,9 +7,11 @@ export default class DockerService { }); } - public runContainer(imageName: string, options: Array): string { + public runContainer(imageName: string, duration: number, options: Array): string { const output = child_process.execSync(`docker run --rm ${options.join(' ')} ${imageName}`, { - stdio: 'pipe' + stdio: 'pipe', + timeout: duration ? duration * 60000 : undefined, + killSignal: 'SIGKILL' }); return output.toString('utf8'); }