From d229a8c6509aaf8530a6b231d51332cb6c02d661 Mon Sep 17 00:00:00 2001
From: Rutger van Bergen <rbergen@xs4all.nl>
Date: Sat, 14 Sep 2024 02:25:05 +0200
Subject: [PATCH] Add timeout to benchmark solution runs (#977)

---
 BENCHMARK.md                    | 11 +++++++++++
 Makefile                        |  2 ++
 tools/src/commands/benchmark.ts | 17 +++++++++++++----
 tools/src/services/docker.ts    |  6 ++++--
 4 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/BENCHMARK.md b/BENCHMARK.md
index dd79da38e..be5bfbbf2 100644
--- a/BENCHMARK.md
+++ b/BENCHMARK.md
@@ -22,6 +22,7 @@ Some solutions are not included in the automated benchmark runs, either because
 - [Running a benchmark of all solutions for a particular language](#running-a-benchmark-of-all-solutions-for-a-particular-language)
 - [Running in unconfined mode](#running-in-unconfined-mode)
 - [Output formats](#output-formats)
+- [Setting the solution timeout](#setting-the-solution-timeout)
 
 ## What operating system to use?
 
@@ -375,3 +376,13 @@ The output format can be controlled via the `FORMATTER` variable like this:
 make FORMATTER=json
 make DIRECTORY=PrimeCrystal/solution_1 FORMATTER=csv
 ```
+
+## Setting the solution timeout
+
+The run of each solution is limited to a certain duration, which is 10 minutes by default.
+You can change this setting through the `TIMEOUT` variable like this:
+
+```shell
+make TIMEOUT=15
+make DIRECTORY=PrimeCPP/solution_2 TIMEOUT=15
+```
diff --git a/Makefile b/Makefile
index 55fdfb371..548d27050 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ SHELL := /bin/bash
 
 DIRECTORY := $(shell pwd)
 FORMATTER := "table"
+TIMEOUT := "10"
 
 .PHONY: all
 all: benchmark
@@ -14,6 +15,7 @@ benchmark: check-env
 	ARGS=("-d $${REALPATH}" "-f $(FORMATTER)"); \
 	[ ! -z $${OUTPUT_FILE} ] && ARGS+=( "-o $${OUTPUT_FILE}" ); \
 	[ ! -z $${UNCONFINED} ] && ARGS+=( "--unconfined" ); \
+	[ ! -z $${TIMEOUT} ] && ARGS+=( "-t $${TIMEOUT}" ); \
 	cd ./tools; npm ci --silent && npm start --silent -- benchmark $${ARGS[@]}
 
 .PHONY: check-env
diff --git a/tools/src/commands/benchmark.ts b/tools/src/commands/benchmark.ts
index 0f34acb74..5dc2a8331 100644
--- a/tools/src/commands/benchmark.ts
+++ b/tools/src/commands/benchmark.ts
@@ -30,9 +30,11 @@ export const command = new Command('benchmark')
   .option('-f, --formatter <type>', 'Output formatter', 'table')
   .option('-o, --output-file <file>', 'Write output to given file')
   .option('-u, --unconfined', 'Run with seccomp:unconfined (native performance for interpreted languages)')
+  .option('-t, --timeout <minutes>', 'Timeout for each benchmark in minutes', '10')
   .action(async (args) => {
     const directory = path.resolve(args.directory as string);
     const unconfined = args.unconfined === true;
+    const timeout = parseInt(args.timeout as string);
 
     logger.info(`Unconfined mode: ${unconfined}`);
 
@@ -106,11 +108,18 @@ export const command = new Command('benchmark')
       let output = '';
       try {
         logger.info(`[${implementation}][${solution}] Running...`);
-        output = dockerService.runContainer(imageName, options);
+        output = dockerService.runContainer(imageName, timeout, options);
       } catch (err) {
-        logger.warn(
-          `[${implementation}][${solution}] Exited with abnormal code: ${err.status}. Results might be partial...`
-        );
+        if (err.signal) {
+          logger.warn(
+            `[${implementation}][${solution}] Killed after ${timeout} minutes with signal: ${err.signal}. Results are likely partial...`
+          );
+        }
+        else {
+            logger.warn(
+              `[${implementation}][${solution}] Exited with abnormal code: ${err.status}. Results might be partial...`
+            );
+        }
         output = err.output
           .filter((block: Buffer | null) => block !== null)
           .map((block: Buffer) => block.toString('utf8'))
diff --git a/tools/src/services/docker.ts b/tools/src/services/docker.ts
index cf891c158..04aafc5fb 100644
--- a/tools/src/services/docker.ts
+++ b/tools/src/services/docker.ts
@@ -7,9 +7,11 @@ export default class DockerService {
     });
   }
 
-  public runContainer(imageName: string, options: Array<string>): string {
+  public runContainer(imageName: string, duration: number, options: Array<string>): string {
     const output = child_process.execSync(`docker run --rm ${options.join(' ')} ${imageName}`, {
-      stdio: 'pipe'
+      stdio: 'pipe',
+      timeout: duration ? duration * 60000 : undefined,
+      killSignal: 'SIGKILL'
     });
     return output.toString('utf8');
   }