diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 2cd223ca..a976f492 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -31,12 +31,11 @@ import glob import os import re -import subprocess import time from connections import github from datetime import datetime, timezone -from tools import args, config +from tools import args, config, run_cmd from pyghee.utils import log, error @@ -57,28 +56,19 @@ def get_current_jobs(self): username = os.getlogin() squeue_cmd = "%s --long --user=%s" % (self.poll_command, username) - log( - "get_current_jobs(): run squeue command: %s" % squeue_cmd, - self.logfile, - ) - - squeue = subprocess.run( + squeue_output, squeue_err, squeue_exitcode = run_cmd( squeue_cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - log( - "get_current_jobs(): squeue output\n%s" % squeue.stdout, - self.logfile, + "get_current_jobs(): squeue command", + log_file=self.logfile, ) + # create dictionary of jobs # if any with the following information per job: # jobid, state, nodelist_reason # skip first two lines of output ("range(2,...)") # TODO check for errors of squeue call current_jobs = {} - lines = str(squeue.stdout, "UTF-8").rstrip().split("\n") + lines = str(squeue_output).rstrip().split("\n") for i in range(2, len(lines)): # assume lines 2 to len(lines) contain jobs job = lines[i].rstrip().split() @@ -187,22 +177,16 @@ def process_new_job(self, new_job): self.scontrol_command, job_id, ) - log( - "process_new_job(): run scontrol command: %s" % scontrol_cmd, - self.logfile, - ) - - scontrol = subprocess.run( + scontrol_output, scontrol_err, scontrol_exitcode = run_cmd( scontrol_cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + "process_new_job(): scontrol command", + log_file=self.logfile, ) # parse output, # look for WorkDir=dir match = re.search(r".* WorkDir=(\S+) .*", - str(scontrol.stdout, "UTF-8")) + str(scontrol_output)) if match: log( "process_new_job(): work dir of job %s: '%s'" @@ -235,25 +219,11 @@ def process_new_job(self, new_job): self.scontrol_command, job_id, ) - log( - "process_new_job(): run scontrol command: %s" % - release_cmd, self.logfile, - ) - release = subprocess.run( + + release_output, release_err, release_exitcode = run_cmd( release_cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - log( - "process_new_job(): scontrol out: %s" - % release.stdout.decode("UTF-8"), - self.logfile, - ) - log( - "process_new_job(): scontrol err: %s" - % release.stderr.decode("UTF-8"), - self.logfile, + "process_new_job(): scontrol command", + log_file=self.logfile, ) # update PR diff --git a/tests/test_task_build.py b/tests/test_task_build.py index f9325ec6..a5f34e9f 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -40,17 +40,22 @@ def test_mkdir(tmpdir): def test_run_cmd(tmpdir): """Tests for run_cmd function.""" - output, err, exit_code = run_cmd("echo hello", 'test', tmpdir) + log_file = os.path.join(tmpdir, "log.txt") + output, err, exit_code = run_cmd("echo hello", 'test', tmpdir, log_file=log_file) assert exit_code == 0 assert output == "hello\n" assert err == "" - output, err, exit_code, = run_cmd("ls -l /does_not_exists.txt", 'fail test', tmpdir) + output, err, exit_code, = run_cmd("ls -l /does_not_exists.txt", 'fail test', tmpdir, log_file=log_file) assert exit_code != 0 assert output == "" assert "No such file or directory" in err - output, err, exit_code = run_cmd("this_command_does_not_exist", 'fail test', tmpdir) + output, err, exit_code = run_cmd("this_command_does_not_exist", 'fail test', tmpdir, log_file=log_file) assert exit_code != 0 assert output == "" assert ("this_command_does_not_exist: command not found" in err or "this_command_does_not_exist: not found" in err) + + output, err, exit_code = run_cmd("echo hello", "test in file", tmpdir, log_file=log_file) + with open(log_file, "r") as fp: + assert "test in file" in fp.read() diff --git a/tools/__init__.py b/tools/__init__.py index a3140b98..93e8f758 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -17,7 +17,7 @@ from pyghee.utils import log -def run_cmd(cmd, log_msg='', working_dir=None): +def run_cmd(cmd, log_msg='', working_dir=None, log_file=None): """Runs a command in the shell Args: @@ -35,28 +35,29 @@ def run_cmd(cmd, log_msg='', working_dir=None): working_dir = os.getcwd() if log_msg: - log(f"run_cmd(): '{log_msg}' by running '{cmd}' in directory '{working_dir}'") + log(f"run_cmd(): '{log_msg}' by running '{cmd}' in directory '{working_dir}'", log_file=log_file) else: - log(f"run_cmd(): Running '{cmd}' in directory '{working_dir}'") + log(f"run_cmd(): Running '{cmd}' in directory '{working_dir}'", log_file=log_file) result = subprocess.run(cmd, cwd=working_dir, shell=True, + encoding="UTF-8", stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout = result.stdout.decode("UTF-8") - stderr = result.stderr.decode("UTF-8") + stdout = result.stdout + stderr = result.stderr exit_code = result.returncode if exit_code != 0: log(f"run_cmd(): Error running '{cmd}' in '{working_dir}\n" f" stdout '{stdout}'\n" f" stderr '{stderr}'\n" - f" exit code {exit_code}") + f" exit code {exit_code}", log_file=log_file) else: log(f"run_cmd(): Result for running '{cmd}' in '{working_dir}\n" f" stdout '{stdout}'\n" f" stderr '{stderr}'\n" - f" exit code {exit_code}") + f" exit code {exit_code}", log_file=log_file) return stdout, stderr, exit_code