Merge pull request EESSI#78 from Hafsa-Naeem/run_cmd_everywhere

implemented the run_cmd everywhere

eessi_bot_job_manager.py

@@ -31,12 +31,11 @@
 import glob
 import os
 import re
-import subprocess
 import time
 
 from connections import github
 from datetime import datetime, timezone
-from tools import args, config
+from tools import args, config, run_cmd
 
 from pyghee.utils import log, error
 
@@ -57,28 +56,19 @@ def get_current_jobs(self):
         username = os.getlogin()
 
         squeue_cmd = "%s --long --user=%s" % (self.poll_command, username)
-        log(
-            "get_current_jobs(): run squeue command: %s" % squeue_cmd,
-            self.logfile,
-        )
-
-        squeue = subprocess.run(
+        squeue_output, squeue_err, squeue_exitcode = run_cmd(
             squeue_cmd,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-        log(
-            "get_current_jobs(): squeue output\n%s" % squeue.stdout,
-            self.logfile,
+            "get_current_jobs(): squeue command",
+            log_file=self.logfile,
         )
+
         # create dictionary of jobs
         # if any with the following information per job:
         #  jobid, state, nodelist_reason
         # skip first two lines of output ("range(2,...)")
         # TODO check for errors of squeue call
         current_jobs = {}
-        lines = str(squeue.stdout, "UTF-8").rstrip().split("\n")
+        lines = str(squeue_output).rstrip().split("\n")
         for i in range(2, len(lines)):
             # assume lines 2 to len(lines) contain jobs
             job = lines[i].rstrip().split()
@@ -187,22 +177,16 @@ def process_new_job(self, new_job):
             self.scontrol_command,
             job_id,
         )
-        log(
-            "process_new_job(): run scontrol command: %s" % scontrol_cmd,
-            self.logfile,
-        )
-
-        scontrol = subprocess.run(
+        scontrol_output, scontrol_err, scontrol_exitcode = run_cmd(
             scontrol_cmd,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+            "process_new_job(): scontrol command",
+            log_file=self.logfile,
         )
 
         # parse output,
         # look for WorkDir=dir
         match = re.search(r".* WorkDir=(\S+) .*",
-                          str(scontrol.stdout, "UTF-8"))
+                          str(scontrol_output))
         if match:
             log(
                 "process_new_job(): work dir of job %s: '%s'"
@@ -235,25 +219,11 @@ def process_new_job(self, new_job):
                 self.scontrol_command,
                 job_id,
             )
-            log(
-                "process_new_job(): run scontrol command: %s" %
-                release_cmd, self.logfile,
-            )
-            release = subprocess.run(
+
+            release_output, release_err, release_exitcode = run_cmd(
                 release_cmd,
-                shell=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-            )
-            log(
-                "process_new_job(): scontrol out: %s"
-                % release.stdout.decode("UTF-8"),
-                self.logfile,
-            )
-            log(
-                "process_new_job(): scontrol err: %s"
-                % release.stderr.decode("UTF-8"),
-                self.logfile,
+                "process_new_job(): scontrol command",
+                log_file=self.logfile,
             )
 
             # update PR

tests/test_task_build.py

@@ -40,17 +40,22 @@ def test_mkdir(tmpdir):
 
 def test_run_cmd(tmpdir):
     """Tests for run_cmd function."""
-    output, err, exit_code = run_cmd("echo hello", 'test', tmpdir)
+    log_file = os.path.join(tmpdir, "log.txt")
+    output, err, exit_code = run_cmd("echo hello", 'test', tmpdir, log_file=log_file)
     assert exit_code == 0
     assert output == "hello\n"
     assert err == ""
 
-    output, err,  exit_code, = run_cmd("ls -l /does_not_exists.txt", 'fail test', tmpdir)
+    output, err,  exit_code, = run_cmd("ls -l /does_not_exists.txt", 'fail test', tmpdir, log_file=log_file)
     assert exit_code != 0
     assert output == ""
     assert "No such file or directory" in err
 
-    output, err, exit_code = run_cmd("this_command_does_not_exist", 'fail test', tmpdir)
+    output, err, exit_code = run_cmd("this_command_does_not_exist", 'fail test', tmpdir, log_file=log_file)
     assert exit_code != 0
     assert output == ""
     assert ("this_command_does_not_exist: command not found" in err or "this_command_does_not_exist: not found" in err)
+
+    output, err, exit_code = run_cmd("echo hello", "test in file", tmpdir, log_file=log_file)
+    with open(log_file, "r") as fp:
+        assert "test in file" in fp.read()

tools/__init__.py

@@ -17,7 +17,7 @@
 from pyghee.utils import log
 
 
-def run_cmd(cmd, log_msg='', working_dir=None):
+def run_cmd(cmd, log_msg='', working_dir=None, log_file=None):
     """Runs a command in the shell
 
     Args:
@@ -35,28 +35,29 @@ def run_cmd(cmd, log_msg='', working_dir=None):
         working_dir = os.getcwd()
 
     if log_msg:
-        log(f"run_cmd(): '{log_msg}' by running '{cmd}' in directory '{working_dir}'")
+        log(f"run_cmd(): '{log_msg}' by running '{cmd}' in directory '{working_dir}'", log_file=log_file)
     else:
-        log(f"run_cmd(): Running '{cmd}' in directory '{working_dir}'")
+        log(f"run_cmd(): Running '{cmd}' in directory '{working_dir}'", log_file=log_file)
 
     result = subprocess.run(cmd,
                             cwd=working_dir,
                             shell=True,
+                            encoding="UTF-8",
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
-    stdout = result.stdout.decode("UTF-8")
-    stderr = result.stderr.decode("UTF-8")
+    stdout = result.stdout
+    stderr = result.stderr
     exit_code = result.returncode
 
     if exit_code != 0:
         log(f"run_cmd(): Error running '{cmd}' in '{working_dir}\n"
             f"           stdout '{stdout}'\n"
             f"           stderr '{stderr}'\n"
-            f"           exit code {exit_code}")
+            f"           exit code {exit_code}", log_file=log_file)
     else:
         log(f"run_cmd(): Result for running '{cmd}' in '{working_dir}\n"
             f"           stdout '{stdout}'\n"
             f"           stderr '{stderr}'\n"
-            f"           exit code {exit_code}")
+            f"           exit code {exit_code}", log_file=log_file)
 
     return stdout, stderr, exit_code