#940 Reformating execute and adding it to batch execution

The function execute from atom_core.system has been reformulated.
It now prints in "real time" what the subprocess is outputting.
It is also now responsible to write stdout and stderr to a file.
This has been done in a way that saving in independent of blocking and the filename can have extra addons.

Then, this has been added to configure and to batch execution.
The configure became significantly simpler, as most local functions have been delegated to this one.
Batch execution suffered some changed to adapt to this, but it is now fully functional.
To achieve this, the function removeColorsFromText needed to be moved from atom_core.utilities to atom_core.system, to remove circular imports.

atom_batch_execution/scripts/batch_execution

@@ -3,23 +3,20 @@
 Runs several calibration executions in batch_execution. A yml file is used to config the batch executions.
 """
 import argparse
-import json
 import os
 import shutil
 import subprocess
 import yaml
 import jinja2
-from jinja2 import Template, Undefined
 from jinja2 import Environment, FileSystemLoader
 from sklearn.model_selection import StratifiedKFold, KFold, LeaveOneOut, StratifiedShuffleSplit
 
 from colorama import Fore, Back, Style
-from pytictoc import TicToc
 
-from atom_core.config_io import atomError, uriReader
+from atom_core.config_io import uriReader
 from atom_core.dataset_io import loadJSONFile
-from atom_core.system import resolvePath
-from atom_core.utilities import atomWarn, removeColorsFromText
+from atom_core.system import resolvePath, execute
+from atom_core.utilities import atomWarn
 
 
 def bprint(text):
@@ -139,7 +136,6 @@ def main():
     config = yaml.safe_load(rendered)
 
     # Create output folder
-    tictoc = TicToc()
     args['output_folder'] = resolvePath(args['output_folder'])
     if not os.path.exists(args['output_folder']):  # create stdout_data folder if it does not exist.
         os.mkdir(args['output_folder'])  # Create the new folder
@@ -151,28 +147,7 @@ def main():
     print('\n')
     bprint('Executing preprocessing command:\n' + config['preprocessing']['cmd'])
 
-    tictoc.tic()
-    proc = subprocess.Popen(config['preprocessing']['cmd'], shell=True,
-                            universal_newlines=True, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-    # proc.wait()  # wait for command to finish. Wait blocks with large pipes.
-    # Check https://docs.python.org/2/library/subprocess.html#subprocess.Popen.wait
-    stdout_data, stderr_data = proc.communicate()  # wait for command to finish
-    toc = str(round(tictoc.tocvalue(), 5))
-
-    if not proc.returncode == 0:  # Check stdout_data of command.
-        print(Fore.RED + Back.YELLOW + 'Error running command. stderr is:' + Style.RESET_ALL)
-        print(stderr_data)
-        exit(0)
-
-    if args['verbose']:
-        print(Fore.BLUE + Back.YELLOW + 'Preprocessing terminated, stdout is:' + Style.RESET_ALL)
-        print(stdout_data)
-
-    # Save stdout and stderr if failed
-    filename = args['output_folder'] + '/preprocessing_stdout.txt'
-    with open(filename, 'w') as f:
-        f.write(removeColorsFromText(stdout_data))
+    execute(config['preprocessing']['cmd'], verbose=args['verbose'], save_path=args['output_folder'], save_filename_additions='preprocessing_')
 
     # Run experiments
     num_experiments = len(config['experiments'].keys())
@@ -193,48 +168,23 @@ def main():
             os.mkdir(experiment_folder)
 
         # Start executing command.
-        tictoc.tic()
-        proc = subprocess.Popen(experiment['cmd'], shell=True, universal_newlines=True, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE)
-
-        # proc.wait()  # wait for command to finish. Wait blocks with large pipes.
-        # Check https://docs.python.org/2/library/subprocess.html#subprocess.Popen.wait
-        stdout_data, stderr_data = proc.communicate()  # wait for command to finish
-        toc = str(round(tictoc.tocvalue(), 5))
-
-        if not proc.returncode == 0:  # Check stdout_data of command.
-            print(Fore.RED + Back.YELLOW + 'Error running command. stderr is:' + Style.RESET_ALL)
-            print(stderr_data)
-            exit(0)
-
-        if args['verbose']:
-            print(Fore.BLUE + Back.YELLOW + 'Experiments' + str(idx) + ' terminated, stdout is:' + Style.RESET_ALL)
-            print(stdout_data)
-
-        # Save stdout and stderr if failed
-        filename = experiment_folder + '/stdout.txt'
-        with open(filename, 'w') as f:
-            f.write(removeColorsFromText(stdout_data))
+        execute(experiment['cmd'], verbose=args['verbose'], save_path=experiment_folder)
 
         # Collect stdout_data files
         for file in experiment['files_to_collect']:
             if file is None:
                 raise ValueError('File in files to collect is None. Aborting.')
 
-            # print(file)
             resolved_file, _, _ = uriReader(file)
-            # print(resolved_file)
 
             if not os.path.exists(resolved_file):
                 raise ValueError('File ' + file + ', resolved to ' + resolved_file +
                                  ' should be collected but does not exist.')
 
             filename_out = experiment_folder + '/' + os.path.basename(resolved_file)
             print(Fore.BLUE + Back.YELLOW + 'Copying file ' + resolved_file + ' to ' + filename_out + Style.RESET_ALL)
-            p = subprocess.Popen('cp ' + resolved_file + ' ' + filename_out, shell=True, universal_newlines=True,
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
+            execute('cp ' + resolved_file + ' ' + filename_out, verbose=False)
 
-        bprint('Experiment complete in ' + toc + ' secs.')
 
 
 if __name__ == "__main__":

atom_batch_execution/scripts/process_results

@@ -4,21 +4,12 @@ Runs several calibration executions in batch_execution. A yml file is used to co
 """
 import argparse
 import glob
-import json
 import os
 import shutil
-import subprocess
-import yaml
-import jinja2
-from jinja2 import Template, Undefined
-from jinja2 import Environment, FileSystemLoader
 
-from colorama import Fore, Back, Style
-from pytictoc import TicToc
+from colorama import Fore, Style
 
-from atom_core.config_io import atomError
 from atom_core.system import resolvePath
-from atom_core.utilities import atomWarn, removeColorsFromText
 import pandas as pd
 
 

atom_calibration/src/atom_calibration/calibration/objective_function.py

@@ -17,7 +17,6 @@
 import atom_core.ros_numpy
 from colorama import Fore, Style
 from scipy.spatial import distance
-from atom_core.utilities import removeColorsFromText
 
 # ROS imports
 from geometry_msgs.msg import Point
@@ -31,6 +30,7 @@
 from atom_core.dataset_io import getPointCloudMessageFromDictionary
 from atom_core.geometry import distance_two_3D_points, isect_line_plane_v3
 from atom_core.cache import Cache
+from atom_core.system import removeColorsFromText
 from atom_calibration.collect.label_messages import pixToWorld, worldToPix
 
 

atom_calibration/templates/configure

@@ -18,36 +18,8 @@ Only modify this file if you know what you are doing!
 """
 
 import argparse
-import subprocess
-
-from colorama import Fore, Back, Style
-from pytictoc import TicToc
-
-
-def run_command(command, tictoc):
-    print('\n\n' + Style.BRIGHT + Fore.BLUE + 'Executing command:' +
-        Style.RESET_ALL + '\n' + Fore.BLUE + command + Style.RESET_ALL)
-    # Start executing command.
-    tictoc.tic()
-    proc = subprocess.Popen(command, shell=True, universal_newlines=True, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE)
-
-    while True:
-        line = proc.stdout.readline()
-        print(line)
-        if not line: break
-
-    stdout_data, stderr_data = proc.communicate()  # wait for command to finish
-    toc = str(round(tictoc.tocvalue(), 5))
-
-    if not proc.returncode == 0:  # Check stdout_data of command.
-        print(Fore.RED + Back.YELLOW + 'Error running command. stderr is:' + Style.RESET_ALL)
-        print(stderr_data)
-        exit(0)
-
-    print(Fore.BLUE + 'Command executed in ' + toc + ' secs.' + Style.RESET_ALL)
-
 
+from atom_core.system import execute
 
 def main():
 
@@ -79,9 +51,7 @@ def main():
     # ---------------------------------------------------
     # Execute command
     # ---------------------------------------------------
-    # To measure time
-    tictoc = TicToc()
-    run_command(command, tictoc)
+    execute(command)
 
 
 if __name__ == '__main__':

atom_core/src/atom_core/system.py

@@ -2,33 +2,96 @@
 # Standard imports
 import os
 import pty
-
-
 import re
 import subprocess
 
-# Ros imports
-from urllib.parse import urlparse
+from colorama import Fore, Style, Back
+from pytictoc import TicToc
 
-import yaml
-import rospkg
-from colorama import Fore, Style
 
 
-def execute(cmd, blocking=True, verbose=True):
+def execute(cmd, blocking=True, verbose=True, save_path=None, save_filename_additions=''):
     """ @brief Executes the command in the shell in a blocking or non-blocking manner
         @param cmd a string with teh command to execute
         @return
     """
+
+    # Open files to save stdout and stderr if save_path is provided
+    stdout_file = None
+    stderr_file = None
+
+    if save_path:
+        stdout_filename = save_path + '/' + save_filename_additions + 'stdout.txt'
+        stderr_filename = save_path + '/' + save_filename_additions + 'stderr.txt'
+        stdout_file = open(stdout_filename, 'w')
+        stderr_file = open(stderr_filename, 'w')
+
+    # Print the command being executed and measure the execution time if verbose is True
     if verbose:
         print("Executing command: " + cmd)
-
-    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if blocking:  # if blocking is True:
-        for line in p.stdout.readlines():
-            if verbose:
-                print(line)
-            p.wait()
+        tictoc = TicToc()
+        tictoc.tic()
+
+    # Execute the command in the shell using subprocess.Popen
+    proc = subprocess.Popen(cmd, shell=True, universal_newlines=True, stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE)
+
+    stdout_data = ''
+    # Read the stdout of the process and print it in real-time
+    if blocking:
+        while True:
+            output = proc.stdout.read(1)
+            if output == '' and proc.poll() is not None:
+                break
+            if output:
+                if verbose:
+                    print(output, end='')
+                # sys.stdout.flush()
+                stdout_data += output
+
+        # Write the stdout data to the file if stdout_file is provided
+        if stdout_file:
+            stdout_file.write(removeColorsFromText(stdout_data))
+            stdout_file.close()
+
+    # Wait for the command to finish and get the stderr data
+    stdout_data, stderr_data = proc.communicate()
+
+    if not blocking:
+        # Write the stdout data to the file if stdout_file is provided
+        if stdout_file:
+            stdout_file.write(removeColorsFromText(stdout_data))
+            stdout_file.close()
+
+    # If the return code is not 0, print the stderr data and exit
+    if not proc.returncode == 0:
+        print(Fore.RED + Back.YELLOW + 'Error running command. stderr is:' + Style.RESET_ALL)
+        print(stderr_data)
+        if stderr_file:
+            stderr_file.write(removeColorsFromText(stderr_data))
+            stderr_file.close()
+        if stdout_file:
+            stdout_file.close()
+        exit(0)
+
+    # Print the execution time if verbose is True
+    if verbose:
+        toc = str(round(tictoc.tocvalue(), 5))
+        print(Fore.BLUE + 'Command executed in ' + toc + ' secs.' + Style.RESET_ALL)
+
+    # Close the stdout and stderr files if provided
+    if stdout_file:
+        stdout_file.close()
+    if stderr_file:
+        stderr_file.close()
+
+
+def removeColorsFromText(text):
+    # Must remove ansi escape characters so that its possible to convert to float
+    # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
+    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+    text_without_color_codes = ansi_escape.sub('', text)
+    return text_without_color_codes
 
 
 def execColored(cmd):

atom_core/src/atom_core/utilities.py

@@ -28,7 +28,7 @@
 from tf.transformations import quaternion_matrix, euler_from_matrix
 
 from atom_core.naming import generateKey
-from atom_core.system import execute
+from atom_core.system import execute, removeColorsFromText
 
 # -------------------------------------------------------------------------------
 # --- FUNCTIONS
@@ -95,14 +95,6 @@ def getNumberQualifier(n, unit='meters'):
             return Fore.RED + '{:.5f}'.format(n) + Style.RESET_ALL
 
 
-def removeColorsFromText(text):
-    # Must remove ansi escape characters so that its possible to convert to float
-    # https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
-    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
-    text_without_color_codes = ansi_escape.sub('', text)
-    return text_without_color_codes
-
-
 def addAveragesBottomRowToTable(table, header):
 
     # Compute averages and add a bottom row