Merge pull request #31 from JCSDA-internal/feature/preprocessor_offset

Add the ability to use the preprocessor with space offset

docs/data-preprocessor.rst

@@ -3,7 +3,9 @@ Data Pre-Processor
 
 The preprocessor looks for the DIRECT_INCLUDE= keyword in the input yaml and
 concatenates the associated file at this point in the input file. The result
-is written to the output file or standard out if - is specified.
+is written to the output file or standard out if - is specified.  If the
+DIRECT_INCLUDE keyword has proceeding spaces these will be used to offset
+the lines which are being concatenated.
 
 It is expected that the keyword in the input yaml file will take the following
 format:

src/yamlprocessor/datapreprocessor.py

@@ -48,16 +48,19 @@ def process_yaml(self, in_yaml, out_yaml):
         for iline in lines:
             # look for specific pattern in each line
             if 'DIRECT_INCLUDE=' in iline:
+                # get spaces before 'DIRECT_INCLUDE'
+                spaces = iline.split('DIRECT_INCLUDE')[0]
                 # retrieve header file
-                yaml_header_File = iline.split('=')[1].rstrip()
+                yaml_aux_File = iline.split('=')[1].rstrip()
                 # replace variables in the string
-                yaml_header_File = self.__replace_placeholders(
-                    yaml_header_File)
-                # open header file
-                with open(yaml_header_File, 'r') as file:
-                    auxFileData = file.read()
-                # update lines for new file
-                new_line.append(auxFileData)
+                yaml_aux_File = self.__replace_placeholders(
+                    yaml_aux_File)
+                # open and read header file
+                with open(yaml_aux_File, 'r') as file:
+                    auxFileData = file.readlines()
+                # Add spaces and append to output
+                for auxline in auxFileData:
+                    new_line.append(spaces + auxline)
             else:
                 new_line.append(iline)
         # save the result

src/yamlprocessor/tests/test_datapreprocess.py

@@ -66,3 +66,46 @@ def test_main_0(tmp_path, yaml):
     outfilename1 = tmp_path / 'test_1.yaml'
     preprocessor.process_yaml(tmp_path / 'in_1.yaml', outfilename1)
     assert yaml.load(outfilename1.open()) == ref_yaml
+
+
+def test_main_1(tmp_path, yaml):
+    """Test direct insert with spaces."""
+    yaml_in = """
+_:
+- &banana 1
+- &groups [4, 5, 6]
+
+data:
+    brain: *banana
+    DIRECT_INCLUDE=$FILE_PATH/aux.yaml
+"""
+    yaml_aux = """
+tel: *groups
+"""
+    reference = """
+_:
+- &banana 1
+- &groups [4, 5, 6]
+
+data:
+    brain: *banana
+    tel: *groups
+"""
+    infilename = tmp_path / 'in_0.yaml'
+    with infilename.open('w') as infile:
+        infile.write(yaml_in)
+
+    auxfilename = tmp_path / 'aux.yaml'
+    with auxfilename.open('w') as auxfile:
+        auxfile.write(yaml_aux)
+
+    # Run preprocessor
+    preprocessor = DataPreProcessor()
+    keymap = {"FILE_PATH": str(tmp_path)}
+    preprocessor.add_replacements_map(keymap)
+    # Setup reference
+    ref_yaml = yaml.load(reference)
+    # Test first style input
+    outfilename0 = tmp_path / 'test_0.yaml'
+    preprocessor.process_yaml(tmp_path / 'in_0.yaml', outfilename0)
+    assert yaml.load(outfilename0.open()) == ref_yaml