pipeline(depls): rework ops-files and vars-files support

Script manifest-lifecycle-generation handle copy of operators-file. It is not possible to use the spruce --go-patch option, because we don't know which template to associate to the ops-file. In this version, it is not possible to ops-file to directly patch credentials (use variable as workaround) Readme extract : #### ops-files By convention, all files in template dir matching ```*-operators.yml``` are used by ```bosh-deployment``` as ```ops-files``` inputs. Theses files **are not processed by spruce**. #### vars-files By convention, all files in template dir matching ```*-vars-tpl.yml``` are processed by spruce and generate ```*-vars.yml``` files. Then files are used by ```bosh-deployment``` as ```vars-files``` inputs. close #33
orange-cloudfoundry · Oct 6, 2017 · ed69701 · ed69701
1 parent 3ad354c
commit ed69701
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 29 deletions.
diff --git a/Readme.md b/Readme.md
@@ -76,10 +76,14 @@ If a template directory contains hook scripts with specific name, then these scr
 ### Bosh cli v2 specific features support
 The newest bosh feature are not implemented in bosh cli v1. So some feature are only available to deployments using bosh cli v2.
 
-#### ops-files and vars-files
-By convention, all files in template dir matching ```*-operators-tpl.yml``` are processed by spruce and generate ```*-operators.yml``` files.
-Then files are used by ```bosh-deployment``` as ```ops-files``` inputs.
-The same mechanism is used for ```*-vars-tpl.yml``` with vars-files.
+#### ops-files
+By convention, all files in template dir matching ```*-operators.yml``` are used by ```bosh-deployment``` 
+as ```ops-files``` inputs. Theses files **are not processed by spruce**.
+
+
+#### vars-files
+By convention, all files in template dir matching ```*-vars-tpl.yml``` are processed by spruce and generate ```*-vars.yml``` files.
+Then files are used by ```bosh-deployment``` as ```vars-files``` inputs.
 
 **Migration v1 to v2 tips**: empty vars-files and ops-files are generated to avoid an error message
 

diff --git a/scripts/manifest/manifest-lifecycle-generation.sh b/scripts/manifest/manifest-lifecycle-generation.sh
@@ -5,13 +5,16 @@ CURRENT_DIR=$(pwd)
 OUTPUT_DIR=${OUTPUT_DIR:-${CURRENT_DIR}/generated-files/}
 COMMON_SCRIPT_DIR=${COMMON_SCRIPT_DIR:-scripts-resource/scripts/manifest}
 
+echo "Coping operators files from '${YML_TEMPLATE_DIR}' to '${OUTPUT_DIR}'"
+find ${YML_TEMPLATE_DIR} -maxdepth 1 -type f -name "*-operators.yml" -exec cp --verbose {} ${OUTPUT_DIR} \;
+
 ${COMMON_SCRIPT_DIR}/generate-manifest.sh
 
 if [ -n "$CUSTOM_SCRIPT_DIR" -a  -f "$CUSTOM_SCRIPT_DIR/post-generate.sh" ]
 then
     echo "post generation script detected"
-    chmod +x $CUSTOM_SCRIPT_DIR/post-generate.sh
-    GENERATE_DIR=$OUTPUT_DIR BASE_TEMPLATE_DIR=$CUSTOM_SCRIPT_DIR $CUSTOM_SCRIPT_DIR/post-generate.sh
+    chmod +x ${CUSTOM_SCRIPT_DIR}/post-generate.sh
+    GENERATE_DIR=${OUTPUT_DIR} BASE_TEMPLATE_DIR=${CUSTOM_SCRIPT_DIR} ${CUSTOM_SCRIPT_DIR}/post-generate.sh
 else
     echo "ignoring post generate. No $CUSTOM_SCRIPT_DIR/post-generate.sh detected"
 

diff --git a/spec/tasks/generate_manifest/additional-resource/template/an-operators.yml b/spec/tasks/generate_manifest/additional-resource/template/an-operators.yml
@@ -0,0 +1,6 @@
+---
+- type: replace
+  path: /releases/-
+  value:
+    name: a-new-release
+    version: 1.0.0
diff --git a/spec/tasks/generate_manifest/additional-resource/template/my-vars-tpl.yml b/spec/tasks/generate_manifest/additional-resource/template/my-vars-tpl.yml
@@ -0,0 +1,5 @@
+iaas:
+  consul:
+    domain: custom.internal.paas
+container: (( concat "backup-" secrets.site ))
+
diff --git a/spec/tasks/generate_manifest/credentials-resource/secrets.yml b/spec/tasks/generate_manifest/credentials-resource/secrets.yml
@@ -1,2 +1,3 @@
 secrets:
-  bosh_uuid: 1234-4567-7898-7654-3210
+  bosh_uuid: 1234-4567-7898-7654-3210
+  site: hidden
diff --git a/spec/tasks/generate_manifest/task_spec.rb b/spec/tasks/generate_manifest/task_spec.rb
@@ -4,57 +4,77 @@
 
 describe 'generate_manifest task' do
 
-  generated_files = nil
+
+  after(:context) do
+    FileUtils.rm_rf @generated_files if File.exist? @generated_files
+  end
 
   context 'when no template are detected' do
 
-    generated_files = nil
     before(:context) do
-      generated_files = Dir.mktmpdir
+      @generated_files = Dir.mktmpdir
 
       @output = execute('-c concourse/tasks/generate-manifest.yml ' \
         '-i scripts-resource=. ' \
         '-i credentials-resource=spec/tasks/generate_manifest/credentials-resource ' \
         '-i additional-resource=spec/tasks/generate_manifest/additional-resource ' \
-        "-o generated-files=#{generated_files} ",
+        "-o generated-files=#{@generated_files} ",
         'YML_TEMPLATE_DIR' =>'',
         'SPRUCE_FILE_BASE_PATH' =>'',
         'YML_FILES' =>'',
         'SUFFIX' => '' )
     end
 
-    after(:context) do
-      FileUtils.rm_rf generated_files
-    end
-
     it 'displays an error message but does not fail concourse task' do
       expect(@output).to include('ls: /*-tpl.yml: No such file or directory')
     end
   end
 
   context 'when templates are found' do
 
+    let(:tpl_yml_files) { Dir[File.join(@additional_resource, 'template', '*-tpl.yml')].sort }
+    let(:operators_yml_files) { Dir[File.join(@additional_resource, 'template', '*-operators.yml')].sort }
+    let(:generated_files_dir_content) { Dir[File.join(@generated_files,'*')] }
+
     before(:context) do
-      generated_files = Dir.mktmpdir
+      @generated_files = Dir.mktmpdir
+      @additional_resource = File.join(File.dirname(__FILE__), 'additional-resource')
+
 
       @output = execute('-c concourse/tasks/generate-manifest.yml ' \
         '-i scripts-resource=. ' \
         '-i credentials-resource=spec/tasks/generate_manifest/credentials-resource ' \
-        '-i additional-resource=spec/tasks/generate_manifest/additional-resource ' \
-        "-o generated-files=#{generated_files} ",
+        "-i additional-resource=#{@additional_resource} " \
+        "-o generated-files=#{@generated_files} ",
         'YML_TEMPLATE_DIR' => 'additional-resource/template',
         'SPRUCE_FILE_BASE_PATH' => 'credentials-resource/',
         'YML_FILES' => "'./credentials-resource/meta.yml ./credentials-resource/secrets.yml'",
         'SUFFIX' => '')
     end
 
-    after(:context) do
-      FileUtils.rm_rf generated_files
+    it 'generates a file per valid template' do
+      tpl_yml_files .map { |filename| File.basename(filename, '-tpl.yml') }
+                    .each do |base_filename|
+                      expected_filename = File.join(@generated_files, base_filename + '.yml')
+                      expect(File,).to exist(expected_filename)
+      end
     end
 
-    it 'generates a file per valid template' do
-      expect(File.exist?(File.join(generated_files, 'first.yml'))).to be_truthy
-      expect(File.exist?(File.join(generated_files, 'second.yml'))).to be_truthy
+    it 'copies operators file to generated_files' do
+      operators_yml_files .map { |filename| File.basename(filename) }
+                          .each do |base_filename|
+        expected_filename = File.join(@generated_files, base_filename)
+        expect(File).to exist(expected_filename), 'expected ' + base_filename + ''
+      end
+    end
+
+    it 'processes only -tpl.yml files and -operators.yml' do
+      generated_files_dir_content = Dir[File.join(@generated_files, '*')]&.map! { |filename| File.basename(filename, '.yml') }
+      expected_content = tpl_yml_files + operators_yml_files
+      expected_content.map! { |filename| File.basename(filename, '.yml') }
+                      .map! { |filename| filename.chomp('-tpl') }
+
+      expect(generated_files_dir_content.size).to be_equal(expected_content.size)
     end
 
     it 'displays an post-generate ignore message' do
@@ -64,21 +84,20 @@
     context 'when processing an invalid template' do
 
       before(:context) do
-        generated_files = Dir.mktmpdir
 
         @output = execute('-c concourse/tasks/generate-manifest.yml ' \
         '-i scripts-resource=. ' \
         '-i credentials-resource=spec/tasks/generate_manifest/credentials-resource ' \
         '-i additional-resource=spec/tasks/generate_manifest/additional-resource ' \
-        "-o generated-files=#{generated_files} ",
+        "-o generated-files=#{@generated_files} ",
                           'YML_TEMPLATE_DIR' => 'additional-resource/invalid-template',
                           'SPRUCE_FILE_BASE_PATH' => 'credentials-resource/',
                           'YML_FILES' => "'./credentials-resource/meta.yml ./credentials-resource/secrets.yml'",
                           'SUFFIX' => '')
       end
 
       after(:context) do
-        FileUtils.rm_rf generated_files
+        FileUtils.rm_rf @generated_files
       end
 
       it 'display an error message' do
@@ -91,13 +110,12 @@
     context 'when a post generate script is detected' do
 
       before(:context) do
-        generated_files = Dir.mktmpdir
 
         @output = execute('-c concourse/tasks/generate-manifest.yml ' \
         '-i scripts-resource=. ' \
         '-i credentials-resource=spec/tasks/generate_manifest/credentials-resource ' \
         '-i additional-resource=spec/tasks/generate_manifest/additional-resource ' \
-        "-o generated-files=#{generated_files} ",
+        "-o generated-files=#{@generated_files} ",
         'YML_TEMPLATE_DIR' => 'additional-resource/template',
         'SPRUCE_FILE_BASE_PATH' => 'credentials-resource/',
         'YML_FILES' => "'./credentials-resource/meta.yml ./credentials-resource/secrets.yml'",
@@ -106,7 +124,7 @@
         )
       end
       after(:context) do
-        FileUtils.rm_rf generated_files
+        FileUtils.rm_rf @generated_files
       end
 
       it 'displays an execution message' do