diff --git a/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb b/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb index 93e6a402..563f5403 100644 --- a/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb +++ b/concourse/pipelines/template/bosh-precompile-pipeline.yml.erb @@ -1,6 +1,12 @@ <% require "#{ops_automation_path}/lib/pipeline_helpers" + def extract_repo_and_version(an_hash) + return {} if an_hash.to_s.empty? + + an_hash&.select {|key, _| %w[repository version].include?(key) } + end + root_deployment_name = depls disabled_deployments = multi_root_dependencies.select do |root_depl_name, deployment_info| deployment_info['status'] == 'disabled' @@ -38,12 +44,17 @@ raise "Inconsistency detected on #{root_deployment_name}: #{releases_inconsistencies[root_deployment_name].to_yaml}" if releases_inconsistencies[root_deployment_name].size >0 uniq_releases_per_root_deployments = Hash.new { |h, k| h[k] = {} } + releases_per_root_deployments_reference = releases_per_root_deployments.dup releases_per_root_deployments.each do |root_depl, releases_info| depends_on = config&.dig(root_depl,'precompile', 'depends-on') || [] puts "#{root_depl} depends_on: #{depends_on}" releases_info.each do |release_name, details| already_defined = false - depends_on.each { |depend_root_depl| already_defined = already_defined || releases_per_root_deployments&.dig(depend_root_depl, release_name) == details } + depends_on.each do |depend_root_depl| + details_from_parent = releases_per_root_deployments_reference&.dig(depend_root_depl, release_name) + standardized_parent_details = extract_repo_and_version(details_from_parent); + already_defined = already_defined || standardized_parent_details == extract_repo_and_version(details) + end uniq_releases_per_root_deployments[root_depl][release_name] = details unless already_defined end end diff --git a/spec/lib/template_processor/template_processor_for_bosh_precompile_pipeline_spec.rb b/spec/lib/template_processor/template_processor_for_bosh_precompile_pipeline_spec.rb index 86ff2feb..05e16620 100644 --- a/spec/lib/template_processor/template_processor_for_bosh_precompile_pipeline_spec.rb +++ b/spec/lib/template_processor/template_processor_for_bosh_precompile_pipeline_spec.rb @@ -190,7 +190,6 @@ filtered_generated_jobs = generated_pipeline['jobs']&.flat_map { |job| job['name'] } expect(filtered_generated_jobs).to match_array(expected_jobs) end - end context 'when disabled deployments are presents' do @@ -662,4 +661,121 @@ expect { template_processing_error }.to raise_error(RuntimeError, /Inconsistency detected on #{root_deployment_name}:.*/) end end + + context 'when a bosh is already precompiled' do + subject { TemplateProcessor.new root_deployment_name, config, processor_context } + + let(:root_deployment_name) { 'child-root-depls' } + let(:loaded_config) do + my_config_yaml = <<~YAML + offline-mode: + stemcells: true + precompile-mode: true + #{root_deployment_name}: + precompile: + depends-on: [parent-root-deployment] + YAML + YAML.safe_load(my_config_yaml) + end + let(:processor_context) do + { depls: root_deployment_name, + root_deployments: [root_deployment_name, 'parent-root-deployment'], + bosh_cert: bosh_cert, + multi_root_dependencies: multi_root_dependencies, + multi_root_ci_deployments: multi_root_ci_deployments, + git_submodules: git_submodules, + config: loaded_config, + ops_automation_path: ops_automation_path } + end + let(:multi_root_dependencies) do + deps_yaml = <<~YAML + parent-root-deployment: + bui: + stemcells: + bosh-openstack-kvm-ubuntu-bionic-go_agent: + releases: + route-registrar-boshrelease: + base_location: https://bosh.io/d/github.com/ + repository: cloudfoundry-community/route-registrar-boshrelease + sha1: xxxddffrpofkldkng654654d8f97g + version: '3' + haproxy-boshrelease: + base_location: https://bosh.io/d/github.com/ + repository: cloudfoundry-community/haproxy-boshrelease + version: 8.0.12 + my_custom_field: 12 + bosh-deployment: + active: true + status: enabled + #{root_deployment_name}: + bosh-bats: + status: disabled + stemcells: + bosh-openstack-kvm-ubuntu-bionic-go_agent: + bosh-deployment: {} + releases: + bosh: + base_location: https://github.com/ + repository: cloudfoundry/bosh + version: 271.0.0 + shield-expe: + stemcells: + bosh-openstack-kvm-ubuntu-bionic-go_agent: + releases: + cf-routing-release: + base_location: https://bosh.io/d/github.com/ + repository: cloudfoundry-incubator/cf-routing-release + version: 0.169.0 + bosh-deployment: + active: true + status: enabled + bui: + stemcells: + bosh-openstack-kvm-ubuntu-bionic-go_agent: + releases: + route-registrar-boshrelease: + base_location: https://bosh.io/d/github.com/ + repository: cloudfoundry-community/route-registrar-boshrelease + version: '3' + haproxy-boshrelease: + base_location: https://bosh.io/d/github.com/ + repository: cloudfoundry-community/haproxy-boshrelease + version: 8.0.12 + bosh-deployment: + active: true + status: enabled + YAML + YAML.safe_load(deps_yaml) + end + let(:generated_pipeline) do + pipeline_template = @processed_template[File.join(@pipelines_dir, @template_pipeline_name)] + generated_pipeline_path = File.join(@pipelines_output_dir, pipeline_template) + YAML.load_file(generated_pipeline_path, aliases: true) + end + + before(:context) do + @output_dir = Dir.mktmpdir('generated-pipelines') + @pipelines_output_dir = File.join(@output_dir, 'pipelines') + @template_pipeline_name = 'bosh-precompile-pipeline.yml.erb' + @pipelines_dir = Dir.mktmpdir('pipeline-templates') + + FileUtils.copy("concourse/pipelines/template/#{@template_pipeline_name}", @pipelines_dir) + end + + after(:context) do + FileUtils.rm_rf(@output_dir) + FileUtils.rm_rf(@pipelines_dir) + end + + before do + @processed_template = subject.process(@pipelines_dir + '/*') + end + + let(:expected_jobs) { %W[compile-and-export-bosh compile-and-export-cf-routing-release init-concourse-boshrelease-and-stemcell-for-#{root_deployment_name} push-boshreleases upload-stemcell-to-director upload-stemcell-to-s3] } + + it 'compiles only bosh releases not compiled by parent root deployment' do + filtered_generated_jobs = generated_pipeline['jobs']&.flat_map { |job| job['name'] } + expect(filtered_generated_jobs).to match_array(expected_jobs) + end + end end