From b0cc6c2e5d89fd49fc2ae069bcd1e12f482a3338 Mon Sep 17 00:00:00 2001 From: Olivier ORAND Date: Wed, 8 Jan 2020 00:03:20 +0100 Subject: [PATCH] task(bosh_delete_*): enhance log messages --- .../tasks/bosh_delete_apply/delete_apply.rb | 4 +- .../tasks/bosh_delete_plan/delete_plan.rb | 7 +- lib/tasks/config_repo/deployments.rb | 34 ++- .../lib/tasks/config_repo/deployments_spec.rb | 279 ++++++++++++++++++ spec/spec_helper.rb | 4 +- 5 files changed, 313 insertions(+), 15 deletions(-) create mode 100644 spec/lib/tasks/config_repo/deployments_spec.rb diff --git a/concourse/tasks/bosh_delete_apply/delete_apply.rb b/concourse/tasks/bosh_delete_apply/delete_apply.rb index d21679a0b..12198a4b1 100644 --- a/concourse/tasks/bosh_delete_apply/delete_apply.rb +++ b/concourse/tasks/bosh_delete_apply/delete_apply.rb @@ -10,11 +10,13 @@ def initialize(list_command = Tasks::Bosh::ListDeployments.new, delete_command = def process expected_deployments = @config_repo_deployments.enabled_deployments + puts "Expected deployments: #{expected_deployments}" protected_deployments = @config_repo_deployments.protected_deployments + puts "Protected deployments: #{protected_deployments}" deployed_bosh_deployments = @list_command_holder.execute puts "Active bosh deployments: #{deployed_bosh_deployments}" - puts "Filtering deployments (ie: removing expected and protected deployments)" + puts "Filtering deployments (ie: excluding expected and protected deployments)" deployed_bosh_deployments.delete_if { |deployment_name| expected_deployments&.include?(deployment_name) || protected_deployments&.include?(deployment_name) } puts "Deployments to delete: #{deployed_bosh_deployments}" diff --git a/concourse/tasks/bosh_delete_plan/delete_plan.rb b/concourse/tasks/bosh_delete_plan/delete_plan.rb index ed5ca1ee3..5bb3a393c 100644 --- a/concourse/tasks/bosh_delete_plan/delete_plan.rb +++ b/concourse/tasks/bosh_delete_plan/delete_plan.rb @@ -1,6 +1,6 @@ require 'json' - +# Monkey patch String class to write using a few colors class String def black "\e[30m#{self}\e[0m" @@ -26,11 +26,13 @@ def process deployments_file = ENV.fetch('OUTPUT_FILE', File.join('deployments-to-delete', 'file.txt')) expected_deployments = @config_repo_deployments.enabled_deployments + puts "Expected deployments detected: #{expected_deployments}" protected_deployments = @config_repo_deployments.protected_deployments + puts "Protected deployments detected: #{protected_deployments}" deployed_bosh_deployments = @list_command_holder.execute puts "Active bosh deployments: #{deployed_bosh_deployments}" - puts "Filtering deployments (ie: removing expected and protected deployments)" + puts "Filtering deployments (ie: excluding expected and protected deployments)" deployed_bosh_deployments.delete_if { |deployment_name| expected_deployments&.include?(deployment_name) || protected_deployments&.include?(deployment_name) } deployed_bosh_deployments.each do |name| @@ -52,7 +54,6 @@ def display_inactive_message(name) "\t - secrets does not enable this deployment\n" \ "\t - deployment secrets dir does not contain 'protect-deployment.yml' mark to skip deletion\n" \ "\tThis bosh deployment is going to be deleted on bosh, and files removed in secrets ('#{name}.yml', '#{name}-fingerprint.yml' and '#{name}-last-deployment-failure.yml').\n" \ - "\tOtherwise deletion is run on an unknown deployment.\n" \ "\t! Waiting for manual approval !\n" \ '' end diff --git a/lib/tasks/config_repo/deployments.rb b/lib/tasks/config_repo/deployments.rb index 28fec5d19..167c81e01 100644 --- a/lib/tasks/config_repo/deployments.rb +++ b/lib/tasks/config_repo/deployments.rb @@ -2,6 +2,8 @@ module Tasks module ConfigRepo # ease config-repository manipulations class Deployments + SPECIAL_DIRECTORIES = %w[terraform-config secrets cf-apps-deployments].freeze + def initialize(config_repo_name = 'config-resource') root_deployment_name = ENV.fetch('ROOT_DEPLOYMENT_NAME', '') raise Tasks::Bosh::EnvVarMissing, "missing environment variable: ROOT_DEPLOYMENT_NAME" if root_deployment_name.to_s.empty? @@ -13,9 +15,7 @@ def initialize(config_repo_name = 'config-resource') def filter_deployments(marker_filename) deployment_files = Dir[File.join(@config_repo_name, @root_deployment_name, '**', marker_filename)] - deployments = deployment_files.map { |filename| File.dirname(filename)&.split("/")&.last }&.sort - puts "Selected deployments (matching #{marker_filename}: #{deployments}" - deployments + deployment_files.map { |filename| File.dirname(filename)&.split("/")&.last }&.sort end def protected_deployments @@ -30,23 +30,24 @@ def bosh_deployments deployments = [] Dir.each_child(@root_deployment_dir) do |deployment_dirname| manifest_dir = File.join(@root_deployment_dir, deployment_dirname) - deployments << deployment_dirname if deployment?(manifest_dir, deployment_dirname) + deployments << deployment_dirname if self.class.deployment?(manifest_dir, deployment_dirname) end deployments.sort end - def deployment?(manifest_dir, name) - return false if name == 'secrets' + def self.deployment?(manifest_dir, name) + return false if SPECIAL_DIRECTORIES.include?(name) manifest_path = File.join(manifest_dir, name + '.yml') manifest_failure_path = File.join(manifest_dir, name + '-last-deployment-failure.yml') - File.exist?(manifest_path) || File.exist?(manifest_failure_path) + enable_deployment_path = File.join(manifest_dir, 'enable-deployment.yml') + File.exist?(manifest_path) || File.exist?(manifest_failure_path) || File.exist?(enable_deployment_path) end def cleanup_disabled_deployments puts "Cleanup deployments directory in config repository" deployments_to_cleanup = disabled_deployments - deployments_to_cleanup.each { |deployment_name| puts cleanup_deployment(deployment_name) } + deployments_to_cleanup.each { |deployment_name| cleanup_deployment(deployment_name) } deployments_to_cleanup end @@ -57,7 +58,7 @@ def cleanup_deployment(deployment_name) full_path = File.join(base_path, filename) File.delete(full_path) if File.exist?(full_path) end - Dir.delete(base_path) if Dir.exist?(base_path) && Dir.empty?(base_path) + delete_empty_directory(base_path, deployment_name) end def disabled_deployments @@ -65,6 +66,21 @@ def disabled_deployments protected_deployments_list = protected_deployments bosh_deployments.delete_if { |deployment_name| expected_deployments_list&.include?(deployment_name) || protected_deployments_list&.include?(deployment_name) } end + + private + + def delete_empty_directory(base_path, deployment_name) + full_cleanup = false + + if Dir.exist?(base_path) && Dir.empty?(base_path) + puts "Deleting #{deployment_name} directory as it is empty" + Dir.delete(base_path) + full_cleanup = true + else + puts "Skipping #{deployment_name} removal, directory not empty" + end + full_cleanup + end end end end diff --git a/spec/lib/tasks/config_repo/deployments_spec.rb b/spec/lib/tasks/config_repo/deployments_spec.rb new file mode 100644 index 000000000..f22ed957d --- /dev/null +++ b/spec/lib/tasks/config_repo/deployments_spec.rb @@ -0,0 +1,279 @@ +require 'spec_helper' +require 'tempfile' +require 'fileutils' +require 'tasks' + +describe Tasks::ConfigRepo::Deployments do + let(:deployments) { described_class.new(my_config_repo_name) } + let(:error_filepath) { Tempfile.new } + let(:my_config_repo_name) { 'my-config-repo' } + let(:my_root_deployment) { 'my-root-deployment' } + let(:protected) { %w[depl-protected-1 depl-protected-2 depl-protected-3] } + let(:protected_paths) { protected.map { |name| File.join('xx', my_root_deployment, name, 'protect-deployment.yml') } } + let(:expected_deployments_result) { %w[depl-a depl-b depl-c depl-protected-1] } + let(:expected_paths) { expected_deployments_result.map { |name| File.join('xx', my_root_deployment, name, 'enable-deployment.yml') } } + let(:deployed) { %w[depl-delete-1 depl-delete-2 depl-delete-3] + expected_deployments_result } + + before do + allow(ENV).to receive(:fetch).with('ROOT_DEPLOYMENT_NAME', anything).and_return(my_root_deployment) + end + + describe ".new" do + context "when the environment is not complete" do + + it "raises an error" do + allow(ENV).to receive(:fetch).and_return("") + + err_msg = "missing environment variable: ROOT_DEPLOYMENT_NAME" + expect { described_class.new }.to raise_error(Tasks::Bosh::EnvVarMissing, err_msg) + end + end + end + + describe ".disabled_deployments" do + let(:my_config_repo_name) { Dir.mktmpdir } + let(:root_deployment_children) { %w[secrets terraform-config] + protected_depl + enabled_depls + disabled_depls } + let(:protected_depl) { %w[protected-1]} + let(:disabled_depls) { %w[deleted-1 deleted-2] } + let(:enabled_depls) { %w[depl-b depl-a] } + + before do + root_deployment_children.each do |name| + path = File.join(my_config_repo_name, my_root_deployment, name) + FileUtils.mkdir_p(path) + FileUtils.touch(File.join(path, 'enable-deployment.yml')) if enabled_depls.include?(name) + FileUtils.touch(File.join(path, 'protect-deployment.yml')) if protected_depl.include?(name) + FileUtils.touch(File.join(path, name + '.yml')) + end + end + + after do + FileUtils.rm_rf(my_config_repo_name) + end + + context "when root deployment has disabled deployments" do + it "returns only deployments" do + expect(deployments.disabled_deployments).to match(disabled_depls) + end + end + + context "when no deployments exists" do + let(:protected_depl) { [] } + let(:disabled_depls) { [] } + let(:enabled_depls) { [] } + + it "returns an empty list" do + expect(deployments.disabled_deployments).to be_empty + end + end + end + + describe ".protected_deployments" do + context "when deployments exist" do + it "returns only deployments marked as protected" do + allow(Dir).to receive(:[]).with(File.join(my_config_repo_name, my_root_deployment, '**', 'protect-deployment.yml')).and_return(protected_paths) + + expect(deployments.protected_deployments).to match(protected) + + expect(Dir).to have_received(:[]) + end + end + + context "when no protected deployments" do + it "returns an empty list" do + allow(Dir).to receive(:[]).with(File.join(my_config_repo_name, my_root_deployment, '**', 'protect-deployment.yml')).and_return([]) + + expect(deployments.protected_deployments).to be_empty + + expect(Dir).to have_received(:[]) + end + end + end + + describe ".enabled_deployments" do + + before do + #allow(Dir).to receive(:[]).with(File.join(my_config_repo_name, my_root_deployment, '**', 'enable-deployment.yml')).and_return(expected_paths) + #allow(Dir).to receive(:exist?).and_return(true) + #allow(Dir).to receive(:empty?).and_return(true) + #allow(Dir).to receive(:delete) + #allow(File).to receive(:exist?).and_return(true) + #allow(File).to receive(:delete) + end + + context "when deployments exist" do + it "returns only deployments marked as enabled" do + allow(Dir).to receive(:[]).with(File.join(my_config_repo_name, my_root_deployment, '**', 'enable-deployment.yml')).and_return(expected_paths) + + expect(deployments.enabled_deployments).to match(expected_deployments_result) + + expect(Dir).to have_received(:[]) + end + end + + context "when no enabled deployments" do + it "returns an empty list" do + allow(Dir).to receive(:[]).with(File.join(my_config_repo_name, my_root_deployment, '**', 'enable-deployment.yml')).and_return([]) + + expect(deployments.enabled_deployments).to be_empty + + expect(Dir).to have_received(:[]) + end + end + end + + describe ".deployment?" do + let(:deployment_name) { 'my-deployment' } + let(:deployment_dir) { File.join(my_config_repo_name, my_root_deployment, deployment_name) } + + before do + allow(File).to receive(:exist?).and_return(false) + end + + context "when deployment name is unexpected" do + unexpected_dirs = %w[terraform-config secrets cf-apps-deployments] + + unexpected_dirs.each do |special_dir| + it "is not detected '#{special_dir}' as a deployment" do + expect(described_class.deployment?(deployment_dir, special_dir)).to be false + + expect(File).not_to have_received(:exist?) + end + end + end + + context "when deployment dir only contains fingerprint file" do + it "is not detected as a deployment" do + expect(described_class.deployment?(deployment_dir, deployment_name)).to be false + + expect(File).not_to have_received(:exist?).with(File.join(deployment_dir, deployment_name + 'fingerprints.yml')) + end + end + + context "when deployment dir only contains manifest failure file" do + it "is a deployment" do + allow(File).to receive(:exist?).with(File.join(deployment_dir, deployment_name + '-last-deployment-failure.yml')).and_return(true) + + expect(described_class.deployment?(deployment_dir, deployment_name)).to be true + + expect(File).to have_received(:exist?).twice + end + end + + context "when deployment dir only contains manifest file" do + it "is a deployment" do + allow(File).to receive(:exist?).with(File.join(deployment_dir, deployment_name + '.yml')).and_return(true) + + expect(described_class.deployment?(deployment_dir, deployment_name)).to be true + + expect(File).to have_received(:exist?) + end + end + + context "when deployment dir only contains 'enable-deployment.yml'" do + it "is a deployment" do + allow(File).to receive(:exist?).with(File.join(deployment_dir, 'enable-deployment.yml')).and_return(true) + + expect(described_class.deployment?(deployment_dir, deployment_name)).to be true + + expect(File).to have_received(:exist?).at_least(2) + end + end + + end + + describe ".bosh_deployments" do + let(:my_config_repo_name) { Dir.mktmpdir } + + before do + FileUtils.mkdir_p(File.join(my_config_repo_name, my_root_deployment)) + end + + after do + FileUtils.rm_rf(my_config_repo_name) + end + + context "when listing deployments in a root deployment" do + let(:root_deployment_children) { %w[depls-b terraform-config depls-a] } + let(:expected_bosh_deployments) { %w[depls-a depls-b] } + + before do + root_deployment_children.each do |name| + path = File.join(my_config_repo_name, my_root_deployment, name) + FileUtils.mkdir_p(path) + FileUtils.touch(File.join(path, name + '.yml')) + end + end + + it "returns only deployments" do + expect(deployments.bosh_deployments).to match(expected_bosh_deployments) + end + end + + context "when no deployments exists" do + it "returns an empty list" do + expect(deployments.bosh_deployments).to be_empty + end + end + end + + describe ".cleanup_deployment" do + let(:deployment_name) { 'my-deployment' } + let(:deployment_dir) { File.join(my_config_repo_name, my_root_deployment, deployment_name) } + + before do + allow(File).to receive(:exist?).and_return(true) + allow(File).to receive(:delete).and_return(1) + allow(Dir).to receive(:exist?).with(deployment_dir).and_return(true) + allow(Dir).to receive(:empty?).with(deployment_dir).and_return(true) + allow(Dir).to receive(:delete).with(deployment_dir).and_return(0) + end + + context "when deployment dir only contains COA files" do + it "deletes all files" do + expect(deployments.cleanup_deployment(deployment_name)).to be true + + expect(File).to have_received(:delete).exactly(3).times + expect(Dir).to have_received(:delete) + end + end + + context "when deployment dir othersfiles" do + it "deletes all files" do + allow(Dir).to receive(:empty?).with(deployment_dir).and_return(false) + + expect(deployments.cleanup_deployment(deployment_name)).to be false + + expect(File).to have_received(:delete).exactly(3).times + expect(Dir).not_to have_received(:delete) + end + end + end + + + describe ".cleanup_disabled_deployments" do + let(:root_deployment_children) { %w[secrets terraform-config] + protected_depl + enabled_depls + deleted_depls } + let(:protected_depl) { %w[protected-1]} + let(:deleted_depls) { %w[deleted-1 deleted-2]} + let(:enabled_depls) { %w[depl-b depl-a] } + let(:expected_bosh_deployments) { %w[depls-a depls-b] } + + before do + root_deployment_children.each do |name| + path = File.join(my_config_repo_name, my_root_deployment, name) + FileUtils.mkdir_p(path) + FileUtils.touch(File.join(path, 'enable-deployment.yml')) if enabled_depls.include?(name) + FileUtils.touch(File.join(path, 'protect-deployment.yml')) if protected_depl.include?(name) + FileUtils.touch(File.join(path, name + '.yml')) + end + end + + after do + FileUtils.rm_rf(my_config_repo_name) + end + + it "returns only deployments" do + expect(deployments.cleanup_disabled_deployments).to match(deleted_depls) + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 5a42852b8..1a15d4a26 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -123,8 +123,8 @@ config.tty = true config.before do - #$stdout = StringIO.new - #$stderr = StringIO.new + $stdout = StringIO.new + $stderr = StringIO.new end config.after(:all) do $stdout = STDOUT