diff --git a/.rubocop.yml b/.rubocop.yml index c1f7c991..81ed3ff9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,12 +1,13 @@ inherit_from: .rubocop_todo.yml AllCops: + TargetRubyVersion: 2.0 Exclude: - 'vendor/**/*' - 'tmp/**/*' - 'pkg/**/*' + - 'lib/monkey_patches.rb' -# we still support ruby 1.8 Style/HashSyntax: Enabled: false diff --git a/.travis.yml b/.travis.yml index 826d5fdd..3821cebc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,26 @@ sudo: false language: ruby cache: bundler +dist: trusty +before_install: + - gem update --system + - gem install bundler script: 'bundle exec rake test' rvm: - - 1.9.3 - 2.0 - - 2.1 - - 2.2 + - 2.1.9 + - 2.2.6 + - 2.3.3 + - 2.4.0 +notifications: + email: false +deploy: + provider: rubygems + api_key: + secure: "Tbf1EbLEobIIox+fftJZADZsfQQ6kl0urcMNetK7NJzFo/negD/WyJIUj3kro/B7buyYADEjTui/JR4o8EPbugfM3ie5vYOd5k3AesSzbdr4BSwGe/cGbGOB7/PZuGfFLkb94/FiCU2mIwibkbh1rHWGlBoPj7ntL0+5ZtdvsM4=" + gem: modulesync + on: + rvm: 2.4.0 + tags: true + all_branches: true + repo: voxpupuli/modulesync diff --git a/CHANGELOG.md b/CHANGELOG.md index 45203bde..4e63f17a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,42 @@ -##2015-08-13 - 0.6.1 +# Changelog + +## 2017-02-13 - 0.7.2 + +Fixes an issue releasing 0.7.1, no functional changes. + +## 2017-02-13 - 0.7.1 + +Fixes an issue releasing 0.7.0, no functional changes. + +## 2017-02-13 - 0.7.0 + +### Summary + +This is the first release from Vox Pupuli, which has taken over maintenance of +modulesync. + +#### Features +- New `msync update` arguments: + - `--git-base` to override `git_base`, e.g. for read-only git clones + - `-s` to skip the current module and continue on error + - `-x` for a negative filter (blacklist) of modules not to update +- Add `-a` argument to `msync hook` to pass additional arguments +- Add `:git_base` and `:namespace` data to `@configs` hash +- Allow `managed_modules.yml` to list modules with a different namespace +- Entire directories can be listed with `unmanaged: true` + +#### Refactoring +- Replace CLI optionparser with thor + +#### Bugfixes +- Fix git 1.8.0 compatibility, detecting when no files are changed +- Fix `delete: true` feature, now deletes files correctly +- Fix handling of `:global` config entries, not interpreted as a path +- Fix push without force to remote branch when no files have changed (#102) +- Output template name when ERB rendering fails +- Remove extraneous whitespace in `--noop` output + +## 2015-08-13 - 0.6.1 ### Summary @@ -27,7 +65,7 @@ This release adds two new flags to help modulesync better integrate with CI tool - Added tests -##2015-06-30 - 0.5.0 +## 2015-06-30 - 0.5.0 ### Summary @@ -37,7 +75,7 @@ This release adds the ability to sync a non-bare local git repo. - Allow one to sync non-bare local git repository -##2015-06-24 - 0.4.0 +## 2015-06-24 - 0.4.0 ### Summary @@ -53,7 +91,7 @@ config. - Fix markdown syntax in README -##2015-03-12 - 0.3.0 +## 2015-03-12 - 0.3.0 ### Summary @@ -87,14 +125,14 @@ number of new flags for updating modules. - Fix non-master branch functionality - Add workarounds for older git versions -##2014-11-16 - 0.2.0 +## 2014-11-16 - 0.2.0 ### Summary This release adds the --filter flag to filter what modules to sync. Also fixes the README to document the very important -m flag. -##2014-9-29 - 0.1.0 +## 2014-9-29 - 0.1.0 ### Summary diff --git a/Gemfile b/Gemfile index 945be86c..1e60c656 100644 --- a/Gemfile +++ b/Gemfile @@ -1,8 +1,3 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org' -if ENV['TRAVIS_RUBY_VERSION'] && ENV['TRAVIS_RUBY_VERSION'] == '1.9.3' - gem 'json_pure', '~> 1.0' - gem 'rubocop', '< 0.42.0' -end - gemspec diff --git a/features/step_definitions/git_steps.rb b/features/step_definitions/git_steps.rb new file mode 100644 index 00000000..998061d4 --- /dev/null +++ b/features/step_definitions/git_steps.rb @@ -0,0 +1,24 @@ +Given 'a mocked git configuration' do + steps %( + Given a mocked home directory + And I run `git config --global user.name Test` + And I run `git config --global user.email test@example.com` + ) +end + +Given 'a remote module repository' do + steps %( + Given a directory named "sources" + And I run `git clone https://github.com/maestrodev/puppet-test sources/puppet-test` + And a file named "managed_modules.yml" with: + """ + --- + - puppet-test + """ + ) + write_file('modulesync.yml', <<-EOS) +--- + namespace: sources + git_base: file://#{expand_path('.')}/ + EOS +end diff --git a/features/update.feature b/features/update.feature index 52c7f064..08e5b63c 100644 --- a/features/update.feature +++ b/features/update.feature @@ -62,6 +62,34 @@ Feature: update When I run `msync update --noop` Then the exit status should be 1 + Scenario: Using skip_broken option with invalid files + Given a file named "managed_modules.yml" with: + """ + --- + - puppet-test + """ + And a file named "modulesync.yml" with: + """ + --- + namespace: maestrodev + git_base: https://github.com/ + """ + And a file named "config_defaults.yml" with: + """ + --- + test: + name: aruba + """ + And a directory named "moduleroot" + And a file named "moduleroot/test" with: + """ + <% @configs.each do |c| -%> + <%= c['name'] %> + <% end %> + """ + When I run `msync update --noop -s` + Then the exit status should be 0 + Scenario: Modifying an existing file Given a file named "managed_modules.yml" with: """ @@ -98,6 +126,29 @@ Feature: update source 'https://somehost.com' """ + Scenario: Modifying an existing file and committing the change + Given a mocked git configuration + And a remote module repository + And a file named "config_defaults.yml" with: + """ + --- + Gemfile: + gem_source: https://somehost.com + """ + And a directory named "moduleroot" + And a file named "moduleroot/Gemfile" with: + """ + source '<%= @configs['gem_source'] %>' + """ + When I run `msync update -m "Update Gemfile" -r test` + Then the exit status should be 0 + Given I cd to "sources/puppet-test" + And I run `git checkout test` + Then the file "Gemfile" should contain: + """ + source 'https://somehost.com' + """ + Scenario: Setting an existing file to unmanaged Given a file named "managed_modules.yml" with: """ @@ -210,13 +261,13 @@ Feature: update spec: unmanaged: true """ - And a directory named "moduleroot" + And a directory named "moduleroot/spec" And a file named "moduleroot/spec/spec_helper.rb" with: """ some spec_helper fud """ And a directory named "modules/puppetlabs-apache/spec" - And a file named "modules/puppetlabs-apache/spec_helper.rb" with: + And a file named "modules/puppetlabs-apache/spec/spec_helper.rb" with: """ This is a fake spec_helper! """ @@ -227,10 +278,11 @@ Feature: update """ And the exit status should be 0 Given I run `cat modules/puppetlabs-apache/spec/spec_helper.rb` - Then the output should not contain: + Then the output should contain: """ - some spec_helper fud + This is a fake spec_helper! """ + And the exit status should be 0 Scenario: Adding a new file in a new subdirectory Given a file named "managed_modules.yml" with: @@ -694,3 +746,25 @@ Feature: update """ echo 'https://github.com/maestrodev' """ + + Scenario: Running the same update twice and pushing to a remote branch + Given a mocked git configuration + And a remote module repository + And a file named "config_defaults.yml" with: + """ + --- + Gemfile: + gem_source: https://somehost.com + """ + And a directory named "moduleroot" + And a file named "moduleroot/Gemfile" with: + """ + source '<%= @configs['gem_source'] %>' + """ + When I run `msync update -m "Update Gemfile" -r test` + Then the exit status should be 0 + Given I remove the directory "modules" + When I run `msync update -m "Update Gemfile" -r test` + Then the exit status should be 0 + Then the output should not contain "error" + Then the output should not contain "rejected" diff --git a/lib/modulesync.rb b/lib/modulesync.rb index 916c0a28..6ed05f12 100644 --- a/lib/modulesync.rb +++ b/lib/modulesync.rb @@ -7,6 +7,7 @@ require 'modulesync/renderer' require 'modulesync/settings' require 'modulesync/util' +require 'monkey_patches' module ModuleSync include Constants @@ -86,6 +87,36 @@ def self.manage_file(filename, settings, options) end end + def self.manage_module(puppet_module, module_files, module_options, defaults, options) + puts "Syncing #{puppet_module}" + namespace, module_name = self.module_name(puppet_module, options[:namespace]) + unless options[:offline] + git_base = options[:git_base] + git_uri = "#{git_base}#{namespace}" + Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {}) + end + module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}") + settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {}, + defaults, + module_configs[GLOBAL_DEFAULTS_KEY] || {}, + module_configs, + :puppet_module => module_name, + :git_base => git_base, + :namespace => namespace) + settings.unmanaged_files(module_files).each do |filename| + puts "Not managing #{filename} in #{module_name}" + end + + files_to_manage = settings.managed_files(module_files) + files_to_manage.each { |filename| manage_file(filename, settings, options) } + + if options[:noop] + Git.update_noop(module_name, options) + elsif !options[:offline] + Git.update(module_name, files_to_manage, options) + end + end + def self.update(options) options = config_defaults.merge(options) defaults = Util.parse_config("#{options[:configs]}/#{CONF_FILE}") @@ -98,32 +129,12 @@ def self.update(options) # managed_modules is either an array or a hash managed_modules.each do |puppet_module, module_options| - puts "Syncing #{puppet_module}" - namespace, module_name = self.module_name(puppet_module, options[:namespace]) - unless options[:offline] - git_base = options[:git_base] - git_uri = "#{git_base}#{namespace}" - Git.pull(git_uri, module_name, options[:branch], options[:project_root], module_options || {}) - end - module_configs = Util.parse_config("#{options[:project_root]}/#{module_name}/#{MODULE_CONF_FILE}") - settings = Settings.new(defaults[GLOBAL_DEFAULTS_KEY] || {}, - defaults, - module_configs[GLOBAL_DEFAULTS_KEY] || {}, - module_configs, - :puppet_module => module_name, - :git_base => git_base, - :namespace => namespace) - settings.unmanaged_files(module_files).each do |filename| - puts "Not managing #{filename} in #{module_name}" - end - - files_to_manage = settings.managed_files(module_files) - files_to_manage.each { |filename| manage_file(filename, settings, options) } - - if options[:noop] - Git.update_noop(module_name, options) - elsif !options[:offline] - Git.update(module_name, files_to_manage, options) + begin + manage_module(puppet_module, module_files, module_options, defaults, options) + rescue + STDERR.puts "Error while updating #{puppet_module}" + raise unless options[:skip_broken] + puts "Skipping #{puppet_module} as update process failed" end end end diff --git a/lib/modulesync/cli.rb b/lib/modulesync/cli.rb index 7b8876b1..93020a78 100644 --- a/lib/modulesync/cli.rb +++ b/lib/modulesync/cli.rb @@ -38,6 +38,7 @@ class Base < Thor option :message, :aliases => '-m', :desc => 'Commit message to apply to updated modules. Required unless running in noop mode.' option :configs, :aliases => '-c', :desc => 'The local directory or remote repository to define the list of managed modules, the file templates, and the default values for template variables.' option :remote_branch, :aliases => '-r', :desc => 'Remote branch name to push the changes to. Defaults to the branch name.' + option :skip_broken, :type => :boolean, :aliases => '-s', :desc => 'Process remaining modules if an error is found', :default => false option :amend, :type => :boolean, :desc => 'Amend previous commit', :default => false option :force, :type => :boolean, :desc => 'Force push amended commit', :default => false option :noop, :type => :boolean, :desc => 'No-op mode', :default => false diff --git a/lib/modulesync/git.rb b/lib/modulesync/git.rb index 0b92c8ce..6bcf8663 100644 --- a/lib/modulesync/git.rb +++ b/lib/modulesync/git.rb @@ -13,6 +13,11 @@ def self.local_branch_exists?(repo, branch) repo.branches.local.collect(&:name).include?(branch) end + def self.remote_branch_differ?(repo, local_branch, remote_branch) + !remote_branch_exists?(repo, remote_branch) || + repo.diff("#{local_branch}..origin/#{remote_branch}").any? + end + def self.switch_branch(repo, branch) return if repo.branch.name == branch @@ -94,17 +99,12 @@ def self.tag(repo, version, tag_pattern) def self.update(name, files, options) module_root = "#{options[:project_root]}/#{name}" message = options[:message] - branch = if options[:remote_branch] - "#{options[:branch]}:#{options[:remote_branch]}" - else - options[:branch] - end repo = ::Git.open(module_root) repo.branch(options[:branch]).checkout files.each do |file| if repo.status.deleted.include?(file) repo.remove(file) - elsif File.exist?(file) + elsif File.exist?("#{module_root}/#{file}") repo.add(file) end end @@ -118,7 +118,13 @@ def self.update(name, files, options) `#{script} #{module_root}` end repo.commit(message, opts_commit) - repo.push('origin', branch, opts_push) + if options[:remote_branch] + if remote_branch_differ?(repo, options[:branch], options[:remote_branch]) + repo.push('origin', "#{options[:branch]}:#{options[:remote_branch]}", opts_push) + end + else + repo.push('origin', options[:branch], opts_push) + end # Only bump/tag if pushing didn't fail (i.e. there were changes) m = Blacksmith::Modulefile.new("#{module_root}/metadata.json") if options[:bump] @@ -140,11 +146,7 @@ def self.update(name, files, options) # https://github.com/schacon/ruby-git/issues/130 def self.untracked_unignored_files(repo) ignore_path = "#{repo.dir.path}/.gitignore" - ignored = if File.exist?(ignore_path) - File.open(ignore_path).read.split - else - [] - end + ignored = File.exist?(ignore_path) ? File.open(ignore_path).read.split : [] repo.status.untracked.keep_if { |f, _| !ignored.any? { |i| File.fnmatch(i, f) } } end diff --git a/lib/modulesync/settings.rb b/lib/modulesync/settings.rb index 672a295d..25e5182d 100644 --- a/lib/modulesync/settings.rb +++ b/lib/modulesync/settings.rb @@ -35,14 +35,14 @@ def managed?(filename) # given a list of existing files in the repo, return everything that we might want to act on def managed_files(file_list) (file_list | defaults.keys | module_configs.keys).select do |f| - managed?(f) && (f != ModuleSync::GLOBAL_DEFAULTS_KEY) + (f != ModuleSync::GLOBAL_DEFAULTS_KEY) && managed?(f) end end # returns a list of files that should not be touched def unmanaged_files(file_list) (file_list | defaults.keys | module_configs.keys).select do |f| - !managed?(f) && (f != ModuleSync::GLOBAL_DEFAULTS_KEY) + (f != ModuleSync::GLOBAL_DEFAULTS_KEY) && !managed?(f) end end end diff --git a/lib/monkey_patches.rb b/lib/monkey_patches.rb new file mode 100644 index 00000000..b33365de --- /dev/null +++ b/lib/monkey_patches.rb @@ -0,0 +1,40 @@ +module Git + class Diff + # Monkey patch process_full_diff until https://github.com/schacon/ruby-git/issues/326 is resolved + def process_full_diff + defaults = { + :mode => '', + :src => '', + :dst => '', + :type => 'modified' + } + final = {} + current_file = nil + full_diff_utf8_encoded = @full_diff.encode("UTF-8", "binary", { + :invalid => :replace, + :undef => :replace + }) + full_diff_utf8_encoded.split("\n").each do |line| + if m = /^diff --git a\/(.*?) b\/(.*?)/.match(line) + current_file = m[1] + final[current_file] = defaults.merge({:patch => line, :path => current_file}) + elsif !current_file.nil? + if m = /^index (.......)\.\.(.......)( ......)*/.match(line) + final[current_file][:src] = m[1] + final[current_file][:dst] = m[2] + final[current_file][:mode] = m[3].strip if m[3] + end + if m = /^([[:alpha:]]*?) file mode (......)/.match(line) + final[current_file][:type] = m[1] + final[current_file][:mode] = m[2] + end + if m = /^Binary files /.match(line) + final[current_file][:binary] = true + end + final[current_file][:patch] << "\n" + line + end + end + final.map { |e| [e[0], DiffFile.new(@base, e[1])] } + end + end +end diff --git a/modulesync.gemspec b/modulesync.gemspec index 1385dd78..bde633df 100644 --- a/modulesync.gemspec +++ b/modulesync.gemspec @@ -3,14 +3,15 @@ lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) Gem::Specification.new do |spec| - spec.name = 'modulesync' - spec.version = '0.6.1' - spec.authors = ['Colleen Murphy'] - spec.email = ['colleen@puppetlabs.com'] - spec.summary = 'Puppet Module Synchronizer' - spec.description = 'Utility to synchronize common files across puppet modules in Github.' - spec.homepage = 'http://github.com/puppetlabs/modulesync' - spec.license = 'Apache 2' + spec.name = 'modulesync' + spec.version = '0.7.2' + spec.authors = ['Vox Pupuli'] + spec.email = ['voxpupuli@groups.io'] + spec.summary = 'Puppet Module Synchronizer' + spec.description = 'Utility to synchronize common files across puppet modules in Github.' + spec.homepage = 'http://github.com/voxpupuli/modulesync' + spec.license = 'Apache-2.0' + spec.required_ruby_version = '>= 2.0.0' spec.files = `git ls-files -z`.split("\x0") spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } @@ -18,12 +19,12 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_development_dependency 'aruba' - spec.add_development_dependency 'bundler', '~> 1.6' + spec.add_development_dependency 'bundler' spec.add_development_dependency 'rake' spec.add_development_dependency 'rspec' spec.add_development_dependency 'rubocop' - spec.add_runtime_dependency 'git', '~>1.2' + spec.add_runtime_dependency 'git', '~>1.3' spec.add_runtime_dependency 'puppet-blacksmith', '~>3.0' spec.add_runtime_dependency 'thor' end diff --git a/spec/unit/modulesync/settings_spec.rb b/spec/unit/modulesync/settings_spec.rb index 5c8566d2..6fb572cb 100644 --- a/spec/unit/modulesync/settings_spec.rb +++ b/spec/unit/modulesync/settings_spec.rb @@ -7,6 +7,7 @@ {}, {}, { 'Rakefile' => { 'unmanaged' => true }, + :global => { 'global' => 'value' }, 'Gemfile' => { 'key' => 'value' }, }, {} )