diff --git a/features/appraisals.feature b/features/appraisals.feature
index 48ed3a8a..a4b3e25a 100644
--- a/features/appraisals.feature
+++ b/features/appraisals.feature
@@ -74,3 +74,11 @@ Feature: run a rake task through several appraisals
     But the output should not contain "Fail 1.3.0"
     And the exit status should be 1
 
+  @disable-bundler
+  Scenario: run a cleanup task
+    When I run `bundle exec rake appraisal:cleanup --trace`
+    Then a file named "gemfiles/1.3.0.gemfile" should not exist
+    And a file named "gemfiles/1.3.0.gemfile.lock" should not exist
+    And a file named "gemfiles/1.3.2.gemfile" should not exist
+    And a file named "gemfiles/1.3.2.gemfile.lock" should not exist
+
diff --git a/lib/appraisal/task.rb b/lib/appraisal/task.rb
index 11ce02ae..60d1dd54 100644
--- a/lib/appraisal/task.rb
+++ b/lib/appraisal/task.rb
@@ -21,6 +21,12 @@ def initialize
           end
         end
 
+        desc "Remove all generated gemfiles from gemfiles/ folder"
+        task :cleanup do
+          require 'fileutils'
+          FileUtils.rm_f Dir['gemfiles/*.{gemfile,gemfile.lock}']
+        end
+
         File.each do |appraisal|
           desc "Run the given task for appraisal #{appraisal.name}"
           task appraisal.name do