diff --git a/test/add_credo_plugin_to_project.exs b/test/add_credo_plugin_to_project.exs
deleted file mode 100644
index 32f851136..000000000
--- a/test/add_credo_plugin_to_project.exs
+++ /dev/null
@@ -1,128 +0,0 @@
-defmodule AstHelper do
-  def update(ast, fun) do
-    case fun.(ast) do
-      {atom, meta, list} when is_list(list) ->
-        {atom, meta, Enum.map(list, &update(&1, fun))}
-
-      list when is_list(list) ->
-        Enum.map(list, &update(&1, fun))
-
-      {key, value} ->
-        {key, update(value, fun)}
-
-      ast ->
-        ast
-    end
-  end
-end
-
-defmodule MixExsFile do
-  def add_dependency(filename, dep_name, requirement \\ ">= 0.0.0") do
-    new_dep_entry = {String.to_atom(dep_name), requirement}
-
-    content =
-      filename
-      |> File.read!()
-      |> Code.string_to_quoted!()
-      |> AstHelper.update(fn
-        {:defp, _, [{:deps, _, _}, [_ | _]]} = ast2 ->
-          AstHelper.update(ast2, fn
-            {:do, list_of_deps} ->
-              list_of_deps_without_dep_name =
-                Enum.reject(list_of_deps, &(to_string(elem(&1, 0)) == dep_name))
-
-              {:do, list_of_deps_without_dep_name ++ [new_dep_entry]}
-
-            value ->
-              value
-          end)
-
-        ast3 ->
-          ast3
-      end)
-      |> Macro.to_string()
-      |> Code.format_string!()
-
-    File.write(filename, content)
-  end
-end
-
-defmodule CredoExsFile do
-  @default_config """
-  %{
-    configs: [
-      %{
-        name: "default"
-      }
-    ]
-  }
-  """
-
-  def add_plugin(filename, plugin_name, plugin_params \\ []) do
-    if !File.exists?(filename) do
-      File.write(filename, @default_config)
-    end
-
-    content =
-      filename
-      |> File.read!()
-      |> Code.string_to_quoted!()
-      |> AstHelper.update(fn
-        {:%{}, meta, arguments} = ast1 ->
-          if arguments[:name] == "default" do
-            new_plugins = [{:"Elixir.#{plugin_name}", plugin_params}]
-
-            plugins_without_new_plugin =
-              Enum.reject(
-                List.wrap(arguments[:plugins]),
-                &(Macro.to_string(elem(&1, 0)) == plugin_name)
-              )
-
-            new_arguments =
-              Keyword.update(
-                arguments,
-                :plugins,
-                new_plugins,
-                fn _ -> plugins_without_new_plugin ++ new_plugins end
-              )
-
-            {:%{}, meta, new_arguments}
-          else
-            ast1
-          end
-
-        ast2 ->
-          ast2
-      end)
-      |> Macro.to_string()
-      |> Code.format_string!()
-
-    File.write(filename, content)
-  end
-end
-
-defmodule Options do
-  def parse(argv) do
-    {parsed, [], []} =
-      OptionParser.parse(argv,
-        strict: [project: :string, plugin_package: :string, plugin_name: :string]
-      )
-
-    options = Enum.into(parsed, %{})
-
-    options
-    |> Map.put(:project, options[:project] || ".")
-    |> Map.put(:plugin_name, options[:plugin_name] || Macro.camelize(options[:plugin_package]))
-  end
-end
-
-options =
-  Options.parse(System.argv())
-  |> IO.inspect(label: "Arguments given")
-
-project_dir = Path.expand(options.project)
-mix_filename = Path.join(project_dir, "mix.exs")
-credo_filename = Path.join(project_dir, ".credo.exs")
-
-MixExsFile.add_dependency(mix_filename, options.plugin_package)
-CredoExsFile.add_plugin(credo_filename, options.plugin_name)
diff --git a/test/regression/run.sh b/test/regression/run.sh
new file mode 100644
index 000000000..593847b5e
--- /dev/null
+++ b/test/regression/run.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# common setup
+
+set -e
+
+DIRNAME=$( cd "$( dirname "$0" )" && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/../.." && pwd )
+
+# script specific sources, variables and function definitions
+
+OLD_CREDO_VERSION=$1
+GIT_REPO=$2
+PROJECT_NAME=$3
+
+mkdir -p tmp
+
+WORKING_DIR=$( cd "$CREDO_ROOT/tmp/$PROJECT_NAME" && pwd )
+
+cd tmp
+
+rm -fr $PROJECT_NAME
+git clone $GIT_REPO $PROJECT_NAME --depth=1 --quiet
+
+cd $CREDO_ROOT
+
+# TODO: add --enable-disabled-checks .
+
+mix credo --working-dir $WORKING_DIR \
+    --strict --format oneline --mute-exit-status | sort | sed -e 's/tmp\///g' > tmp/results_current.txt
+
+cp test/regression/run_older_credo_version.exs tmp/
+cd tmp
+
+mix run --no-mix-exs run_older_credo_version.exs $OLD_CREDO_VERSION --working-dir $WORKING_DIR \
+    --strict --format oneline --mute-exit-status | sort | sed -e 's/tmp\///g' > results_old.txt
+
+diff results_old.txt results_current.txt && echo "No differences in issues."
\ No newline at end of file
diff --git a/test/regression/run_older_credo_version.exs b/test/regression/run_older_credo_version.exs
new file mode 100644
index 000000000..d325cb325
--- /dev/null
+++ b/test/regression/run_older_credo_version.exs
@@ -0,0 +1,7 @@
+[credo_version|rest] = System.argv()
+
+Mix.install([
+  {:credo, credo_version}
+])
+
+Credo.run(rest)
diff --git a/test/run_on_project.sh b/test/run_on_project.sh
index d1d6599b4..fa45fab2e 100755
--- a/test/run_on_project.sh
+++ b/test/run_on_project.sh
@@ -5,7 +5,7 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # script specific sources, variables and function definitions
 
@@ -19,7 +19,7 @@ CREDO_ARG5=$7
 
 # setup
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mkdir -p tmp
 
diff --git a/test/test_if_tests_fail_after_resetting_lib.sh b/test/test_if_tests_fail_after_resetting_lib.sh
index 2c296ec1d..38e90fde5 100644
--- a/test/test_if_tests_fail_after_resetting_lib.sh
+++ b/test/test_if_tests_fail_after_resetting_lib.sh
@@ -5,11 +5,11 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # execution
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 git checkout master lib/
 
diff --git a/test/test_on_projects.sh b/test/test_on_projects.sh
deleted file mode 100755
index 136ebcbf5..000000000
--- a/test/test_on_projects.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/bin/bash
-
-# common setup
-
-set -e
-
-DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
-
-# script specific sources, variables and function definitions
-
-METRIC_BASE=${1:-full}
-CREDO_ARG1=$2
-CREDO_ARG2=$3
-CREDO_ARG3=$4
-CREDO_ARG4=$5
-CREDO_ARG5=$6
-
-# setup
-
-cd $PROJECT_ROOT
-
-mkdir -p tmp
-
-mix deps.get
-mix compile
-
-# execution
-
-# Community projects
-sh $DIRNAME/run_on_project.sh https://github.com/elixirscript/elixirscript.git elixirscript
-sh $DIRNAME/run_on_project.sh https://github.com/ueberauth/guardian.git guardian
-sh $DIRNAME/run_on_project.sh https://github.com/bitwalker/distillery.git distillery
-sh $DIRNAME/run_on_project.sh https://github.com/bitwalker/timex.git timex
-sh $DIRNAME/run_on_project.sh https://github.com/michalmuskala/jason.git jason
-sh $DIRNAME/run_on_project.sh https://github.com/thoughtbot/ex_machina.git ex_machina
-sh $DIRNAME/run_on_project.sh https://github.com/graphql-elixir/graphql.git graphql
-sh $DIRNAME/run_on_project.sh https://github.com/absinthe-graphql/absinthe.git absinthe
-sh $DIRNAME/run_on_project.sh https://github.com/devinus/poison.git poison
-sh $DIRNAME/run_on_project.sh https://github.com/dashbitco/mox.git mox
-sh $DIRNAME/run_on_project.sh https://github.com/PragTob/benchee.git benchee
-
-# Elixir
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-lang/elixir.git elixir
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-lang/ex_doc.git ex_doc
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-lang/flow.git flow
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-lang/gettext.git gettext
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-lang/gen_stage.git gen_stage
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-ecto/ecto.git ecto
-sh $DIRNAME/run_on_project.sh https://github.com/elixir-plug/plug.git plug
-
-# Phoenix
-sh $DIRNAME/run_on_project.sh https://github.com/phoenixframework/phoenix.git phoenix
-sh $DIRNAME/run_on_project.sh https://github.com/phoenixframework/phoenix_html.git phoenix_html
-sh $DIRNAME/run_on_project.sh https://github.com/phoenixframework/phoenix_pubsub.git phoenix_pubsub
-sh $DIRNAME/run_on_project.sh https://github.com/phoenixframework/phoenix_ecto.git phoenix_ecto
-sh $DIRNAME/run_on_project.sh https://github.com/phoenixframework/phoenix_live_reload.git phoenix_live_reload
-
-# Nerves
-sh $DIRNAME/run_on_project.sh https://github.com/nerves-project/nerves.git nerves
-
-# teardown
-
-echo ""
-echo "All tests done."
diff --git a/test/test_phoenix_compatibility.sh b/test/test_phoenix_compatibility.sh
index 58d825245..1ee7ee341 100755
--- a/test/test_phoenix_compatibility.sh
+++ b/test/test_phoenix_compatibility.sh
@@ -9,7 +9,7 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # script specific sources, variables and function definitions
 
@@ -20,7 +20,7 @@ PROJECT_DIRNAME=tmp/$PROJECT_NAME
 
 yes | mix archive.install hex phx_new
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mkdir -p tmp
 
@@ -39,6 +39,6 @@ echo ""
 echo "--> Running Credo ..."
 echo ""
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mix credo $PROJECT_DIRNAME