Add script for regression testing against older versions

rrrene · Feb 10, 2024 · a8b4f68 · a8b4f68
1 parent b83bd94
commit a8b4f68
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 200 deletions.
diff --git a/test/add_credo_plugin_to_project.exs b/test/add_credo_plugin_to_project.exs
diff --git a/test/regression/run.sh b/test/regression/run.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# common setup
+
+set -e
+
+DIRNAME=$( cd "$( dirname "$0" )" && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/../.." && pwd )
+
+# script specific sources, variables and function definitions
+
+OLD_CREDO_VERSION=$1
+GIT_REPO=$2
+PROJECT_NAME=$3
+
+mkdir -p tmp
+
+WORKING_DIR=$( cd "$CREDO_ROOT/tmp/$PROJECT_NAME" && pwd )
+
+cd tmp
+
+rm -fr $PROJECT_NAME
+git clone $GIT_REPO $PROJECT_NAME --depth=1 --quiet
+
+cd $CREDO_ROOT
+
+# TODO: add --enable-disabled-checks .
+
+mix credo --working-dir $WORKING_DIR \
+    --strict --format oneline --mute-exit-status | sort | sed -e 's/tmp\///g' > tmp/results_current.txt
+
+cp test/regression/run_older_credo_version.exs tmp/
+cd tmp
+
+mix run --no-mix-exs run_older_credo_version.exs $OLD_CREDO_VERSION --working-dir $WORKING_DIR \
+    --strict --format oneline --mute-exit-status | sort | sed -e 's/tmp\///g' > results_old.txt
+
+diff results_old.txt results_current.txt && echo "No differences in issues."
diff --git a/test/regression/run_older_credo_version.exs b/test/regression/run_older_credo_version.exs
@@ -0,0 +1,7 @@
+[credo_version|rest] = System.argv()
+
+Mix.install([
+  {:credo, credo_version}
+])
+
+Credo.run(rest)
diff --git a/test/run_on_project.sh b/test/run_on_project.sh
@@ -5,7 +5,7 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # script specific sources, variables and function definitions
 
@@ -19,7 +19,7 @@ CREDO_ARG5=$7
 
 # setup
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mkdir -p tmp
 

diff --git a/test/test_if_tests_fail_after_resetting_lib.sh b/test/test_if_tests_fail_after_resetting_lib.sh
@@ -5,11 +5,11 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # execution
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 git checkout master lib/
 

diff --git a/test/test_on_projects.sh b/test/test_on_projects.sh
diff --git a/test/test_phoenix_compatibility.sh b/test/test_phoenix_compatibility.sh
@@ -9,7 +9,7 @@
 set -e
 
 DIRNAME=$( cd "$( dirname "$0" )" && pwd )
-PROJECT_ROOT=$( cd "$DIRNAME/.." && pwd )
+CREDO_ROOT=$( cd "$DIRNAME/.." && pwd )
 
 # script specific sources, variables and function definitions
 
@@ -20,7 +20,7 @@ PROJECT_DIRNAME=tmp/$PROJECT_NAME
 
 yes | mix archive.install hex phx_new
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mkdir -p tmp
 
@@ -39,6 +39,6 @@ echo ""
 echo "--> Running Credo ..."
 echo ""
 
-cd $PROJECT_ROOT
+cd $CREDO_ROOT
 
 mix credo $PROJECT_DIRNAME