Feat/1786 precommit hooks (#3273)

* testing precommit withnew flag * testing precommit withnew flag * gitcfg task and precommit updates * secrets-checking scripts changes * lint failure * finding breaking lint issues for backend * finding breaking lint issues for backend * resolving frontend linting * many config changes * trying with alertmanager w/o pass * set pre-push +x * explicit install to maybe resolve Jan's issue * updating install logic to also work on circleci * cleanup for testing * force install, removing caught key in AM.yml * For Jan, switching how to invoke docker compose * Updating git-secrets for reusability * another 'docker compose' mixup * removing commented out code * reverting trufflehog code * accounting for deleted files
raft-tech · Dec 6, 2024 · cb403d4 · cb403d4
1 parent 52653f2
commit cb403d4
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 58 deletions.
diff --git a/.gitconfig b/.gitconfig
@@ -1,17 +1,20 @@
 [secrets]
 	providers = git secrets --aws-provider
-	patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}
-	patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')?
-	patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')?
-	patterns = .+_KEY=.+
 	allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+
 	allowed = ./tdrs-backend/.env.example:.*
 	allowed = ./tdrs-backend/docker-compose.yml:57:.*
-	allowed = ./tdrs-backend/manifest.proxy.yml:*
+
+	allowed = ./tdrs-frontend/node_modules*
 	allowed = regexes.json:.*
 	allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.*
 	allowed = scripts/deploy-backend.sh:.+:DJANGO_SECRET_KEY=..python -c .from secrets import token_urlsafe. print.token_urlsafe..*
 	allowed = .git/config:.*
 	allowed = .gitconfig:.*
-	allowed = .*DJANGO_SECRET_KEY=.*
+	allowed = .*DJANGO_SECRET_KEY=.*  #this is auto-generated in deployed environments
+	allowed = ./tdrs-backend/manifest.proxy.yml:*
 	allowed = ./tdrs-backend/plg/loki/manifest.yml:*
+	patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}
+	patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')?
+	patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')?
+	patterns = .+_KEY=.+
+	patterns = .+smtp_auth_password: .[^{]+
diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+zsh ./scripts/git-secrets-check.sh local
diff --git a/.githooks/pre-push b/.githooks/pre-push
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+task frontend-lint 2>/dev/null
+if [ $? != "0" ]; then
+  echo "Frontend lint failed"
+  exit 1
+fi
+
+task backend-lint 2>/dev/null
+if [ $? != "0" ]; then
+  echo "Backend lint failed"
+  exit 1
+fi
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Temporary Assistance for Needy Families (TANF) Data Portal - TDP
+ # Temporary Assistance for Needy Families (TANF) Data Portal - TDP
 
 Welcome to the project for the New TANF Data Portal, which will replace the legacy TANF Data Reporting System!
 

diff --git a/Taskfile.yml b/Taskfile.yml
@@ -2,6 +2,11 @@ version: '3'
 
 tasks:
 
+  gitcfg:
+    desc: Configure git
+    cmds:
+      - git config core.hooksPath .githooks
+
   create-network:
     desc: Create the external network
     cmds:
@@ -12,10 +17,10 @@ tasks:
     dir: tdrs-backend
     cmds:
       - task: create-network
-      - docker-compose -f docker-compose.yml up -d --build
-      - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations"
-      - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate"
-      - docker-compose -f docker-compose.yml down
+      - docker compose -f docker-compose.yml up -d --build
+      - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations"
+      - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate"
+      - docker compose -f docker-compose.yml down
       - task: sentry-down
 
   clone-sentry-repo:
@@ -43,7 +48,7 @@ tasks:
       - docker cp .env sentry:/self-hosted/.env
       - docker exec sentry bash -c "cd self-hosted && ./install.sh --skip-user-creation --no-report-self-hosted-issues"
       # create a new user
-      - docker exec sentry bash -c "cd self-hosted && docker-compose run --rm web createuser --email [email protected] --password admin --superuser"
+      - docker exec sentry bash -c "cd self-hosted && docker compose run --rm web createuser --email [email protected] --password admin --superuser"
       # copy backup.json file to sentry
       - docker cp backup.json sentry:/self-hosted/sentry/backup.json
       # restore backup
@@ -58,73 +63,73 @@ tasks:
     desc: Start sentry service
     dir: sentry
     cmds:
-      - docker exec sentry bash -c "cd self-hosted && docker-compose up -d"
+      - docker exec sentry bash -c "cd self-hosted && docker compose up -d"
 
   sentry-down:
     desc: Stop sentry service
     dir: sentry
     cmds:
-      - docker exec sentry bash -c "cd self-hosted && docker-compose down"
+      - docker exec sentry bash -c "cd self-hosted && docker compose down"
 
   drop-db:
     desc: Drop the backend database
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml down
+      - docker compose -f docker-compose.yml down
       - docker volume rm tdrs-backend_postgres_data
 
   backend-up:
     desc: Start backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml up -d
+      - docker compose -f docker-compose.yml up -d
 
   backend-down:
     desc: Stop backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml down
+      - docker compose -f docker-compose.yml down
 
   backend-logs:
     desc: Show and follow backend web server logs
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml logs -f
+      - docker compose -f docker-compose.yml logs -f
 
   backend-restart:
     desc: Restart backend web server
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml restart
+      - docker compose -f docker-compose.yml restart
 
   backend-bash:
     desc: Open a shell in the backend container
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml exec web sh
+      - docker compose -f docker-compose.yml exec web sh
 
   backend-shell:
     desc: Open a Django shell in the backend container
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py shell"
+      - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py shell"
 
   backend-exec:
     desc: Execute a command in the backend container
     dir: tdrs-backend
     vars:
       CMD: '{{.CMD}}'
     cmds:
-      - docker-compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}"
+      - docker compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}"
 
   backend-exec-seed-db:
     desc: Execute seed_db command in the backend container
     dir: tdrs-backend
     vars:
       CMD: '{{.CMD}}'
     cmds:
-      - docker-compose -f docker-compose.yml up -d
-      - docker-compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db"
+      - docker compose -f docker-compose.yml up -d
+      - docker compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db"
 
   backend-pytest:
     desc: 'Run pytest in the backend container E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"'
@@ -133,37 +138,37 @@ tasks:
       PYTEST_ARGS: '{{.PYTEST_ARGS | default "."}}'
     cmds:
       - task backend-up
-      - docker-compose -f docker-compose.yml exec web sh -c "pytest {{.PYTEST_ARGS}}"
+      - docker compose -f docker-compose.yml exec web sh -c "pytest {{.PYTEST_ARGS}}"
 
   backend-remove-volumes:
     desc: Remove the backend volumes
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml down -v
+      - docker compose -f docker-compose.yml down -v
 
   backend-lint:
     desc: Run flake8 in the backend container
     dir: tdrs-backend
     cmds:
       - task backend-up
-      - docker-compose -f docker-compose.yml exec web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi"
+      - docker compose -f docker-compose.yml exec -T web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi"
 
   backend-pip-lock:
     #TODO: Add a task to lock the pip dependencies
     desc: Lock the pip dependencies
     dir: tdrs-backend
     cmds:
       - task: backend-up
-      - docker-compose -f docker-compose.yml exec web sh -c "pipenv lock"
+      - docker compose -f docker-compose.yml exec web sh -c "pipenv lock"
 
   psql:
     desc: Open a psql shell in the backend container
     dir: tdrs-backend
     cmds:
       - task create-network || true
-      - docker-compose -f docker-compose.yml up -d postgres
+      - docker compose -f docker-compose.yml up -d postgres
       - sleep 5
-      - docker-compose -f docker-compose.yml exec postgres sh -c "psql -U tdpuser -d tdrs_test"
+      - docker compose -f docker-compose.yml exec postgres sh -c "psql -U tdpuser -d tdrs_test"
 
   clean:
     desc: Remove all containers, networks, and volumes
@@ -177,25 +182,25 @@ tasks:
     desc: Start clamav service
     dir: tdrs-backend
     cmds:
-      - docker-compose -f docker-compose.yml up -d clamav-rest
+      - docker compose -f docker-compose.yml up -d clamav-rest
 
   frontend-up:
     desc: Start frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml up -d
+      - docker compose -f docker-compose.yml up -d
 
   frontend-down:
     desc: Stop frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml down
+      - docker compose -f docker-compose.yml down
 
   frontend-restart:
     desc: Restart frontend web server
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml restart
+      - docker compose -f docker-compose.yml restart
 
   frontend-av:
     desc: Start frontend with optional clamav service
@@ -210,43 +215,43 @@ tasks:
     desc: Initialize the frontend project
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml up -d --build
-      - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "apk add nodejs npm"
-      - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "npm install"
-      - docker-compose -f docker-compose.yml down
+      - docker compose -f docker-compose.yml up -d --build
+      - docker compose -f docker-compose.yml exec tdp-frontend sh -c "apk add nodejs npm"
+      - docker compose -f docker-compose.yml exec tdp-frontend sh -c "npm install"
+      - docker compose -f docker-compose.yml down
 
   frontend-test:
     desc: Run frontend tests
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d
-      - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test"
+      - docker compose -f docker-compose.local.yml up tdp-frontend-test -d
+      - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test"
 
   frontend-test-cov:
     desc: Run frontend tests with coverage
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d
-      - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov"
+      - docker compose -f docker-compose.local.yml up tdp-frontend-test -d
+      - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov"
 
   frontend-lint:
     desc: Run eslint in the frontend container
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.local.yml up -d tdp-frontend-test --quiet-pull
-      - docker-compose -f docker-compose.yml exec tdp-frontend-test sh -c "npm run lint"
+      - docker compose -f docker-compose.local.yml up -d tdp-frontend-test --quiet-pull
+      - docker compose -f docker-compose.yml exec -T tdp-frontend-test sh -c "npm run lint"
 
   frontend-logs:
     desc: Show and follow frontend web server logs
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml logs -f
+      - docker compose -f docker-compose.yml logs -f
 
   frontend-bash:
     desc: Open a shell in the frontend container
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml exec tdp-frontend bash
+      - docker compose -f docker-compose.yml exec tdp-frontend bash
 
   up:
     desc: Start both frontend and backend web servers

diff --git a/scripts/git-secrets-check.sh b/scripts/git-secrets-check.sh
@@ -1,29 +1,57 @@
 #!/bin/bash
 set -e
+islocal=$1
 
-if [ -d /tmp/git-secrets ]; then
+if [[ $(uname -s) == "Darwin" ]]; then  # Mac OSX check
+    gs_path="/usr/local/bin"
+else # Linux, we're likely running in CircleCI
+    gs_path="/usr/sbin"
+fi
+
+if [ -f "$gs_path/git-secrets" ]; then
     echo The command git-secrets is available
 else
     echo The command git-secrets is not available, cloning...
     git clone [email protected]:awslabs/git-secrets.git /tmp/git-secrets/
     if [ -f /tmp/git-secrets/git-secrets ]; then
-	echo "Moving git secrets into PATH"
-        sudo cp /tmp/git-secrets/git-secrets /usr/sbin/
+
+        echo "Moving git secrets into PATH"
+        sudo cp /tmp/git-secrets/git-secrets $gs_path/
+        $gs_path/git-secrets --install -f
+        rm -rf /tmp/git-secrets #cleanup of clone dir
     else
-	echo "Git clone failed for git-secrets"
+	    echo "Git clone failed for git-secrets"
     fi
 fi
 
 # ensure we have correct configs in place
-[ -f ../.gitconfig ]
-cat .gitconfig >> .git/config 
-echo "Git-Secrets Config loaded:"
-grep -A10 secrets .git/config
-# grep will return non-zero code if nothing found, failing the build
+if [ -f .gitconfig ]; then
+    cat .gitconfig >> .git/config 
+    echo "Git-Secrets Config loaded:"
+    grep -A10 secrets .git/config
+    # grep will return non-zero code if nothing found, failing the build
+fi
 
-echo "git-secrets-check.sh: Scanning repo ..."
-git secrets --scan -r ../
-retVal=$?
+if [ $islocal ]; then
+    echo "git-secrets-check.sh: Scanning files staged for commit ..."
+    setopt shwordsplit
+    staged_files=$(git diff --cached --name-status | grep -vE "D|^R[0-9]+"| cut -f2 | xargs)
+
+    for filename in $staged_files; do
+        echo "git-secrets-check.sh: Scanning $filename ..."
+        git secrets --scan $filename
+        retVal=$?
+        if [[ $retVal -ne 0 ]]; then
+            echo "git-secrets found issues, prevented commit."
+            return 1
+        fi
+    done
+
+else
+    echo "git-secrets-check.sh: Scanning repo ..."
+    git secrets --scan -r ../
+    retVal=$?
+fi
 
 # if there are issues, they will be listed then script will abort here
 if [[ $retVal -eq 0 ]]; then
@@ -33,3 +61,6 @@ else
   return 1
 fi
 
+#cleanup for testing
+rm -rf /tmp/git-secrets
+sudo rm -f $gs_path/git-secrets