ci: add step to scan for secrets with trufflehog

.circleci/config.yml

@@ -298,13 +298,16 @@ commands:
 
 jobs:
   secrets-check:
-    executor: machine-executor
+    executor: docker-executor
     branches:
     steps:
       - checkout
       - run:
-          name: Scan repository for committed secrets.
+          name: "git-secrets: Scan repository for committed secrets"
           command: ./scripts/git-secrets-check.sh
+      - run:
+          name: "trufflehog: Scan repository for committed secrets"
+          command: ./scripts/trufflehog-check.sh $CIRCLE_BRANCH
   test-backend:
     executor: machine-executor
     branches:

.gitconfig

@@ -6,7 +6,10 @@
 	patterns = .+_KEY=.+
 	allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+
 	allowed = ./tdrs-backend/.env.example:.*
+	allowed = ./tdrs-backend/docker-compose.yml:57:.*
+	allowed = regexes.json:.*
+	allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.*
+	allowed = ./scripts/set-backend-env-vars.sh:24:DJANGO_SECRET_KEY=.*
 	allowed = .git/config:.*
 	allowed = .gitconfig:.*
 	allowed = .*DJANGO_SECRET_KEY=local
-

regexes.json

@@ -0,0 +1,14 @@
+{
+    "RSA private key": "-----BEGIN RSA PRIVATE KEY-----",
+    "SSH (DSA) private key": "-----BEGIN DSA PRIVATE KEY-----",
+    "SSH (EC) private key": "-----BEGIN EC PRIVATE KEY-----",
+    "PGP private key block": "-----BEGIN PGP PRIVATE KEY BLOCK-----",
+    "AWS API Key": "((?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})",
+    "GitHub": "[gG][iI][tT][hH][uU][bB].*['|\"][0-9a-zA-Z]{35,40}['|\"]",
+    "Generic API Key": "[aA][pP][iI]_?[kK][eE][yY].*['|\"][0-9a-zA-Z]{32,45}['|\"]",
+    "Generic Secret": "[sS][eE][cC][rR][eE][tT].*['|\"][0-9a-zA-Z]{32,45}['|\"]",
+    "Heroku API Key": "[hH][eE][rR][oO][kK][uU].*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}",
+    "Password in URL": "[a-zA-Z]{3,10}://[^/\\s:@]{3,20}:[^/\\s:@]{3,20}@.{1,100}[\"'\\s]",
+    "DJANGO_SECRET_KEY": "DJANGO_SECRET_KEY=[A-Za-z0-9]{50}",
+    "JWT_KEY": "JWT_KEY=[A-Za-z0-9]+"
+}

scripts/git-secrets-check.sh

@@ -10,10 +10,14 @@ fi
 
 # ensure we have correct configs in place
 [ -f ../.gitconfig ]
-cat ../.gitconfig >> .git/config 
+cat .gitconfig >> .git/config 
+echo "Git-Secrets Config loaded:"
+grep -A10 secrets .git/config
+# grep will return non-zero code if nothing found, failing the build
+
 echo "git-secrets-check.sh: Scanning repo ..."
 /tmp/git-secrets/git-secrets --scan -r ../
 
 # if there are issues, they will be listed then script will abort here
 # else
-echo "git-secrets-check.sh: No issues found"
+echo "git-secrets-check.sh: No issues found"

scripts/trufflehog-check.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+if ! [ -x "$(command -v truffleHog)" ]; then
+    echo The command truffleHog is not available, installing...
+
+    # Install truffleHog in a venv
+    python -m venv trufflehog-check
+    source trufflehog-check/bin/activate
+    python -m pip install --upgrade pip
+    pip install truffleHog
+else
+    echo The command truffleHog is available
+fi
+
+echo "trufflehog-check.sh: Scanning repo ..."
+
+last_merge=$(git log --format=format:"%H" -n 1 raft-tdp-main)
+
+# $1 - The first argument to this script, the current git branch name
+# --since_commit - Look at all commits since the last merge into raft-tdp-main
+# --entropy=True - Entropy checks on large git diffs
+python ./trufflehog-check/lib/python3.8/site-packages/truffleHog/truffleHog.py \
+  --regex \
+  --entropy=True \
+  --branch "$1" \
+  --since_commit "$last_merge" \
+  --exclude_paths ./trufflehog-exclude-patterns.txt \
+  --rules ./regexes.json \
+  https://github.com/raft-tech/TANF-app
+
+# if there are issues, they will be listed then script will abort here
+
+echo "trufflehog-check.sh: No issues found"

trufflehog-exclude-patterns.txt

@@ -0,0 +1,7 @@
+(.*/)?test/(.*/)?
+(.*/)?htmlcov/(.*/)?
+(.*/)?docs/(.*/)?
+(.*/)?static/index_files/analytics.js
+(.*/)?regexes.json
+(.*/)?yarn.lock
+(.*/)?Pipfile.lock