diff --git a/.gitconfig b/.gitconfig index f70bcd581..b3cc6696c 100644 --- a/.gitconfig +++ b/.gitconfig @@ -1,17 +1,20 @@ [secrets] providers = git secrets --aws-provider - patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16} - patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')? - patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')? - patterns = .+_KEY=.+ allowed = [A-Z]+_KEY=..echo \".{S3_CREDENTIALS}\" [|] jq -r .+ allowed = ./tdrs-backend/.env.example:.* allowed = ./tdrs-backend/docker-compose.yml:57:.* - allowed = ./tdrs-backend/manifest.proxy.yml:* + + allowed = ./tdrs-frontend/node_modules* allowed = regexes.json:.* allowed = ./scripts/copy-login-gov-keypair.sh:14:JWT_KEY=.* allowed = scripts/deploy-backend.sh:.+:DJANGO_SECRET_KEY=..python -c .from secrets import token_urlsafe. print.token_urlsafe..* allowed = .git/config:.* allowed = .gitconfig:.* - allowed = .*DJANGO_SECRET_KEY=.* + allowed = .*DJANGO_SECRET_KEY=.* #this is auto-generated in deployed environments + allowed = ./tdrs-backend/manifest.proxy.yml:* allowed = ./tdrs-backend/plg/loki/manifest.yml:* + patterns = (A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16} + patterns = (\"|')?(AWS|aws|Aws)?_?(SECRET|secret|Secret)?_?(ACCESS|access|Access)?_?(KEY|key|Key)(\"|')?\\s*(:|=>|=)\\s*(\"|')?[A-Za-z0-9/\\+=]{40}(\"|')? + patterns = (\"|')?(AWS|aws|Aws)?_?(ACCOUNT|account|Account)_?(ID|id|Id)?(\"|')?\\s*(:|=>|=)\\s*(\"|')?[0-9]{4}\\-?[0-9]{4}\\-?[0-9]{4}(\"|')? + patterns = .+_KEY=.+ + patterns = .+smtp_auth_password: .[^{]+ diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 000000000..7da1e7bb0 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +zsh ./scripts/git-secrets-check.sh local diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 000000000..51e4e28ff --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +task frontend-lint 2>/dev/null +if [ $? != "0" ]; then + echo "Frontend lint failed" + exit 1 +fi + +task backend-lint 2>/dev/null +if [ $? != "0" ]; then + echo "Backend lint failed" + exit 1 +fi \ No newline at end of file diff --git a/README.md b/README.md index c7ed080a9..ce86a895b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Temporary Assistance for Needy Families (TANF) Data Portal - TDP + # Temporary Assistance for Needy Families (TANF) Data Portal - TDP Welcome to the project for the New TANF Data Portal, which will replace the legacy TANF Data Reporting System! diff --git a/Taskfile.yml b/Taskfile.yml index 9f2488455..61bc997a2 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,6 +2,11 @@ version: '3' tasks: + gitcfg: + desc: Configure git + cmds: + - git config core.hooksPath .githooks + create-network: desc: Create the external network cmds: @@ -12,10 +17,10 @@ tasks: dir: tdrs-backend cmds: - task: create-network - - docker-compose -f docker-compose.yml up -d --build - - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations" - - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate" - - docker-compose -f docker-compose.yml down + - docker compose -f docker-compose.yml up -d --build + - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations" + - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate" + - docker compose -f docker-compose.yml down - task: sentry-down clone-sentry-repo: @@ -43,7 +48,7 @@ tasks: - docker cp .env sentry:/self-hosted/.env - docker exec sentry bash -c "cd self-hosted && ./install.sh --skip-user-creation --no-report-self-hosted-issues" # create a new user - - docker exec sentry bash -c "cd self-hosted && docker-compose run --rm web createuser --email admin@tanf.com --password admin --superuser" + - docker exec sentry bash -c "cd self-hosted && docker compose run --rm web createuser --email admin@tanf.com --password admin --superuser" # copy backup.json file to sentry - docker cp backup.json sentry:/self-hosted/sentry/backup.json # restore backup @@ -58,56 +63,56 @@ tasks: desc: Start sentry service dir: sentry cmds: - - docker exec sentry bash -c "cd self-hosted && docker-compose up -d" + - docker exec sentry bash -c "cd self-hosted && docker compose up -d" sentry-down: desc: Stop sentry service dir: sentry cmds: - - docker exec sentry bash -c "cd self-hosted && docker-compose down" + - docker exec sentry bash -c "cd self-hosted && docker compose down" drop-db: desc: Drop the backend database dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml down + - docker compose -f docker-compose.yml down - docker volume rm tdrs-backend_postgres_data backend-up: desc: Start backend web server dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml up -d + - docker compose -f docker-compose.yml up -d backend-down: desc: Stop backend web server dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml down + - docker compose -f docker-compose.yml down backend-logs: desc: Show and follow backend web server logs dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml logs -f + - docker compose -f docker-compose.yml logs -f backend-restart: desc: Restart backend web server dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml restart + - docker compose -f docker-compose.yml restart backend-bash: desc: Open a shell in the backend container dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml exec web sh + - docker compose -f docker-compose.yml exec web sh backend-shell: desc: Open a Django shell in the backend container dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py shell" + - docker compose -f docker-compose.yml exec web sh -c "python ./manage.py shell" backend-exec: desc: Execute a command in the backend container @@ -115,7 +120,7 @@ tasks: vars: CMD: '{{.CMD}}' cmds: - - docker-compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}" + - docker compose -f docker-compose.yml exec web sh -c "python manage.py {{.CMD}}" backend-exec-seed-db: desc: Execute seed_db command in the backend container @@ -123,8 +128,8 @@ tasks: vars: CMD: '{{.CMD}}' cmds: - - docker-compose -f docker-compose.yml up -d - - docker-compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db" + - docker compose -f docker-compose.yml up -d + - docker compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db" backend-pytest: desc: 'Run pytest in the backend container E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"' @@ -133,20 +138,20 @@ tasks: PYTEST_ARGS: '{{.PYTEST_ARGS | default "."}}' cmds: - task backend-up - - docker-compose -f docker-compose.yml exec web sh -c "pytest {{.PYTEST_ARGS}}" + - docker compose -f docker-compose.yml exec web sh -c "pytest {{.PYTEST_ARGS}}" backend-remove-volumes: desc: Remove the backend volumes dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml down -v + - docker compose -f docker-compose.yml down -v backend-lint: desc: Run flake8 in the backend container dir: tdrs-backend cmds: - task backend-up - - docker-compose -f docker-compose.yml exec web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi" + - docker compose -f docker-compose.yml exec -T web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi" backend-pip-lock: #TODO: Add a task to lock the pip dependencies @@ -154,16 +159,16 @@ tasks: dir: tdrs-backend cmds: - task: backend-up - - docker-compose -f docker-compose.yml exec web sh -c "pipenv lock" + - docker compose -f docker-compose.yml exec web sh -c "pipenv lock" psql: desc: Open a psql shell in the backend container dir: tdrs-backend cmds: - task create-network || true - - docker-compose -f docker-compose.yml up -d postgres + - docker compose -f docker-compose.yml up -d postgres - sleep 5 - - docker-compose -f docker-compose.yml exec postgres sh -c "psql -U tdpuser -d tdrs_test" + - docker compose -f docker-compose.yml exec postgres sh -c "psql -U tdpuser -d tdrs_test" clean: desc: Remove all containers, networks, and volumes @@ -177,25 +182,25 @@ tasks: desc: Start clamav service dir: tdrs-backend cmds: - - docker-compose -f docker-compose.yml up -d clamav-rest + - docker compose -f docker-compose.yml up -d clamav-rest frontend-up: desc: Start frontend web server dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml up -d + - docker compose -f docker-compose.yml up -d frontend-down: desc: Stop frontend web server dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml down + - docker compose -f docker-compose.yml down frontend-restart: desc: Restart frontend web server dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml restart + - docker compose -f docker-compose.yml restart frontend-av: desc: Start frontend with optional clamav service @@ -210,43 +215,43 @@ tasks: desc: Initialize the frontend project dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml up -d --build - - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "apk add nodejs npm" - - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "npm install" - - docker-compose -f docker-compose.yml down + - docker compose -f docker-compose.yml up -d --build + - docker compose -f docker-compose.yml exec tdp-frontend sh -c "apk add nodejs npm" + - docker compose -f docker-compose.yml exec tdp-frontend sh -c "npm install" + - docker compose -f docker-compose.yml down frontend-test: desc: Run frontend tests dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d - - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test" + - docker compose -f docker-compose.local.yml up tdp-frontend-test -d + - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test" frontend-test-cov: desc: Run frontend tests with coverage dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d - - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov" + - docker compose -f docker-compose.local.yml up tdp-frontend-test -d + - docker compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov" frontend-lint: desc: Run eslint in the frontend container dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.local.yml up -d tdp-frontend-test --quiet-pull - - docker-compose -f docker-compose.yml exec tdp-frontend-test sh -c "npm run lint" + - docker compose -f docker-compose.local.yml up -d tdp-frontend-test --quiet-pull + - docker compose -f docker-compose.yml exec -T tdp-frontend-test sh -c "npm run lint" frontend-logs: desc: Show and follow frontend web server logs dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml logs -f + - docker compose -f docker-compose.yml logs -f frontend-bash: desc: Open a shell in the frontend container dir: tdrs-frontend cmds: - - docker-compose -f docker-compose.yml exec tdp-frontend bash + - docker compose -f docker-compose.yml exec tdp-frontend bash up: desc: Start both frontend and backend web servers diff --git a/scripts/git-secrets-check.sh b/scripts/git-secrets-check.sh index f371f303e..6f6c85f8b 100755 --- a/scripts/git-secrets-check.sh +++ b/scripts/git-secrets-check.sh @@ -1,29 +1,57 @@ #!/bin/bash set -e +islocal=$1 -if [ -d /tmp/git-secrets ]; then +if [[ $(uname -s) == "Darwin" ]]; then # Mac OSX check + gs_path="/usr/local/bin" +else # Linux, we're likely running in CircleCI + gs_path="/usr/sbin" +fi + +if [ -f "$gs_path/git-secrets" ]; then echo The command git-secrets is available else echo The command git-secrets is not available, cloning... git clone git@github.com:awslabs/git-secrets.git /tmp/git-secrets/ if [ -f /tmp/git-secrets/git-secrets ]; then - echo "Moving git secrets into PATH" - sudo cp /tmp/git-secrets/git-secrets /usr/sbin/ + + echo "Moving git secrets into PATH" + sudo cp /tmp/git-secrets/git-secrets $gs_path/ + $gs_path/git-secrets --install -f + rm -rf /tmp/git-secrets #cleanup of clone dir else - echo "Git clone failed for git-secrets" + echo "Git clone failed for git-secrets" fi fi # ensure we have correct configs in place -[ -f ../.gitconfig ] -cat .gitconfig >> .git/config -echo "Git-Secrets Config loaded:" -grep -A10 secrets .git/config -# grep will return non-zero code if nothing found, failing the build +if [ -f .gitconfig ]; then + cat .gitconfig >> .git/config + echo "Git-Secrets Config loaded:" + grep -A10 secrets .git/config + # grep will return non-zero code if nothing found, failing the build +fi -echo "git-secrets-check.sh: Scanning repo ..." -git secrets --scan -r ../ -retVal=$? +if [ $islocal ]; then + echo "git-secrets-check.sh: Scanning files staged for commit ..." + setopt shwordsplit + staged_files=$(git diff --cached --name-status | grep -vE "D|^R[0-9]+"| cut -f2 | xargs) + + for filename in $staged_files; do + echo "git-secrets-check.sh: Scanning $filename ..." + git secrets --scan $filename + retVal=$? + if [[ $retVal -ne 0 ]]; then + echo "git-secrets found issues, prevented commit." + return 1 + fi + done + +else + echo "git-secrets-check.sh: Scanning repo ..." + git secrets --scan -r ../ + retVal=$? +fi # if there are issues, they will be listed then script will abort here if [[ $retVal -eq 0 ]]; then @@ -33,3 +61,6 @@ else return 1 fi +#cleanup for testing +rm -rf /tmp/git-secrets +sudo rm -f $gs_path/git-secrets \ No newline at end of file