diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 000000000..74f3e9c7c --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,196 @@ +version: '3' + +tasks: + + create-network: + desc: Create the external network + cmds: + - docker network create external-net + + init-backend: + desc: Initialize the backend project + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml up -d --build + - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py makemigrations" + - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py migrate" + - docker-compose -f docker-compose.yml down + + drop-db: + desc: Drop the backend database + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml down + - docker volume rm tdrs-backend_postgres_data + + backend-up: + desc: Start backend web server + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml up -d + + backend-down: + desc: Stop backend web server + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml down + + backend-logs: + desc: Show and follow backend web server logs + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml logs -f + + backend-restart: + desc: Restart backend web server + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml restart -d + + backend-bash: + desc: Open a shell in the backend container + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml exec web sh + + backend-shell: + desc: Open a Django shell in the backend container + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml exec web sh -c "python ./manage.py shell" + + backend-pytest: + desc: 'Run pytest in the backend container E.g: task backend-pytest PYTEST_ARGS="tdpservice/test/ -s -vv"' + dir: tdrs-backend + vars: + PYTEST_ARGS: '{{.PYTEST_ARGS | default "."}}' + cmds: + - task backend-up + - docker-compose -f docker-compose.yml exec web sh -c "pytest {{.PYTEST_ARGS}}" + + backend-remove-volumes: + desc: Remove the backend volumes + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml down -v + + backend-lint: + desc: Run flake8 in the backend container + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml exec web sh -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi" + + backend-pip-lock: + #TODO: Add a task to lock the pip dependencies + desc: Lock the pip dependencies + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml exec web sh -c "pipenv lock" + + psql: + desc: Open a psql shell in the backend container + dir: tdrs-backend + cmds: + - task create-network || true + - docker-compose -f docker-compose.yml up -d postgres + - sleep 5 + - docker-compose -f docker-compose.yml exec postgres sh -c "psql -U tdpuser -d tdrs_test" + + clean: + desc: Remove all containers, networks, and volumes + cmds: + - docker-compose -f tdrs-backend/docker-compose.yml down -v + - docker-compose -f tdrs-frontend/docker-compose.yml down -v + - docker system prune -f -a + + clamav-up: + desc: Start clamav service + dir: tdrs-backend + cmds: + - docker-compose -f docker-compose.yml up -d clamav-rest + + frontend-up: + desc: Start frontend web server + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml up -d + + frontend-down: + desc: Stop frontend web server + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml down + + frontend-restart: + desc: Restart frontend web server + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml restart -d + + frontend-av: + desc: Start frontend with optional clamav service + dir: tdrs-frontend + cmds: + - task: frontend-up + - task: clamav-up + + + # need more work + frontend-init: + desc: Initialize the frontend project + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml up -d --build + - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "apk add nodejs npm" + - docker-compose -f docker-compose.yml exec tdp-frontend sh -c "npm install" + - docker-compose -f docker-compose.yml down + + frontend-test: + desc: Run frontend tests + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d + - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test" + + frontend-test-cov: + desc: Run frontend tests with coverage + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.local.yml up tdp-frontend-test -d + - docker-compose -f docker-compose.local.yml exec tdp-frontend-test sh -c "npm run test:cov" + + frontend-lint: + desc: Run eslint in the frontend container + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.local.yml up -d tdp-frontend-test --quiet-pull + - docker-compose -f docker-compose.yml exec tdp-frontend-test sh -c "npm run lint" + + frontend-logs: + desc: Show and follow frontend web server logs + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml logs -f + + frontend-bash: + desc: Open a shell in the frontend container + dir: tdrs-frontend + cmds: + - docker-compose -f docker-compose.yml exec tdp-frontend sh + + up: + desc: Start both frontend and backend web servers + cmds: + - task: backend-up + - task: frontend-up + + down: + desc: Stop both frontend and backend web servers + cmds: + - task: backend-down + - task: frontend-down + + help: + desc: Show this help message + cmds: + - task --list \ No newline at end of file diff --git a/commands.sh b/commands.sh deleted file mode 100644 index 0ecd0fe91..000000000 --- a/commands.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/sh -# You will need to set this variable to match your local directory structure -# TDRS_HOME="$HOME/Where/Ever/You/Want/TANF-app" - -# navigate terminal to tdrs home if $TDRS_HOME is set -alias cd-tdrs='cd "$TDRS_HOME"' - -# navigate terminal to tdrs frontend if $TDRS_HOME is set -alias cd-tdrs-frontend='cd "$TDRS_HOME/tdrs-frontend"' - -# navigate terminal to tdrs backend if $TDRS_HOME is set -alias cd-tdrs-backend='cd "$TDRS_HOME/tdrs-backend"' - -# shortcut for applying all relavent compose files for local development -# I.E. `cd-tdrs-frontend && tdrs-compose-local up` -alias tdrs-compose-local='docker-compose -f docker-compose.local.yml' - -# Stop tdrs backend entirely, then start it up again -alias tdrs-backend-hard-restart='tdrs-stop-backend && tdrs-start-backend' - -# shortcut for running bash commands in backend container -alias tdrs-backend-exec='tdrs-compose-backend exec web /bin/bash' - -# Open shell_plus for django backend inside of container -alias tdrs-django-shell='tdrs-compose-backend run --rm web bash -c "python manage.py shell_plus"' - -# start both the frontend and backend -alias tdrs-start='tdrs-start-backend && tdrs-start-frontend' - -# Stop both the frontend and the backend -alias tdrs-stop='tdrs-stop-frontend && tdrs-stop-backend' - -# Restart frontend and backend -alias tdrs-restart='tdrs-restart-backend && tdrs-restart-frontend' - -# start all backend containers -alias tdrs-start-backend='tdrs-compose-backend up -d' - -# run npm install updating all dependencies and start the dev server -alias tdrs-start-frontend='tdrs-compose-frontend up -d' - -# Stop all containers for the backend -alias tdrs-stop-backend='tdrs-compose-backend down' - -# stop the frontend development server -alias tdrs-stop-frontend='tdrs-compose-frontend down' - -# restart the frontends, mainly to rebuild dependencies -alias tdrs-restart-frontend='tdrs-compose-frontend restart' - -# restart all containers for the backend -alias tdrs-restart-backend='tdrs-compose-backend restart' - -# to restart just django, keeping the other containers intact. -alias tdrs-restart-django='tdrs-compose-backend restart web' - -# starts containers with the optional clamav image -alias tdrs-start-av='tdrs-start-frontend --remove-orphans && cd-tdrs-backend && tdrs-compose-local up -d --remove-orphans && docker-compose up -d clamav-rest && cd ..' - -# Run frontend unit tests through jest -alias tdrs-run-jest='tdrs-npm-run test' - -# Run frontend unit tests through jest with coverage report -alias tdrs-run-jest-cov='tdrs-npm-run test:cov' - -# run any new migrations for django backend -alias tdrs-run-migrations='tdrs-compose-backend run web python manage.py migrate' - -# Generate new migrations from changes to models for django backend -alias tdrs-make-migrations='tdrs-compose-backend run --rm web python manage.py makemigrations' - -# Nuke all non running docker data -alias tdrs-prune-all-docker-data='docker system prune -a && docker system prune --volumes' - -# Run eslint against frontend source from frontend container -alias tdrs-lint-frontend='tdrs-npm-run lint' - -# Opens up logs for backend -alias tdrs-logs-backend='docker logs tdrs-backend-web-1 -f' - -# Update backend lockfile -alias tdrs-piplock="tdrs-compose-backend run --rm web pipenv lock" - -# I had to deal with the following issue: -# https://stackoverflow.com/questions/27093746/django-stops-working-with-runtimeerror-populate-isnt-reentrant -# So let's create an alias to ssh into container and sed the file -alias tdrs-fix-django-populate='tdrs-backend-exec && sed -i "s/raise Runtime..populate.. isn.t reentrant../self.app_configs = {}/g" /usr/local/lib/python3.10/site-packages/django/apps/registry.py' - -# A recurring pattern I was doing to get a fresh setup -alias tdrs-fresh-start='tdrs-stop && docker system prune --volumes && tdrs-start' - -# Deploy current branch to the given environment -# See comments in main function for to set up Circle CI token -alias tdrs-deploy='tdrs-run-deploy' - -# run flake8 against backend source from inside of web container -tdrs-lint-backend() { - tdrs-compose-backend run --rm web bash -c "flake8 . && if [ $? -eq 0 ]; then echo 'Flake8 linter found no issues'; fi" -} - -# create docker network for tdrs if it doesn't exist - tdrs-docker-net() { - docker network inspect external-net >/dev/null 2>&1 \ - || docker network create external-net - } - -# short cut for running compose sub commands on backend -tdrs-compose-backend() { - cd-tdrs - tdrs-docker-net - cd tdrs-backend && tdrs-compose-local $@ - cd .. -} - -# short cut for running compose sub commands on backend -tdrs-compose-frontend() { - cd-tdrs - cd tdrs-frontend && tdrs-compose-local $@ - cd .. -} - -# Stop the backend if its running and rebuild the docker container for django -tdrs-rebuild-backend() { - cd-tdrs - tdrs-stop-backend - cd tdrs-backend && tdrs-compose-local up --build -d web - cd .. -} - -# Fix all automatically fixable linting errors for the frontend -tdrs-fix-lint-frontend() { - cd-tdrs-frontend - eslint --fix ./src - cd .. -} - -# Shortcut for running npm scripts for the frontend -tdrs-npm-run() { - cd-tdrs - cd tdrs-frontend/ && npm run $@ - cd .. -} - -# Run pa11y tests on frontend -tdrs-run-pa11y() { - cd tdrs-frontend; mkdir pa11y-screenshots/; npm run test:accessibility - cd .. -} - - -# Spin up backend services and run pytest in docker -tdrs-run-pytest () { - - cd-tdrs - tdrs-start-av - cd tdrs-backend/ - - # to escape quoted arguements that would be passed to docker inside of a quote - if [ "$#" -lt 1 ]; then - quoted_args="" - else - quoted_args="$(printf " %q" "${@}")" - fi - tdrs-compose-local run --rm web bash -c "./wait_for_services.sh && pytest ${quoted_args}" - cd .. -} - - -# Run owasp scan for backend assuming circle ci environment -tdrs-run-backend-owasp() { - if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then - docker network create external-net - fi - cd-tdrs-backend - - # We don't need to use the local compose file - # because we are trying to simulate a production environment - - docker-compose up -d --build - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://web:8080 \ - --timeout 60 \ - -- echo \"Django is ready\"" - cd .. - cd-tdrs-frontend - docker-compose up -d --build - cd .. - ./scripts/zap-scanner.sh backend circle -} - -# Run owasp scan for frontend assuming circle ci environment -tdrs-run-frontend-owasp() { - if [[ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]]; then - docker network create external-net - fi - cd-tdrs-backend - - # We don't need to use the local compose file - # because we are trying to simulate a production environment - - docker-compose up -d --build - docker-compose run --rm zaproxy bash -c \ - "PATH=$PATH:/home/zap/.local/bin && - pip install wait-for-it && - wait-for-it --service http://web:8080 \ - --timeout 60 \ - -- echo \"Django is ready\"" - cd .. - cd-tdrs-frontend - docker-compose up -d --build - cd .. - ./scripts/zap-scanner.sh frontend circle -} - -tdrs-run-deploy() { - # Circle CI token can be generated here: https://app.circleci.com/settings/user/tokens - # Once generated add it to your shell profile as CIRCLE_CI_TOKEN - # Use like: tdrs-deploy sandbox - TARGET_ENV=$1 - BRANCH=$(git rev-parse --abbrev-ref HEAD) - - echo "Deploying branch $BRANCH to $TARGET_ENV" - - curl --request POST \ - --url https://circleci.com/api/v2/project/github/raft-tech/TANF-app/pipeline \ - --header 'Circle-Token: '$CIRCLE_CI_TOKEN \ - --header 'content-type: application/json' \ - --data '{"parameters":{"triggered": true, "run_dev_deployment": true, "target_env":"'$TARGET_ENV'"}, "branch":"'$BRANCH'"}' -} - -# List all aliases and functions associated with tdrs -alias tdrs-functions='declare -F|grep tdrs && alias|grep tdrs|cut -d" " -f1 --complement' - -# Get logs on backend -alias tdrs-backend-log="docker logs $(docker ps|grep web|awk '{print $1}')" diff --git a/docs/Security-Compliance/Security-Controls/cm-7-2/images/clamavcm7.PNG b/docs/Security-Compliance/Security-Controls/cm-7-2/images/clamavcm7.PNG index 7dd04cadd..d3a599839 100644 Binary files a/docs/Security-Compliance/Security-Controls/cm-7-2/images/clamavcm7.PNG and b/docs/Security-Compliance/Security-Controls/cm-7-2/images/clamavcm7.PNG differ diff --git a/tdrs-backend/README.md b/tdrs-backend/README.md index 2cb44b007..83a7d2f60 100644 --- a/tdrs-backend/README.md +++ b/tdrs-backend/README.md @@ -8,6 +8,7 @@ Backend API Service for TDP. Deployed to Cloud.gov at https://tdp-backend.app.cl - [Login.gov Account](https://login.gov/) - [Cloud.gov Account](https://cloud.gov/) - [Cloud Foundry CLI](https://docs.cloudfoundry.org/cf-cli/install-go-cli.html) +- [Task file](https://taskfile.dev/installation/) # Contents @@ -25,6 +26,8 @@ This project uses a Pipfile for dependency management. **Commands are to be executed from within the `tdrs-backend` directory** +Note: first step is to install Taskfile + 1.) Configure your local environment by copying over the .env.example file ```bash $ cp .env.example .env @@ -41,7 +44,7 @@ with the email you use to login to [login.gov](https://login.gov) ```bash # Merge in local overrides for docker-compose by using -f flag and specifying both # This allows environment variables to be passed in from .env files locally. -$ docker-compose -f docker-compose.yml -f docker-compose.local.yml up --build -d +$ task backend-up ``` This command will start the following containers: @@ -59,7 +62,7 @@ a64c18db30ed localstack/localstack:0.12.9 "docker-entrypoint.sh" 2 hour 6.) To `exec` into the PostgreSQL database in the container. ```bash -$ docker exec -it tdrs-backend_postgres_1 psql -U tdpuser -d tdrs_test +$ task psql ``` 7.) For configuration of a superuser for admin tasks please refer to the [user_role_management.md](../docs/user_role_management.md) guide. @@ -67,13 +70,19 @@ $ docker exec -it tdrs-backend_postgres_1 psql -U tdpuser -d tdrs_test 8.) Backend project tear down: ```bash - $ docker-compose down --remove-orphans + $ task backend-down ``` 9.) The `postgres` and `localstack` containers use [Docker Named Volumes](https://spin.atomicobject.com/2019/07/11/docker-volumes-explained/) to persist container data between tear down and restart of containers. To clear all stored data and reset to an initial state, pass the `-v` flag when tearing down the containers: ```bash - $ docker-compose down -v + $ task backend-remove-volume +``` + +10.) To remove all volumes, containers and images, we can run the following command. Note that this will remove all containers including containers outside of this project. + +```bash +$ task clean ``` ---- @@ -116,13 +125,13 @@ s3_client.generate_presigned_url(**params) 1. Run local unit tests by executing the following command. ```bash -$ docker-compose run --rm web bash -c "./wait_for_services.sh && pytest" +$ task backend-pytest ``` 2. Run local linting tests by executing the following command: ```bash -$ docker-compose run --rm web bash -c "flake8 ." +$ task backend-lint ``` The [flake8](https://flake8.pycqa.org/en/latest/) linter is configured to check the formatting of the source against this [setup.cfg](./setup.cfg#L20-L34) file. diff --git a/tdrs-frontend/docker-compose.local.yml b/tdrs-frontend/docker-compose.local.yml index 5664b16d8..5f868bd23 100644 --- a/tdrs-frontend/docker-compose.local.yml +++ b/tdrs-frontend/docker-compose.local.yml @@ -16,6 +16,18 @@ services: - ./:/home/node/app networks: - local + + tdp-frontend-test: + stdin_open: true # docker run -i + tty: true # docker run -t + build: + context: . + target: localdev + command: sleep infinity + volumes: + - ./:/home/node/app + networks: + - local networks: local: driver: bridge