From 3524e9e82f328dfca1f9441a361700d56a051cba Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:51:51 -0500 Subject: [PATCH 01/37] Remove --runInBand --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 94b2ee5164..12f042baf1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "server:debug": "nodemon src/index.js --exec babel-node --inspect", "client": "yarn --cwd frontend start", "test": "jest src", - "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --runInBand --coverage --reporters=default --reporters=jest-junit", + "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit", "test:all": "yarn test:ci && yarn --cwd frontend test:ci", "lint": "eslint src", "lint:ci": "eslint -f eslint-formatter-multiple src", From 60b68e0b1af1dc23de1b6a92d6675b03a5295a4e Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 11:12:39 -0500 Subject: [PATCH 02/37] Add some docs around testing --- docs/testing.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/testing.md diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000000..ef828edbb9 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,31 @@ +# Testing + +## Tips and Caveats when writing tests + + +### Handling async/promises + +When writing tests that rely on asynchronous operations, such as writing to the database, take care to make sure that those operations are resolved before any tests that rely on them run. If you need to create database records in a setup function such as `beforeAll`, you will want to make sure all async/promise operations resolve before subsequent tests run. You can make sure multiple await (promise) operations resolve by using [`Promise.all()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all) (which takes an iterable of promises). + +Here's how that might look: + +``` +let a = await someAsyncFun(); +let b = await anotherAsyncFun(); + +return Promise.all([a, b]); +``` + + +## Testing in Docker + +When running tests in Docker, be aware that there are tests that will modify/delete database records. For tests to run, the 'db' service needs to exist and `db:migrate` and `db:seed` need to have been run (to create the tables and populate certain records). + +In the `docker-compose.yml` configuration, the database is set up to persist to a volume, "dbdata", so database records will persist between runs of the 'db' service, unless you remove that volume explicitly (e.g. `docker volume rm` or `docker-compose down --volumes`). + + +### Notes on docker-compose and multiple configurations + +`docker-compose` has a feature for providing multiple `docker-compose.*.yml` files where subsequent files can override settings in previous files, which sounds like it would suit the use case of running docker for local development and for testing. However, the ability to [override configurations](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) is limited. While experimenting with overrides, it became clear that doing so would require a minimum of three docker-compose.yml files: one "base", one for local development, one for running tests. Trying to compose docker-compose.yml files would be complicated. + +In addition, while experimenting with multiple configuration files, it became clear that docker was unable to differentiate between different versions of the same service. Trying to override the 'db' service for testing would not work as expected: if the local/dev 'db' service had already been created, that one would be used when tests were run. From 8484d6ec91f6cc74b1bbd6b71fdbbd0f61432186 Mon Sep 17 00:00:00 2001 From: Daniel Riquiac Gopar Date: Mon, 1 Feb 2021 13:05:09 -0800 Subject: [PATCH 03/37] Allow search to be case-insensitive --- frontend/src/pages/Admin/__tests__/index.js | 21 +++++++++++++++++++-- frontend/src/pages/Admin/index.js | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/frontend/src/pages/Admin/__tests__/index.js b/frontend/src/pages/Admin/__tests__/index.js index d73fa2767e..96cd540cad 100644 --- a/frontend/src/pages/Admin/__tests__/index.js +++ b/frontend/src/pages/Admin/__tests__/index.js @@ -29,7 +29,7 @@ describe('Admin Page', () => { const users = [ { id: 2, - email: 'email', + email: 'gs@hogwarts.com', name: undefined, homeRegionId: 1, role: 'Grantee Specialist', @@ -37,7 +37,7 @@ describe('Admin Page', () => { }, { id: 3, - email: 'email', + email: 'potter@hogwarts.com', name: 'Harry Potter', homeRegionId: 1, role: 'Grantee Specialist', @@ -63,6 +63,23 @@ describe('Admin Page', () => { expect(links[0]).toHaveTextContent('Harry Potter'); }); + it('User list is filterable by email', async () => { + const filter = await screen.findByLabelText('Filter Users'); + userEvent.type(filter, '@hogwarts.com'); + const sideNav = screen.getByTestId('sidenav'); + const links = within(sideNav).getAllByRole('link'); + expect(links.length).toBe(2); + }) + + it('user filtering is case-insentive', async () => { + const filter = await screen.findByLabelText('Filter Users'); + userEvent.type(filter, 'harry'); + const sideNav = screen.getByTestId('sidenav'); + const links = within(sideNav).getAllByRole('link'); + expect(links.length).toBe(1); + expect(links[0]).toHaveTextContent('Harry Potter'); + }) + it('allows a user to be selected', async () => { const button = await screen.findByText('Harry Potter'); userEvent.click(button); diff --git a/frontend/src/pages/Admin/index.js b/frontend/src/pages/Admin/index.js index 4c1fe51a31..c4e9802977 100644 --- a/frontend/src/pages/Admin/index.js +++ b/frontend/src/pages/Admin/index.js @@ -81,7 +81,7 @@ function Admin(props) { const filteredUsers = _.filter(users, (u) => { const { email, name } = u; - return `${email}${name}`.includes(userSearch); + return `${email}${name}`.toLowerCase().includes(userSearch.toLowerCase()); }); const onSave = async (newUser) => { From 4ef5971baf8683d7b98ef6d74fe4f0e9ba328b6e Mon Sep 17 00:00:00 2001 From: Daniel Riquiac Gopar Date: Mon, 1 Feb 2021 13:18:42 -0800 Subject: [PATCH 04/37] Fix Eslint issue --- frontend/src/pages/Admin/__tests__/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/Admin/__tests__/index.js b/frontend/src/pages/Admin/__tests__/index.js index 96cd540cad..417bfd905f 100644 --- a/frontend/src/pages/Admin/__tests__/index.js +++ b/frontend/src/pages/Admin/__tests__/index.js @@ -69,7 +69,7 @@ describe('Admin Page', () => { const sideNav = screen.getByTestId('sidenav'); const links = within(sideNav).getAllByRole('link'); expect(links.length).toBe(2); - }) + }); it('user filtering is case-insentive', async () => { const filter = await screen.findByLabelText('Filter Users'); @@ -78,7 +78,7 @@ describe('Admin Page', () => { const links = within(sideNav).getAllByRole('link'); expect(links.length).toBe(1); expect(links[0]).toHaveTextContent('Harry Potter'); - }) + }); it('allows a user to be selected', async () => { const button = await screen.findByText('Harry Potter'); From 3f43234e6051cd0175a86d82ec5871520a580e06 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 16:25:52 -0500 Subject: [PATCH 05/37] Fix example --- docs/testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index ef828edbb9..6184aba6d6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -10,8 +10,8 @@ When writing tests that rely on asynchronous operations, such as writing to the Here's how that might look: ``` -let a = await someAsyncFun(); -let b = await anotherAsyncFun(); +let a = someAsyncFun(); +let b = anotherAsyncFun(); return Promise.all([a, b]); ``` From deac317ffba5ec7b11a52f7a7ae41e0e2b922a3a Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Tue, 26 Jan 2021 15:42:12 -0500 Subject: [PATCH 06/37] Rename docker-compose file used for dynamic security scans --- .circleci/config.yml | 12 ++++++------ docker-compose-test.yml => docker-compose-dss.yml | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename docker-compose-test.yml => docker-compose-dss.yml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9adf5ba70..7265c80d1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,12 +240,12 @@ jobs: - run: name: Start up local server command: | # production style build (single BE server with static FE) - docker-compose -f docker-compose-test.yml run --rm server yarn install --production=false - docker-compose -f docker-compose-test.yml run --rm server yarn --cwd frontend install --production=false - docker-compose -f docker-compose-test.yml run --rm server yarn build - docker-compose -f docker-compose-test.yml run --rm server yarn --cwd frontend run build - docker-compose -f docker-compose-test.yml up -d - docker-compose -f docker-compose-test.yml exec server yarn db:migrate:ci + docker-compose -f docker-compose-dss.yml run --rm server yarn install --production=false + docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend install --production=false + docker-compose -f docker-compose-dss.yml run --rm server yarn build + docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend run build + docker-compose -f docker-compose-dss.yml up -d + docker-compose -f docker-compose-dss.yml exec server yarn db:migrate:ci - run: name: Pull OWASP ZAP docker image command: docker pull owasp/zap2docker-weekly diff --git a/docker-compose-test.yml b/docker-compose-dss.yml similarity index 100% rename from docker-compose-test.yml rename to docker-compose-dss.yml From f3f90222c78658ca0e6616ede22afc188d3568fd Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:11:08 -0500 Subject: [PATCH 07/37] Create bin/docker script, which will make command composition easier --- bin/docker | 195 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bin/docker diff --git a/bin/docker b/bin/docker new file mode 100755 index 0000000000..1d304f39e1 --- /dev/null +++ b/bin/docker @@ -0,0 +1,195 @@ +#!/bin/bash + +hr="==========" + +sequelizecli="node_modules/.bin/sequelize" + +setup() { + # First, setup up frontend and backend, install dependencies + # yarn docker:deps + setup_backend + setup_frontend + + db_setup "" + + # Stop the db manually? + # db_stop +} + +setup_backend() { + echo "${hr} Setting up backend ${hr}" + docker-compose run --rm backend yarn install +} + +setup_frontend() { + echo "${hr} Setting up frontend ${hr}" + docker-compose run --rm frontend yarn install +} + +teardown() { + docker-compose down --volumes +} + +db_migrate() { + local -a options=(status undo) + local option="${1}" + # If 'undo' param provided undo + if [[ "undo" == "${option}" ]]; then + echo "${hr} Undoing db migrations ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate:undo + exit 0 + fi + + if [[ "status" == "${option}" ]]; then + echo "${hr} Checking db migration status ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate:status + exit 0 + fi + + # Create/update database tables + # yarn docker:db:migrate + echo "${hr} Migrating db ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate +} + +db_seed() { + local option="${1}" + # If 'undo' param provided undo + if [[ "undo" == "${option}" ]]; then + echo "${hr} Unseeding db ${hr}" + docker-compose run --rm backend "$sequelizecli" db:seed:undo:all + exit 0 + fi + + # Populate db with data + # yarn docker:db:seed + echo "${hr} Seeding database ${hr}" + docker-compose run --rm backend yarn db:seed +} + +db_setup() { + db_migrate "" + db_seed "" +} + +db_stop() { + docker-compose stop db +} + +# Need reliable way to remove db volume and only db volume +# db_destroy() { +# db_stop +# docker-compose rm -v -- db +# docker volume prune +# } + +db() { + declare -a options=(setup migrate seed stop) + option="${1}" + for i in "${options[@]}"; do + if [ "${i}" == "${option}" ]; then + shift 2>/dev/null + "db_${option}" "$@" + exit 0 + fi + done + + # If no matching option, print options + echo "db expects one of the following: ${options[*]}" + exit 1 +} + +test_frontend() { + echo "${hr} Running frontend tests ${hr}" + docker-compose run --rm frontend yarn test:ci +} + +test_backend() { + echo "${hr} Running backend tests ${hr}" + docker-compose run --rm backend yarn test:ci +} + +# tests plural, because `test` is a builtin shell command +tests() { + declare -a options=(frontend backend) + option="${1}" + + # Check if user specified 'frontend' or 'backend' option + if [[ -n $option ]]; then + for i in "${options[@]}"; do + if [ "${i}" == "${option}" ]; then + "test_${option}" + exit 0 + fi + done + fi + + # Otherwise, run both + test_frontend + test_backend + +} + +lint_backend() { + echo "${hr} Linting backend ${hr}" + docker-compose run --rm backend yarn lint:ci +} + +lint_frontend() { + echo "${hr} Linting frontend ${hr}" + docker-compose run --rm frontend yarn lint:ci +} + +lint() { + local -a options=(frontend backend) + option="${1}" + + for i in "${options[@]}"; do + if [[ "${i}" == "$option" ]]; then + "lint_${option}" + exit 0 + fi + done + + lint_frontend + lint_backend +} + +start() { + docker-compose up +} + +stop() { + docker-compose down +} + +# ====== Main ===== + +# Pull first param as subcommand, default to 'usage' if no args provided +subcommand="${1-usage}"; +# Shift parameters, so $@a has remaining params +shift 2>/dev/null + +# If subcommand is empty string, print usage +if [ -z "$subcommand" ]; then + usage + exit 1 +fi + +usage() { + echo "Please provide a subcommand: ${commands[*]}" +} + +# Array of subcommands +declare -a commands=(usage setup teardown start stop db lint tests) + +# Loop over commands array, look for match with $subcommand. Execute fn if match +for i in "${commands[@]}"; do + if [[ "${i}" == "${subcommand}" ]]; then + "${subcommand}" "$@" + exit 0 + fi +done + +# If we didn't run a subcommand and exit already, print usage +usage From 44af291dd235220a8ec2842d6a5c3442d42954e8 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:51:51 -0500 Subject: [PATCH 08/37] Remove --runInBand --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 94b2ee5164..12f042baf1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "server:debug": "nodemon src/index.js --exec babel-node --inspect", "client": "yarn --cwd frontend start", "test": "jest src", - "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --runInBand --coverage --reporters=default --reporters=jest-junit", + "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit", "test:all": "yarn test:ci && yarn --cwd frontend test:ci", "lint": "eslint src", "lint:ci": "eslint -f eslint-formatter-multiple src", From 9b422f35b4f750430fa98d18149739de01937550 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 17:45:18 -0500 Subject: [PATCH 09/37] Add option parsing and rename docker script --- bin/{docker => docker-run} | 101 ++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 23 deletions(-) rename bin/{docker => docker-run} (65%) diff --git a/bin/docker b/bin/docker-run similarity index 65% rename from bin/docker rename to bin/docker-run index 1d304f39e1..765a26575d 100755 --- a/bin/docker +++ b/bin/docker-run @@ -2,7 +2,23 @@ hr="==========" -sequelizecli="node_modules/.bin/sequelize" +declare sequelizecli="node_modules/.bin/sequelize" + +declare -a environments=("local" "test") +declare env="local" + +# Array of subcommands +declare -a subcommands=( + usage + setup + teardown + start + stop + db + lint + tests +) + setup() { # First, setup up frontend and backend, install dependencies @@ -163,33 +179,72 @@ stop() { docker-compose down } +usage() { + # shellcheck disable=SC2086 + cat <&2 +Run commands in Docker containers +Usage: $0 [-h] [-e env] +Subcommands: + ${subcommands[*]} +EOF +} + # ====== Main ===== -# Pull first param as subcommand, default to 'usage' if no args provided -subcommand="${1-usage}"; -# Shift parameters, so $@a has remaining params -shift 2>/dev/null +main() { + # Parse -options + while getopts ":e:h" opt; do + case $opt in + e) + env="${OPTARG-local}" + local env_match=0 + for e in "${environments[@]}"; do + if [[ "$e" == "$env" ]]; then + env_match=1; + break; + fi + done + if [[ $env_match -eq 0 ]]; then + echo "Invalid environment '${env}'"; + exit 1; + fi + ;; + h) + usage + exit + ;; + \?) + echo "Invalid option" >&2 + echo + ;; + esac + done -# If subcommand is empty string, print usage -if [ -z "$subcommand" ]; then - usage - exit 1 -fi + # After parsing options, shift so $@ has only params + shift $((OPTIND -1)) -usage() { - echo "Please provide a subcommand: ${commands[*]}" -} + # Pull first param as subcommand, default to 'usage' if no args provided + local subcmd="${1-usage}"; -# Array of subcommands -declare -a commands=(usage setup teardown start stop db lint tests) + # Shift parameters, so $@a has remaining params + shift 2>/dev/null -# Loop over commands array, look for match with $subcommand. Execute fn if match -for i in "${commands[@]}"; do - if [[ "${i}" == "${subcommand}" ]]; then - "${subcommand}" "$@" - exit 0 + # If subcommand is empty string, print usage + if [ -z "$subcmd" ]; then + usage + exit 1 fi -done -# If we didn't run a subcommand and exit already, print usage -usage + # Loop over subcommands, look for match with $subcommand. Execute fn if match + for i in "${subcommands[@]}"; do + if [[ "${i}" == "${subcmd}" ]]; then + "${subcmd}" "$@" + exit 0 + fi + done + + # If we didn't run a subcommand and exit already, print usage + usage +} + +main "${@-}" From 84b24b86c079b0b66d341372991a13d64ef93aa9 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 18:55:25 -0500 Subject: [PATCH 10/37] Multiple docker-compose configs for multiple environments --- bin/docker-run | 61 +++++++++++++++++++++++++++++----------- docker-compose.local.yml | 12 ++++++++ docker-compose.test.yml | 7 +++++ docker-compose.yml | 6 ---- 4 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 docker-compose.local.yml create mode 100644 docker-compose.test.yml diff --git a/bin/docker-run b/bin/docker-run index 765a26575d..533eeaa2a4 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -5,7 +5,8 @@ hr="==========" declare sequelizecli="node_modules/.bin/sequelize" declare -a environments=("local" "test") -declare env="local" + +declare -a conf # Array of subcommands declare -a subcommands=( @@ -34,16 +35,18 @@ setup() { setup_backend() { echo "${hr} Setting up backend ${hr}" - docker-compose run --rm backend yarn install + docker-compose "${conf[@]}" run --rm backend yarn install } setup_frontend() { echo "${hr} Setting up frontend ${hr}" - docker-compose run --rm frontend yarn install + docker-compose "${conf[@]}" run --rm frontend yarn install } teardown() { - docker-compose down --volumes + docker-compose \ + "${conf[@]}" \ + down --volumes } db_migrate() { @@ -52,20 +55,20 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate:undo + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:undo exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate:status + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:status exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate } db_seed() { @@ -73,14 +76,14 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose run --rm backend "$sequelizecli" db:seed:undo:all + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:seed:undo:all exit 0 fi # Populate db with data # yarn docker:db:seed echo "${hr} Seeding database ${hr}" - docker-compose run --rm backend yarn db:seed + docker-compose "${conf[@]}" run --rm backend yarn db:seed } db_setup() { @@ -89,7 +92,7 @@ db_setup() { } db_stop() { - docker-compose stop db + docker-compose "${conf[@]}" stop db } # Need reliable way to remove db volume and only db volume @@ -117,12 +120,12 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" - docker-compose run --rm frontend yarn test:ci + docker-compose "${conf[@]}" run --rm frontend yarn test:ci } test_backend() { echo "${hr} Running backend tests ${hr}" - docker-compose run --rm backend yarn test:ci + docker-compose "${conf[@]}" run --rm backend yarn test:ci } # tests plural, because `test` is a builtin shell command @@ -148,12 +151,12 @@ tests() { lint_backend() { echo "${hr} Linting backend ${hr}" - docker-compose run --rm backend yarn lint:ci + docker-compose "${conf[@]}" run --rm backend yarn lint:ci } lint_frontend() { echo "${hr} Linting frontend ${hr}" - docker-compose run --rm frontend yarn lint:ci + docker-compose "${conf[@]}" run --rm frontend yarn lint:ci } lint() { @@ -172,13 +175,36 @@ lint() { } start() { - docker-compose up + docker-compose "${conf[@]}" up } stop() { - docker-compose down + docker-compose "${conf[@]}" down } +# ===== Process env configs + +# We want to specify multiple config files, per +# https://docs.docker.com/compose/extends/#multiple-compose-files +compose_conf() { + local env="${1}" + local envfile="docker-compose.${env}.yml" + + if [[ ! -e "$envfile" ]]; then + echo "$envfile does not appear to exist!" + exit 1 + fi + + configs=( + '-f docker-compose.yml' + "-f docker-compose.${env}.yml" + ) + + echo "${configs[@]}" +} + +# ===== Usage ===== + usage() { # shellcheck disable=SC2086 cat <&2 @@ -192,6 +218,7 @@ EOF # ====== Main ===== main() { + local env="local" # Parse -options while getopts ":e:h" opt; do case $opt in @@ -235,6 +262,8 @@ main() { exit 1 fi + conf=($(compose_conf "$env")) + # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000000..0ce8c997b4 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,12 @@ +version: "3.5" +services: + backend: + environment: + - POSTGRES_HOST=postgres_docker_dev + db: + container_name: postgres_docker_dev + volumes: + - dbdata:/var/lib/postgresql/data + +volumes: + dbdata: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000000..3953f45f96 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + backend: + environment: + - POSTGRES_HOST=postgres_docker_test + db: + container_name: postgres_docker_test diff --git a/docker-compose.yml b/docker-compose.yml index e3fda8a1b9..30e74cd554 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,6 @@ services: - "8080:8080" depends_on: - db - environment: - - POSTGRES_HOST=postgres_docker volumes: - ".:/app:rw" frontend: @@ -36,12 +34,9 @@ services: - BACKEND_PROXY=http://backend:8080 db: image: postgres:12.4 - container_name: postgres_docker env_file: .env ports: - "6543:5432" - volumes: - - dbdata:/var/lib/postgresql/data minio: image: minio/minio env_file: .env @@ -57,5 +52,4 @@ services: volumes: - dbdata: {} miniodata: {} From 92a41c4f75a48c57a40498f122a7cf8ccc1e618d Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Thu, 28 Jan 2021 12:26:52 -0500 Subject: [PATCH 11/37] bin/docker-run fixes, improvements --- bin/docker-run | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index 533eeaa2a4..0bea731f25 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -2,14 +2,12 @@ hr="==========" -declare sequelizecli="node_modules/.bin/sequelize" - -declare -a environments=("local" "test") - -declare -a conf +readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" +readonly DEFAULT_ENV='local' +readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") # Array of subcommands -declare -a subcommands=( +readonly -a subcommands=( usage setup teardown @@ -20,6 +18,8 @@ declare -a subcommands=( tests ) +declare -a conf + setup() { # First, setup up frontend and backend, install dependencies @@ -55,20 +55,20 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:undo + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:undo exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:status + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:status exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate } db_seed() { @@ -76,7 +76,7 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:seed:undo:all + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:seed:undo:all exit 0 fi @@ -217,24 +217,32 @@ EOF # ====== Main ===== +check_env() { + local env_match=0 + for e in "${ENVIRONMENTS[@]}"; do + if [[ "$e" == "${1}" ]]; then + env_match=1; + break; + fi + done + if [[ $env_match -eq 0 ]]; then + echo "Invalid environment '${1}'"; + echo "Valid environments are: ${ENVIRONMENTS[*]}" + exit 1; + fi +} + main() { - local env="local" + docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" + local env_conf="$docker_env" + check_env "$env_conf" + # Parse -options while getopts ":e:h" opt; do case $opt in e) - env="${OPTARG-local}" - local env_match=0 - for e in "${environments[@]}"; do - if [[ "$e" == "$env" ]]; then - env_match=1; - break; - fi - done - if [[ $env_match -eq 0 ]]; then - echo "Invalid environment '${env}'"; - exit 1; - fi + env_conf="${OPTARG:-$env_conf}" + check_env "$env_conf" ;; h) usage @@ -251,7 +259,7 @@ main() { shift $((OPTIND -1)) # Pull first param as subcommand, default to 'usage' if no args provided - local subcmd="${1-usage}"; + local subcmd="${1:-usage}"; # Shift parameters, so $@a has remaining params shift 2>/dev/null @@ -262,7 +270,8 @@ main() { exit 1 fi - conf=($(compose_conf "$env")) + # 'conf' is an array, so we follow https://github.com/koalaman/shellcheck/wiki/SC2207 + IFS=" " read -r -a conf <<< "$(compose_conf "$env_conf")" # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do @@ -276,4 +285,4 @@ main() { usage } -main "${@-}" +main "$@" From 32f38c88813113731c536cd6430bd273aec799c9 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Thu, 28 Jan 2021 13:27:07 -0500 Subject: [PATCH 12/37] Minor docker-run improvements --- bin/docker-run | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index 0bea731f25..e65860aa7a 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -5,6 +5,7 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" readonly DEFAULT_ENV='local' readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") +readonly DB_SERVICE='db' # Array of subcommands readonly -a subcommands=( @@ -15,7 +16,7 @@ readonly -a subcommands=( stop db lint - tests + 'test' ) declare -a conf @@ -102,6 +103,15 @@ db_stop() { # docker volume prune # } +db_exists() { + # TODO: Less naive detection of database service/image/volume + db_lookup="$(docker-compose images --quiet $DB_SERVICE)" + if [[ -n "$db_lookup" ]]; then + return 1 + fi + return 0 +} + db() { declare -a options=(setup migrate seed stop) option="${1}" @@ -124,12 +134,20 @@ test_frontend() { } test_backend() { + # Check if db_exists, if not setup db + local -i has_db + has_db=$(db_exists) + if [[ has_db -eq 0 ]]; then + echo "${hr} Setting up database prior to testing ${hr}" + db_setup + fi echo "${hr} Running backend tests ${hr}" docker-compose "${conf[@]}" run --rm backend yarn test:ci + db_stop } -# tests plural, because `test` is a builtin shell command -tests() { +# 'run_tests', because `test` is a builtin shell command +run_tests() { declare -a options=(frontend backend) option="${1}" @@ -276,6 +294,9 @@ main() { # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then + if [[ "${subcmd}" == "test" ]]; then + subcmd="run_tests"; + fi "${subcmd}" "$@" exit 0 fi From de13b1febdd9abaf1a5ed9ab87e24aee43b76765 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 10:40:09 -0500 Subject: [PATCH 13/37] Change 'conf' to be a string, since that's how it gets used. Echo cmds before running them --- bin/docker-run | 70 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index e65860aa7a..d84e35ebe8 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -19,8 +19,15 @@ readonly -a subcommands=( 'test' ) -declare -a conf - +# conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf +declare conf + +# Expects a command as a string, which it will echo and then execute +echo_exec() { + local cmd="${1}" + echo "$cmd" + $cmd +} setup() { # First, setup up frontend and backend, install dependencies @@ -36,17 +43,19 @@ setup() { setup_backend() { echo "${hr} Setting up backend ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn install + echo_exec "docker-compose ${conf} run --rm backend yarn install" } setup_frontend() { echo "${hr} Setting up frontend ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn install + echo_exec "docker-compose ${conf} run --rm frontend yarn install" + echo "$cmd" + $cmd } teardown() { docker-compose \ - "${conf[@]}" \ + "${conf}" \ down --volumes } @@ -56,20 +65,26 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:undo + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" + echo "$cmd" + $cmd exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:status + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" + echo "$cmd" + $cmd exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" + echo "$cmd" + $cmd } db_seed() { @@ -77,14 +92,18 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:seed:undo:all + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" + echo "$cmd" + $cmd exit 0 fi # Populate db with data # yarn docker:db:seed echo "${hr} Seeding database ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn db:seed + echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" + echo "$cmd" + $cmd } db_setup() { @@ -93,7 +112,9 @@ db_setup() { } db_stop() { - docker-compose "${conf[@]}" stop db + echo_exec "docker-compose ${conf} stop db" + echo "$cmd" + $cmd } # Need reliable way to remove db volume and only db volume @@ -130,7 +151,9 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn test:ci + echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" + echo "$cmd" + $cmd } test_backend() { @@ -142,7 +165,9 @@ test_backend() { db_setup fi echo "${hr} Running backend tests ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn test:ci + echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" + echo "$cmd" + $cmd db_stop } @@ -169,12 +194,16 @@ run_tests() { lint_backend() { echo "${hr} Linting backend ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn lint:ci + echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" + echo "$cmd" + $cmd } lint_frontend() { echo "${hr} Linting frontend ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn lint:ci + echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" + echo "$cmd" + $cmd } lint() { @@ -193,11 +222,15 @@ lint() { } start() { - docker-compose "${conf[@]}" up + echo_exec "docker-compose ${conf} up" + echo "$cmd" + $cmd } stop() { - docker-compose "${conf[@]}" down + echo_exec "docker-compose ${conf} down" + echo "$cmd" + $cmd } # ===== Process env configs @@ -288,8 +321,7 @@ main() { exit 1 fi - # 'conf' is an array, so we follow https://github.com/koalaman/shellcheck/wiki/SC2207 - IFS=" " read -r -a conf <<< "$(compose_conf "$env_conf")" + conf=$(compose_conf "$env_conf") # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do From a232a9155c190fbdc5acf1cbe6484803f1e40072 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 12:48:13 -0500 Subject: [PATCH 14/37] docker-run: 'test' uses test env by default --- bin/docker-run | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index d84e35ebe8..46f51db433 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -4,7 +4,8 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" readonly DEFAULT_ENV='local' -readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") +readonly TEST_ENV='test' +readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") readonly DB_SERVICE='db' # Array of subcommands @@ -21,6 +22,7 @@ readonly -a subcommands=( # conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf declare conf +declare -i env_override=0 # Expects a command as a string, which it will echo and then execute echo_exec() { @@ -112,7 +114,7 @@ db_setup() { } db_stop() { - echo_exec "docker-compose ${conf} stop db" + echo_exec "docker-compose ${conf} stop db" echo "$cmd" $cmd } @@ -252,6 +254,7 @@ compose_conf() { ) echo "${configs[@]}" + return 0 } # ===== Usage ===== @@ -285,6 +288,9 @@ check_env() { main() { docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" + if [[ -n "$DOCKER_ENV" ]]; then + env_override=1 + fi local env_conf="$docker_env" check_env "$env_conf" @@ -294,6 +300,7 @@ main() { e) env_conf="${OPTARG:-$env_conf}" check_env "$env_conf" + env_override=1 ;; h) usage @@ -327,6 +334,12 @@ main() { for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then if [[ "${subcmd}" == "test" ]]; then + if [[ $env_override -eq 0 ]]; then + echo "${hr}> Test commands use 'test' config, unless overridden <${hr}" + conf=$(compose_conf "$TEST_ENV") + else + echo "Running tests in '$env_conf'" + fi subcmd="run_tests"; fi "${subcmd}" "$@" From ac43405c04826797c2fe7bce2b4f55b23deafda6 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:19:07 -0500 Subject: [PATCH 15/37] Undo the composed compose files; docker-compose wasn't handling differend db instances differently --- bin/docker-run | 26 +++++++++-------------- docker-compose.local.yml | 12 ----------- docker-compose.test.yml | 45 +++++++++++++++++++++++++++++++++++++++- docker-compose.yml | 6 ++++++ 4 files changed, 60 insertions(+), 29 deletions(-) delete mode 100644 docker-compose.local.yml diff --git a/bin/docker-run b/bin/docker-run index 46f51db433..ac40ce54d0 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -120,11 +120,10 @@ db_stop() { } # Need reliable way to remove db volume and only db volume -# db_destroy() { -# db_stop -# docker-compose rm -v -- db -# docker volume prune -# } +db_destroy() { + echo "${hr} Destroying db ${hr}" + echo_exec "docker-compose ${conf} down --volumes" +} db_exists() { # TODO: Less naive detection of database service/image/volume @@ -154,8 +153,6 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" - echo "$cmd" - $cmd } test_backend() { @@ -168,9 +165,7 @@ test_backend() { fi echo "${hr} Running backend tests ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" - echo "$cmd" - $cmd - db_stop + db_destroy } # 'run_tests', because `test` is a builtin shell command @@ -243,17 +238,16 @@ compose_conf() { local env="${1}" local envfile="docker-compose.${env}.yml" + if [[ "$env" == "local" ]]; then + envfile="docker-compose.yml" + fi + if [[ ! -e "$envfile" ]]; then echo "$envfile does not appear to exist!" exit 1 fi - configs=( - '-f docker-compose.yml' - "-f docker-compose.${env}.yml" - ) - - echo "${configs[@]}" + echo "-f ${envfile}" return 0 } diff --git a/docker-compose.local.yml b/docker-compose.local.yml deleted file mode 100644 index 0ce8c997b4..0000000000 --- a/docker-compose.local.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.5" -services: - backend: - environment: - - POSTGRES_HOST=postgres_docker_dev - db: - container_name: postgres_docker_dev - volumes: - - dbdata:/var/lib/postgresql/data - -volumes: - dbdata: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 3953f45f96..1776c0854b 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,7 +1,50 @@ version: "3.5" services: backend: + build: + context: . + command: yarn server + user: ${CURRENT_USER:-root} + ports: + - "8080:8080" + depends_on: + - testdb environment: - POSTGRES_HOST=postgres_docker_test - db: + volumes: + - ".:/app:rw" + frontend: + build: + context: . + command: yarn start + user: ${CURRENT_USER:-root} + stdin_open: true + ports: + - "3000:3000" + volumes: + - "./frontend:/app:rw" + - "./scripts:/app/scripts" + environment: + - BACKEND_PROXY=http://backend:8080 + testdb: + image: postgres:12.4 container_name: postgres_docker_test + env_file: .env + ports: + - "6543:5432" + minio: + image: minio/minio + env_file: .env + ports: + - "9000:9000" + volumes: + - miniodata:/data + command: server /data + aws-cli: + image: amazon/aws-cli + env_file: .env + command: ["--endpoint-url", "$S3_ENDPOINT", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"] + + +volumes: + miniodata: {} diff --git a/docker-compose.yml b/docker-compose.yml index 30e74cd554..e3fda8a1b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: - "8080:8080" depends_on: - db + environment: + - POSTGRES_HOST=postgres_docker volumes: - ".:/app:rw" frontend: @@ -34,9 +36,12 @@ services: - BACKEND_PROXY=http://backend:8080 db: image: postgres:12.4 + container_name: postgres_docker env_file: .env ports: - "6543:5432" + volumes: + - dbdata:/var/lib/postgresql/data minio: image: minio/minio env_file: .env @@ -52,4 +57,5 @@ services: volumes: + dbdata: {} miniodata: {} From 188b05f75d63755454204f48878f95e8bd2ef8f6 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:20:16 -0500 Subject: [PATCH 16/37] Remove 2-liners in favor of echo_exec function --- bin/docker-run | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index ac40ce54d0..8061cbfdae 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -51,8 +51,6 @@ setup_backend() { setup_frontend() { echo "${hr} Setting up frontend ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn install" - echo "$cmd" - $cmd } teardown() { @@ -68,16 +66,12 @@ db_migrate() { if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" - echo "$cmd" - $cmd exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" - echo "$cmd" - $cmd exit 0 fi @@ -85,8 +79,6 @@ db_migrate() { # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" - echo "$cmd" - $cmd } db_seed() { @@ -95,8 +87,6 @@ db_seed() { if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" - echo "$cmd" - $cmd exit 0 fi @@ -104,8 +94,6 @@ db_seed() { # yarn docker:db:seed echo "${hr} Seeding database ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" - echo "$cmd" - $cmd } db_setup() { @@ -115,8 +103,6 @@ db_setup() { db_stop() { echo_exec "docker-compose ${conf} stop db" - echo "$cmd" - $cmd } # Need reliable way to remove db volume and only db volume @@ -192,15 +178,11 @@ run_tests() { lint_backend() { echo "${hr} Linting backend ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" - echo "$cmd" - $cmd } lint_frontend() { echo "${hr} Linting frontend ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" - echo "$cmd" - $cmd } lint() { @@ -220,14 +202,10 @@ lint() { start() { echo_exec "docker-compose ${conf} up" - echo "$cmd" - $cmd } stop() { echo_exec "docker-compose ${conf} down" - echo "$cmd" - $cmd } # ===== Process env configs From ece7bb4b25a2f4fb920435b48e569198bf190405 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:40:59 -0500 Subject: [PATCH 17/37] docker-run: Make default config a readonly var --- bin/docker-run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/docker-run b/bin/docker-run index 8061cbfdae..f5aed4bb72 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -3,6 +3,7 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" +readonly DEFAULT_CONFIG='docker-compose.yml' readonly DEFAULT_ENV='local' readonly TEST_ENV='test' readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") @@ -217,7 +218,7 @@ compose_conf() { local envfile="docker-compose.${env}.yml" if [[ "$env" == "local" ]]; then - envfile="docker-compose.yml" + envfile="$DEFAULT_CONFIG" fi if [[ ! -e "$envfile" ]]; then From c8886f0d4364cb21e2af2d683d815bf90e8365ca Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:18:35 -0500 Subject: [PATCH 18/37] Add simpler run-tests script. Improve docker-compose.test.yml with namespaced services, remove minodata volume. --- bin/run-tests | 11 +++++++++++ docker-compose.test.yml | 29 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 11 deletions(-) create mode 100755 bin/run-tests diff --git a/bin/run-tests b/bin/run-tests new file mode 100755 index 0000000000..a1b7f9a63e --- /dev/null +++ b/bin/run-tests @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "Running tests in using test config 'docker-compose.test.yml'" + +docker-compose \ + -f 'docker-compose.test.yml' \ + run test-backend bash -c "yarn db:migrate; yarn db:seed; yarn test:ci" +wait +docker-compose \ + -f 'docker-compose.test.yml' \ + down --volumes diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 1776c0854b..8ffc9491db 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,6 +1,6 @@ version: "3.5" services: - backend: + test-backend: build: context: . command: yarn server @@ -8,12 +8,14 @@ services: ports: - "8080:8080" depends_on: - - testdb + - test-db environment: - POSTGRES_HOST=postgres_docker_test volumes: - ".:/app:rw" - frontend: + networks: + - test + test-frontend: build: context: . command: yarn start @@ -25,26 +27,31 @@ services: - "./frontend:/app:rw" - "./scripts:/app/scripts" environment: - - BACKEND_PROXY=http://backend:8080 - testdb: + - BACKEND_PROXY=http://test-backend:8080 + networks: + - test + test-db: image: postgres:12.4 container_name: postgres_docker_test env_file: .env ports: - "6543:5432" - minio: + networks: + - test + test-minio: image: minio/minio env_file: .env ports: - "9000:9000" - volumes: - - miniodata:/data command: server /data + networks: + - test aws-cli: image: amazon/aws-cli env_file: .env command: ["--endpoint-url", "$S3_ENDPOINT", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"] + networks: + - test - -volumes: - miniodata: {} +networks: + test: From 8b0beced865b3a9733bb5c61123dcc9d33451981 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:20:07 -0500 Subject: [PATCH 19/37] Remove bin/docker-run since we are no longer composing multiple docker-compose files. Small scripts from here on out --- bin/docker-run | 327 ------------------------------------------------- 1 file changed, 327 deletions(-) delete mode 100755 bin/docker-run diff --git a/bin/docker-run b/bin/docker-run deleted file mode 100755 index f5aed4bb72..0000000000 --- a/bin/docker-run +++ /dev/null @@ -1,327 +0,0 @@ -#!/bin/bash - -hr="==========" - -readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" -readonly DEFAULT_CONFIG='docker-compose.yml' -readonly DEFAULT_ENV='local' -readonly TEST_ENV='test' -readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") -readonly DB_SERVICE='db' - -# Array of subcommands -readonly -a subcommands=( - usage - setup - teardown - start - stop - db - lint - 'test' -) - -# conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf -declare conf -declare -i env_override=0 - -# Expects a command as a string, which it will echo and then execute -echo_exec() { - local cmd="${1}" - echo "$cmd" - $cmd -} - -setup() { - # First, setup up frontend and backend, install dependencies - # yarn docker:deps - setup_backend - setup_frontend - - db_setup "" - - # Stop the db manually? - # db_stop -} - -setup_backend() { - echo "${hr} Setting up backend ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn install" -} - -setup_frontend() { - echo "${hr} Setting up frontend ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn install" -} - -teardown() { - docker-compose \ - "${conf}" \ - down --volumes -} - -db_migrate() { - local -a options=(status undo) - local option="${1}" - # If 'undo' param provided undo - if [[ "undo" == "${option}" ]]; then - echo "${hr} Undoing db migrations ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" - exit 0 - fi - - if [[ "status" == "${option}" ]]; then - echo "${hr} Checking db migration status ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" - exit 0 - fi - - # Create/update database tables - # yarn docker:db:migrate - echo "${hr} Migrating db ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" -} - -db_seed() { - local option="${1}" - # If 'undo' param provided undo - if [[ "undo" == "${option}" ]]; then - echo "${hr} Unseeding db ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" - exit 0 - fi - - # Populate db with data - # yarn docker:db:seed - echo "${hr} Seeding database ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" -} - -db_setup() { - db_migrate "" - db_seed "" -} - -db_stop() { - echo_exec "docker-compose ${conf} stop db" -} - -# Need reliable way to remove db volume and only db volume -db_destroy() { - echo "${hr} Destroying db ${hr}" - echo_exec "docker-compose ${conf} down --volumes" -} - -db_exists() { - # TODO: Less naive detection of database service/image/volume - db_lookup="$(docker-compose images --quiet $DB_SERVICE)" - if [[ -n "$db_lookup" ]]; then - return 1 - fi - return 0 -} - -db() { - declare -a options=(setup migrate seed stop) - option="${1}" - for i in "${options[@]}"; do - if [ "${i}" == "${option}" ]; then - shift 2>/dev/null - "db_${option}" "$@" - exit 0 - fi - done - - # If no matching option, print options - echo "db expects one of the following: ${options[*]}" - exit 1 -} - -test_frontend() { - echo "${hr} Running frontend tests ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" -} - -test_backend() { - # Check if db_exists, if not setup db - local -i has_db - has_db=$(db_exists) - if [[ has_db -eq 0 ]]; then - echo "${hr} Setting up database prior to testing ${hr}" - db_setup - fi - echo "${hr} Running backend tests ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" - db_destroy -} - -# 'run_tests', because `test` is a builtin shell command -run_tests() { - declare -a options=(frontend backend) - option="${1}" - - # Check if user specified 'frontend' or 'backend' option - if [[ -n $option ]]; then - for i in "${options[@]}"; do - if [ "${i}" == "${option}" ]; then - "test_${option}" - exit 0 - fi - done - fi - - # Otherwise, run both - test_frontend - test_backend - -} - -lint_backend() { - echo "${hr} Linting backend ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" -} - -lint_frontend() { - echo "${hr} Linting frontend ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" -} - -lint() { - local -a options=(frontend backend) - option="${1}" - - for i in "${options[@]}"; do - if [[ "${i}" == "$option" ]]; then - "lint_${option}" - exit 0 - fi - done - - lint_frontend - lint_backend -} - -start() { - echo_exec "docker-compose ${conf} up" -} - -stop() { - echo_exec "docker-compose ${conf} down" -} - -# ===== Process env configs - -# We want to specify multiple config files, per -# https://docs.docker.com/compose/extends/#multiple-compose-files -compose_conf() { - local env="${1}" - local envfile="docker-compose.${env}.yml" - - if [[ "$env" == "local" ]]; then - envfile="$DEFAULT_CONFIG" - fi - - if [[ ! -e "$envfile" ]]; then - echo "$envfile does not appear to exist!" - exit 1 - fi - - echo "-f ${envfile}" - return 0 -} - -# ===== Usage ===== - -usage() { - # shellcheck disable=SC2086 - cat <&2 -Run commands in Docker containers -Usage: $0 [-h] [-e env] -Subcommands: - ${subcommands[*]} -EOF -} - -# ====== Main ===== - -check_env() { - local env_match=0 - for e in "${ENVIRONMENTS[@]}"; do - if [[ "$e" == "${1}" ]]; then - env_match=1; - break; - fi - done - if [[ $env_match -eq 0 ]]; then - echo "Invalid environment '${1}'"; - echo "Valid environments are: ${ENVIRONMENTS[*]}" - exit 1; - fi -} - -main() { - docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" - if [[ -n "$DOCKER_ENV" ]]; then - env_override=1 - fi - local env_conf="$docker_env" - check_env "$env_conf" - - # Parse -options - while getopts ":e:h" opt; do - case $opt in - e) - env_conf="${OPTARG:-$env_conf}" - check_env "$env_conf" - env_override=1 - ;; - h) - usage - exit - ;; - \?) - echo "Invalid option" >&2 - echo - ;; - esac - done - - # After parsing options, shift so $@ has only params - shift $((OPTIND -1)) - - # Pull first param as subcommand, default to 'usage' if no args provided - local subcmd="${1:-usage}"; - - # Shift parameters, so $@a has remaining params - shift 2>/dev/null - - # If subcommand is empty string, print usage - if [ -z "$subcmd" ]; then - usage - exit 1 - fi - - conf=$(compose_conf "$env_conf") - - # Loop over subcommands, look for match with $subcommand. Execute fn if match - for i in "${subcommands[@]}"; do - if [[ "${i}" == "${subcmd}" ]]; then - if [[ "${subcmd}" == "test" ]]; then - if [[ $env_override -eq 0 ]]; then - echo "${hr}> Test commands use 'test' config, unless overridden <${hr}" - conf=$(compose_conf "$TEST_ENV") - else - echo "Running tests in '$env_conf'" - fi - subcmd="run_tests"; - fi - "${subcmd}" "$@" - exit 0 - fi - done - - # If we didn't run a subcommand and exit already, print usage - usage -} - -main "$@" From 43e18c98b6b383976cd56c7396acb5b8d3756a24 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:25:35 -0500 Subject: [PATCH 20/37] One last rename --- .circleci/config.yml | 12 ++++++------ docker-compose-dss.yml => docker-compose.dss.yml | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename docker-compose-dss.yml => docker-compose.dss.yml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7265c80d1f..4141476207 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,12 +240,12 @@ jobs: - run: name: Start up local server command: | # production style build (single BE server with static FE) - docker-compose -f docker-compose-dss.yml run --rm server yarn install --production=false - docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend install --production=false - docker-compose -f docker-compose-dss.yml run --rm server yarn build - docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend run build - docker-compose -f docker-compose-dss.yml up -d - docker-compose -f docker-compose-dss.yml exec server yarn db:migrate:ci + docker-compose -f docker-compose.dss.yml run --rm server yarn install --production=false + docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false + docker-compose -f docker-compose.dss.yml run --rm server yarn build + docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build + docker-compose -f docker-compose.dss.yml up -d + docker-compose -f docker-compose.dss.yml exec server yarn db:migrate:ci - run: name: Pull OWASP ZAP docker image command: docker pull owasp/zap2docker-weekly diff --git a/docker-compose-dss.yml b/docker-compose.dss.yml similarity index 100% rename from docker-compose-dss.yml rename to docker-compose.dss.yml From 90c4e502edb12a7d496d81a5a551c83542ccd4d0 Mon Sep 17 00:00:00 2001 From: Josh Salisbury Date: Mon, 1 Feb 2021 18:41:14 -0600 Subject: [PATCH 21/37] Managers can add notes and set status of reports * Review page is now different if the user looking at the page is the approving manager * Manager's review page includes the user's additional note and allows the manager to leave a manager note * Manager note added to DB * Report state is shown in the side nav's "review" section once it leaves "draft" mode * Policy, service and route added to update status of a report --- frontend/src/App.js | 2 +- .../components/Navigator/__tests__/index.js | 5 +- .../Navigator/components/SideNav.css | 7 +- .../Navigator/components/SideNav.js | 8 +- .../src/components/Navigator/constants.js | 2 + frontend/src/components/Navigator/index.js | 16 +- frontend/src/fetchers/activityReports.js | 6 + .../ActivityReport/Pages/ReviewSubmit.css | 8 + .../ActivityReport/Pages/ReviewSubmit.js | 166 ++++++++++++++---- .../Pages/__tests__/ReviewSubmit.js | 34 +++- .../src/pages/ActivityReport/Pages/index.js | 36 ++-- .../pages/ActivityReport/__tests__/index.js | 1 + frontend/src/pages/ActivityReport/index.js | 26 ++- src/middleware/authMiddleware.test.js | 7 + ...8-add-manager-notes-to-activity-reports.js | 16 ++ src/models/activityReport.js | 4 + src/policies/activityReport.js | 4 + src/policies/activityReport.test.js | 14 ++ src/routes/activityReports/handlers.js | 33 +++- src/routes/activityReports/handlers.test.js | 43 ++++- src/routes/activityReports/index.js | 9 +- src/services/activityReports.js | 11 ++ 22 files changed, 376 insertions(+), 82 deletions(-) create mode 100644 frontend/src/pages/ActivityReport/Pages/ReviewSubmit.css create mode 100644 src/migrations/20210201174748-add-manager-notes-to-activity-reports.js diff --git a/frontend/src/App.js b/frontend/src/App.js index c992efaef2..88b9b04c9f 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -76,7 +76,7 @@ function App() { ( - + )} /> {admin diff --git a/frontend/src/components/Navigator/__tests__/index.js b/frontend/src/components/Navigator/__tests__/index.js index fde326316e..a11f34e5f7 100644 --- a/frontend/src/components/Navigator/__tests__/index.js +++ b/frontend/src/components/Navigator/__tests__/index.js @@ -42,7 +42,7 @@ const pages = [ label: 'review page', path: 'review', review: true, - render: (hookForm, allComplete, formData, submitted, onSubmit) => ( + render: (hookForm, allComplete, formData, onSubmit) => (
@@ -59,6 +59,9 @@ describe('Navigator', () => { {}} + approvingManager={false} defaultValues={{ first: '', second: '' }} pages={pages} currentPage={currentPage} diff --git a/frontend/src/components/Navigator/components/SideNav.css b/frontend/src/components/Navigator/components/SideNav.css index 685fc85f03..d917223677 100644 --- a/frontend/src/components/Navigator/components/SideNav.css +++ b/frontend/src/components/Navigator/components/SideNav.css @@ -49,8 +49,13 @@ color: #f8f8f8; } +.smart-hub--tag-needs-action { + background-color: #f9e0e4; + color: #d42240; +} + .smart-hub--tag { - width: 84px; + width: 94px; text-align: center; font-weight: normal; display: inline-block; diff --git a/frontend/src/components/Navigator/components/SideNav.js b/frontend/src/components/Navigator/components/SideNav.js index f1e172ff23..7de4990ba3 100644 --- a/frontend/src/components/Navigator/components/SideNav.js +++ b/frontend/src/components/Navigator/components/SideNav.js @@ -13,7 +13,7 @@ import moment from 'moment'; import Container from '../../Container'; import './SideNav.css'; import { - NOT_STARTED, IN_PROGRESS, COMPLETE, SUBMITTED, + NOT_STARTED, IN_PROGRESS, COMPLETE, SUBMITTED, APPROVED, NEEDS_ACTION, } from '../constants'; const tagClass = (state) => { @@ -26,6 +26,10 @@ const tagClass = (state) => { return 'smart-hub--tag-complete'; case SUBMITTED: return 'smart-hub--tag-submitted'; + case APPROVED: + return 'smart-hub--tag-submitted'; + case NEEDS_ACTION: + return 'smart-hub--tag-needs-action'; default: return ''; } @@ -46,7 +50,7 @@ function SideNav({ > {page.label} - {page.state + {page.state !== 'draft' && ( {page.state} diff --git a/frontend/src/components/Navigator/constants.js b/frontend/src/components/Navigator/constants.js index 35cbee5d85..b644ef86cb 100644 --- a/frontend/src/components/Navigator/constants.js +++ b/frontend/src/components/Navigator/constants.js @@ -2,3 +2,5 @@ export const NOT_STARTED = 'Not started'; export const IN_PROGRESS = 'In progress'; export const COMPLETE = 'Complete'; export const SUBMITTED = 'Submitted'; +export const APPROVED = 'Approved'; +export const NEEDS_ACTION = 'Needs Action'; diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 1b7f0c2650..a50fb62a24 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -15,7 +15,7 @@ import moment from 'moment'; import Container from '../Container'; import { - IN_PROGRESS, COMPLETE, SUBMITTED, + IN_PROGRESS, COMPLETE, } from './constants'; import SideNav from './components/SideNav'; import NavigatorHeader from './components/NavigatorHeader'; @@ -24,11 +24,13 @@ function Navigator({ initialData, pages, onFormSubmit, - submitted, + onReview, currentPage, additionalData, onSave, autoSaveInterval, + approvingManager, + status, }) { const [formData, updateFormData] = useState(initialData); const [errorMessage, updateErrorMessage] = useState(); @@ -36,7 +38,6 @@ function Navigator({ const { pageState } = formData; const page = pages.find((p) => p.path === currentPage); - const submittedNavState = submitted ? SUBMITTED : null; const allComplete = _.every(pageState, (state) => state === COMPLETE); const hookForm = useForm({ @@ -101,7 +102,7 @@ function Navigator({ const navigatorPages = pages.map((p) => { const current = p.position === page.position; const stateOfPage = current ? IN_PROGRESS : pageState[p.position]; - const state = p.review ? submittedNavState : stateOfPage; + const state = p.review ? status : stateOfPage; return { label: p.label, onNavigation: () => onSaveForm(false, p.position), @@ -128,9 +129,10 @@ function Navigator({ hookForm, allComplete, formData, - submitted, onFormSubmit, additionalData, + onReview, + approvingManager, )} {!page.review && ( @@ -156,8 +158,10 @@ function Navigator({ Navigator.propTypes = { initialData: PropTypes.shape({}), onFormSubmit: PropTypes.func.isRequired, - submitted: PropTypes.bool.isRequired, onSave: PropTypes.func.isRequired, + status: PropTypes.string.isRequired, + onReview: PropTypes.func.isRequired, + approvingManager: PropTypes.bool.isRequired, pages: PropTypes.arrayOf( PropTypes.shape({ review: PropTypes.bool.isRequired, diff --git a/frontend/src/fetchers/activityReports.js b/frontend/src/fetchers/activityReports.js index 5e4e92f4b3..476d4d734e 100644 --- a/frontend/src/fetchers/activityReports.js +++ b/frontend/src/fetchers/activityReports.js @@ -39,3 +39,9 @@ export const getCollaborators = async (region) => { const collaborators = await get(url); return collaborators.json(); }; + +export const reviewReport = async (reportId, data) => { + const url = join(activityReportUrl, reportId.toString(10), 'review'); + const report = await put(url, data); + return report.json(); +}; diff --git a/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.css b/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.css new file mode 100644 index 0000000000..82dbcd2e28 --- /dev/null +++ b/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.css @@ -0,0 +1,8 @@ +.smart-hub--creator-notes { + padding: 1rem; + background-color: #f8f8f8; +} + +#status { + max-width: 270px; +} diff --git a/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.js b/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.js index 6670b9f252..4b5e12f3d7 100644 --- a/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.js +++ b/frontend/src/pages/ActivityReport/Pages/ReviewSubmit.js @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState } from 'react'; import PropTypes from 'prop-types'; import { Dropdown, Form, Label, Fieldset, Textarea, Alert, Button, Accordion, @@ -6,16 +6,52 @@ import { import { Helmet } from 'react-helmet'; import Container from '../../../components/Container'; +import './ReviewSubmit.css'; + +const possibleStatus = [ + 'Approved', + 'Needs Action', +]; const ReviewSubmit = ({ - hookForm, allComplete, onSubmit, submitted, reviewItems, approvers, + hookForm, + allComplete, + onSubmit, + onReview, + reviewItems, + approvers, + approvingManager, + initialData, }) => { const { handleSubmit, register, formState } = hookForm; + const { additionalNotes } = initialData; const { isValid } = formState; const valid = allComplete && isValid; - const onFormSubmit = (data) => { - onSubmit(data); + const [submitted, updateSubmitted] = useState(false); + const [reviewed, updateReviewed] = useState(false); + const [error, updateError] = useState(); + + const onFormSubmit = async (data) => { + try { + await onSubmit(data); + updateSubmitted(true); + updateError(); + } catch (e) { + updateSubmitted(false); + updateError('Unable to submit report'); + } + }; + + const onFormReview = async (data) => { + try { + await onReview(data); + updateReviewed(true); + updateError(); + } catch (e) { + updateReviewed(false); + updateError('Unable to review report'); + } }; const setValue = (e) => { @@ -32,44 +68,92 @@ const ReviewSubmit = ({ -

Submit Report

- {submitted - && ( - - Success -
- This report was successfully submitted for approval -
- )} - {!allComplete - && ( + {error && ( - Incomplete report + Error
- This report cannot be submitted until all sections are complete + {error}
)} -
-
- -