From 984284bc87ad3e51a2c8ef5fe65a21146b8b4eaf Mon Sep 17 00:00:00 2001 From: David Burke Date: Thu, 21 Nov 2024 09:26:12 -0500 Subject: [PATCH 1/4] Start attempting a single compoes file option --- .gitignore | 1 - docker-compose.yml | 102 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 5d259a00..23db1f11 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ log/ backups/ secrets/ build -docker-compose.yml .transifexrc .idea/* *.override.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ec3a327b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,102 @@ +x-environment: &default-environment + PUBLIC_REQUEST_SCHEME: http + INTERNAL_DOMAIN_NAME: kobo.local + PUBLIC_DOMAIN_NAME: kobo.local + KOBOFORM_URL: http://kf.kobo.local + KOBOCAT_URL: http://kf.kobo.local + KOBOFORM_PUBLIC_SUBDOMAIN: kf + KOBOCAT_PUBLIC_SUBDOMAIN: kc + ENKETO_EXPRESS_PUBLIC_SUBDOMAIN: ee + ENKETO_REDIS_MAIN_URL: redis://redis:6379 +services: + postgres: + image: postgis/postgis + environment: + POSTGRES_HOST_AUTH_METHOD: trust + networks: + kobo-be-network: + aliases: + - postgres.kobo.private + mongo: + image: mongo + networks: + kobo-be-network: + aliases: + - mongo.kobo.private + redis: + image: redis + networks: + kobo-be-network: + aliases: + - redis.kobo.private + kpi: &django + image: kobotoolbox/kpi:2.024.33c + environment: + <<: *default-environment + DJANGO_DEBUG: "true" + USE_X_FORWARDED_HOST: "true" + PYTHONUNBUFFERED: 1 + DATABASE_URL: postgis://postgres:postgres@postgres:5432/postgres + KC_DATABASE_URL: postgis://postgres:postgres@postgres:5432/kobocat + SESSION_COOKIE_DOMAIN: .kobo.local + REDIS_SESSION_URL: redis://redis:6379 + CELERY_BROKER_URL: redis://redis:6379 + CACHE_URL: redis://redis:6379 + SERVICE_ACCOUNT_BACKEND_URL: redis://redis:6379 + command: ./manage.py runserver 0:8000 + volumes: + - ./.vols/static/kpi:/srv/src/kpi/staticfiles + networks: + kobo-fe-network: + aliases: + - kpi.docker.container + - kpi.internal + - kobocat.docker.container + - kobocat.internal + kobo-be-network: + aliases: + - kpi.docker.container + worker: + <<: *django + command: celery -A kobo worker -l info -s /tmp/celerybeat-schedule --pool threads -Q kpi_low_priority_queue,kpi_queue,kobocat_queue + nginx: + image: nginx + environment: + <<: *default-environment + TEMPLATED_VAR_REFS: "$${PUBLIC_REQUEST_SCHEME} $${INTERNAL_DOMAIN_NAME} $${PUBLIC_DOMAIN_NAME} $${KOBOFORM_PUBLIC_SUBDOMAIN} $${KOBOCAT_PUBLIC_SUBDOMAIN} $${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}" + networks: + kobo-fe-network: + aliases: + - nginx.internal + - nginx.docker.container + ports: + - 80:80 + volumes: + - ./.vols/static:/srv/www:ro + - ./.vols/kobocat_media_uploads:/media + - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro + - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh + - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts + enketo_express: + image: kobotoolbox/enketo-express-extra-widgets:6.2.2 + environment: + <<: *default-environment + ENKETO_REDIS_CACHE_URL: "redis://redis:6379" + ENKETO_MAX_PROCESSES: 1 + ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kf.kobo.local:8080 + ENKETO_LINKED_FORM_AND_DATA_SERVER_AUTHENTICATION_ALLOW_INSECURE_TRANSPORT: "true" + command: node app.js + networks: + kobo-fe-network: + aliases: + - enketo_express.internal + - enketo_express.docker.container + kobo-be-network: + aliases: + - enketo_express + +networks: + kobo-fe-network: + driver: bridge + kobo-be-network: + driver: bridge From 072a3ceab0542bd8f840342670c4a62c6b9ba081 Mon Sep 17 00:00:00 2001 From: David Burke Date: Thu, 21 Nov 2024 15:47:22 -0500 Subject: [PATCH 2/4] . --- docker-compose.yml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ec3a327b..e04f5c0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,10 @@ x-environment: &default-environment PUBLIC_REQUEST_SCHEME: http INTERNAL_DOMAIN_NAME: kobo.local PUBLIC_DOMAIN_NAME: kobo.local - KOBOFORM_URL: http://kf.kobo.local - KOBOCAT_URL: http://kf.kobo.local + KOBOCAT_URL: http://kc.kobo.local + KOBOCAT_INTERNAL_URL: http://kobocat:8000 KOBOFORM_PUBLIC_SUBDOMAIN: kf KOBOCAT_PUBLIC_SUBDOMAIN: kc - ENKETO_EXPRESS_PUBLIC_SUBDOMAIN: ee ENKETO_REDIS_MAIN_URL: redis://redis:6379 services: postgres: @@ -14,21 +13,15 @@ services: environment: POSTGRES_HOST_AUTH_METHOD: trust networks: - kobo-be-network: - aliases: - - postgres.kobo.private + - kobo-be-network mongo: image: mongo networks: - kobo-be-network: - aliases: - - mongo.kobo.private + - kobo-be-network redis: image: redis networks: - kobo-be-network: - aliases: - - redis.kobo.private + - kobo-be-network kpi: &django image: kobotoolbox/kpi:2.024.33c environment: @@ -43,6 +36,9 @@ services: CELERY_BROKER_URL: redis://redis:6379 CACHE_URL: redis://redis:6379 SERVICE_ACCOUNT_BACKEND_URL: redis://redis:6379 + ENKETO_URL: http://ee.kobo.local + ENKETO_INTERNAL_URL: http://ee.kobo.internal + ENKETO_API_KEY: "abc123" command: ./manage.py runserver 0:8000 volumes: - ./.vols/static/kpi:/srv/src/kpi/staticfiles @@ -53,6 +49,7 @@ services: - kpi.internal - kobocat.docker.container - kobocat.internal + - kobocat kobo-be-network: aliases: - kpi.docker.container @@ -69,6 +66,12 @@ services: aliases: - nginx.internal - nginx.docker.container + - kf.kobo.local + - kc.kobo.local + - ee.kobo.local + - kf.kobo.internal + - kc.kobo.internal + - ee.kobo.internal ports: - 80:80 volumes: @@ -79,12 +82,17 @@ services: - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts enketo_express: image: kobotoolbox/enketo-express-extra-widgets:6.2.2 + init: true environment: <<: *default-environment ENKETO_REDIS_CACHE_URL: "redis://redis:6379" ENKETO_MAX_PROCESSES: 1 - ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kf.kobo.local:8080 + #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kc.kobo.local + #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kpi:8000 + #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kc.kobo.local + ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: "" ENKETO_LINKED_FORM_AND_DATA_SERVER_AUTHENTICATION_ALLOW_INSECURE_TRANSPORT: "true" + ENKETO_LINKED_FORM_AND_DATA_SERVER_API_KEY: "abc123" command: node app.js networks: kobo-fe-network: From e2d95f74ad7dc0509d9e639fdd356b877c33c043 Mon Sep 17 00:00:00 2001 From: David Burke Date: Thu, 21 Nov 2024 16:31:34 -0500 Subject: [PATCH 3/4] docker-compose.yml is small and works. --- docker-compose.yml | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e04f5c0d..33a5942e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,9 +26,9 @@ services: image: kobotoolbox/kpi:2.024.33c environment: <<: *default-environment - DJANGO_DEBUG: "true" + DJANGO_DEBUG: "true" # Remove? USE_X_FORWARDED_HOST: "true" - PYTHONUNBUFFERED: 1 + PYTHONUNBUFFERED: 1 # Move to Dockerfile? DATABASE_URL: postgis://postgres:postgres@postgres:5432/postgres KC_DATABASE_URL: postgis://postgres:postgres@postgres:5432/kobocat SESSION_COOKIE_DOMAIN: .kobo.local @@ -37,22 +37,14 @@ services: CACHE_URL: redis://redis:6379 SERVICE_ACCOUNT_BACKEND_URL: redis://redis:6379 ENKETO_URL: http://ee.kobo.local - ENKETO_INTERNAL_URL: http://ee.kobo.internal - ENKETO_API_KEY: "abc123" command: ./manage.py runserver 0:8000 volumes: - ./.vols/static/kpi:/srv/src/kpi/staticfiles networks: kobo-fe-network: aliases: - - kpi.docker.container - - kpi.internal - - kobocat.docker.container - - kobocat.internal - kobocat - kobo-be-network: - aliases: - - kpi.docker.container + kobo-be-network: {} worker: <<: *django command: celery -A kobo worker -l info -s /tmp/celerybeat-schedule --pool threads -Q kpi_low_priority_queue,kpi_queue,kobocat_queue @@ -64,14 +56,9 @@ services: networks: kobo-fe-network: aliases: - - nginx.internal - - nginx.docker.container - kf.kobo.local - kc.kobo.local - ee.kobo.local - - kf.kobo.internal - - kc.kobo.internal - - ee.kobo.internal ports: - 80:80 volumes: @@ -87,21 +74,12 @@ services: <<: *default-environment ENKETO_REDIS_CACHE_URL: "redis://redis:6379" ENKETO_MAX_PROCESSES: 1 - #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kc.kobo.local - #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kpi:8000 - #ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: kc.kobo.local ENKETO_LINKED_FORM_AND_DATA_SERVER_SERVER_URL: "" ENKETO_LINKED_FORM_AND_DATA_SERVER_AUTHENTICATION_ALLOW_INSECURE_TRANSPORT: "true" - ENKETO_LINKED_FORM_AND_DATA_SERVER_API_KEY: "abc123" command: node app.js networks: - kobo-fe-network: - aliases: - - enketo_express.internal - - enketo_express.docker.container - kobo-be-network: - aliases: - - enketo_express + - kobo-fe-network + - kobo-be-network networks: kobo-fe-network: From 19a3a6bbbde3f6b8e384e02d2828a1421b8f0dd9 Mon Sep 17 00:00:00 2001 From: David Burke Date: Mon, 25 Nov 2024 16:30:19 -0500 Subject: [PATCH 4/4] Start extending compose files --- README.md | 224 ++------------------------ README.old.md | 217 +++++++++++++++++++++++++ docker-compose.external-databases.yml | 23 +++ docker-compose.frontend.yml | 40 +++-- docker-compose.yml | 14 +- 5 files changed, 287 insertions(+), 231 deletions(-) create mode 100644 README.old.md create mode 100644 docker-compose.external-databases.yml diff --git a/README.md b/README.md index b2dd95dc..dab53b57 100644 --- a/README.md +++ b/README.md @@ -1,217 +1,23 @@ -# kobo-docker +# Kobo Docker -1. [Introduction](#introduction) -1. [_Two PostgreSQL databases are now required_](#two-postgresql-databases-are-now-required) -1. [Important notice when upgrading from commit `5c2ef02` (March 4, 2019) or earlier](#important-notice-when-upgrading-from-commit-5c2ef02-march-4-2019-or-earlier) -1. [Important notice when upgrading from commit between `51aeccb` (March 11, 2019) and `2.022.44` (November 17, 2022)](#important-notice-when-upgrading-from-commit-between-51aeccb-march-11-2019-and-202244-november-17-2022) -1. [Architecture](#architecture) -1. [Setup procedure](#setup-procedure) -1. [Usage](#usage) - - Start/stop - - Backups - - Restore backups - - Maintenance -1. [Troubleshooting](#troubleshooting) -1. [Redis performance](#redis-performance) +kobo-docker runs [KoboToolbox](http://www.kobotoolbox.org) survey data collection platform, using [Docker Compose](https://docker.com). +kobo-docker may be used for development and single server production deployments. +See general documentation at [support.kobotoolbox.org](https://support.kobotoolbox.org/kobo_your_servers.html). -## Introduction +## Run KoboToolbox all-in-one -kobo-docker is used to run a copy of the [KoboToolbox](http://www.kobotoolbox.org) survey data collection platform on a machine of your choosing. It relies on [Docker](https://docker.com) to separate the different parts of KoBo into different containers (which can be thought of as lighter-weight virtual machines) and [Docker Compose](https://docs.docker.com/compose/) to configure, run, and connect those containers. +## Run KoboToolbox with your own databases +`docker compose -f docker-compose.external-databases.yml up` -## _Two PostgreSQL databases are now required_ +## Developing KoboToolbox -Prior to release [`2.020.18`](https://github.com/kobotoolbox/kobo-docker/releases/tag/2.020.18), -[KPI](https://github.com/kobotoolbox/kpi) and -[KoBoCAT](https://github.com/kobotoolbox/kobocat) both shared a common PostgreSQL -database. They now each have their own, separate databases. -**If you are upgrading an existing single-database installation, you must follow [these instructions](https://community.kobotoolbox.org/t/upgrading-to-separate-databases-for-kpi-and-kobocat/7202)** to migrate the KPI tables to a new database and adjust your configuration appropriately. -This assumes your last upgrade was **more recent** than March 4, 2019. If not, you must [upgrade your databases](#important-notice-when-upgrading-from-commit-5c2ef02-march-4-2019-or-earlier) before proceeding. +### First run -If you do not want to upgrade at this time, please use the [`shared-database-obsolete`](https://github.com/kobotoolbox/kobo-docker/tree/shared-database-obsolete) branch instead. - - -## Important notice when upgrading from commit [`5c2ef02` (March 4, 2019)](https://github.com/kobotoolbox/kobo-docker/commit/5c2ef0273339bee5c374830f72e52945947042a8) or earlier - -Running current releases of KoboToolbox requires you to upgrade your PostgreSQL and MongoDB databases. Please follow [these instructions](./doc/March-2019-Upgrade.md). - -If you do not, the application may not start or your data may not be visible. - -## Important notice when upgrading from commit between [`51aeccb` (March 11, 2019)](https://github.com/kobotoolbox/kobo-docker/commit/51aeccb91316d640f30e28190b936ae48d56c468) and [`2.022.44` (November 17, 2022)](https://github.com/kobotoolbox/kobo-docker/releases/tag/2.022.44) - -Running current releases of KoboToolbox requires you to upgrade your PostgreSQL and MongoDB databases. Please follow [these instructions](./doc/November-2022-Upgrade.md). - -If you do not, the application may not start or your data may not be visible. - -## Architecture - -Below is a diagram (made with [Lucidchart](https://www.lucidchart.com)) of the containers that make up a running kobo-docker system and their connections. - -![Diagram of Docker Containers](./doc/container-diagram.svg) - -### Secure your installation -This version of kobo-docker does **not** expose backend container ports, but [previous versions did](https://github.com/kobotoolbox/kobo-docker/pull/280), relying on a firewall to prevent unauthorized access. You should always verify that your database ports (by default 5432, 27017, 6379, 6380) are not accessible to the public. - -If you want to use kobo-docker with separate front-end and back-end servers, you will need to expose ports, and **you MUST use a firewall**. The firewall is required to allow only the `frontend` containers to access PostgreSQL, Redis, and MongoDB. - - -## Setup procedure - -This procedure has been simplified by using [kobo-install](https://github.com/kobotoolbox/kobo-install ""). -Please use it to install kobo-docker. - -Already have an existing installation? Please see below. - -1. **Migrating from RabbitMQ to Redis as the Celery (asynchronous task) broker** - - The easiest way is to rely on [kobo-install](https://github.com/kobotoolbox/kobo-install) to generate the correct environment files. - - If you want to change it manually, edit: - - - `kobo-env/envfiles/kpi.txt` - ```diff - - KPI_BROKER_URL=amqp://kpi:kpi@rabbit.[internal domain name]:5672/kpi - + KPI_BROKER_URL=redis://redis-main.[internal domain name]:6389/1 - ``` - - `kobo-env/envfiles/kobocat.txt` - ```diff - - KOBOCAT_BROKER_URL=amqp://kobocat: kobocat@rabbit.[internal domain name]:5672/kobocat - + KOBOCAT_BROKER_URL=redis://redis-main.[internal domain name]:6389/2 - ``` - -1. **Load balancing and redundancy** - - 1. Load balancing - kobo-docker has two different composer files. One for `frontend` and one for `backend`. - - 1. `frontend`: - - NGINX - - [KoBoCAT](https://github.com/kobotoolbox/kobocat) - - [KPI](https://github.com/kobotoolbox/kpi) - - [Enketo Express](https://github.com/enketo/enketo-express) - - 1. `backend`: - - PostgreSQL - - MongoDB - - Redis - - Docker-compose for `frontend` can be started on its own server, same thing for `backend`. Users can start as many front-end servers they want. A load balancer can spread the traffic between front-end servers. - kobo-docker uses (private) domain names between `frontend` and `backend`. - It's fully customizable in configuration files. Once again, [kobo-install](https://github.com/kobotoolbox/kobo-install) does simplify the job by creating the configuration files for you. - - 1. Redundancy - `Backend` containers not redundant yet. Only `PostgreSQL` can be configured in `Primary/Secondary` mode where `Secondary` is a real-time read-only replica. - - This is a diagram that shows how kobo-docker can be used for a load-balanced/(almost) redundant solution. - - _NB: The diagram is based on AWS infrastructure, but it's not required to host your environment there._ - - ![aws diagram](./doc/aws-diagram.svg) - -## Usage -It's recommended to create `*.override.yml` docker-compose files to customize your environment. It makes easier to update. -Samples are provided. Remove `.sample` extension and update them to match your environment. - -- `docker-compose.frontend.override.yml` -- `docker-compose.backend.primary.override.yml` -- `docker-compose.backend.secondary.override.yml` (if a PostgreSQL replica is used) - -1. **Start/start containers** - - ``` - # Start - $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml up -d - $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml up -d - - # Stop - $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml stop - $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml stop - ``` - -1. **Backups** - - Automatic, periodic backups of KoBoCAT media, MongoDB, PostgreSQL and Redis can be individually enabled by uncommenting (and optionally customizing) the `*_BACKUP_SCHEDULE` variables in your envfiles. - - - `deployments/envfiles/databases.txt` (MongoDB, PostgreSQL, Redis) - - `deployments/envfiles/kobocat.txt` (KoBoCat media) - - When enabled, timestamped backups will be placed in backups/kobocat, backups/mongo, backups/postgres and backups/redis respectively. - - If `AWS` credentials and `AWS S3` backup bucket name are provided, the backups are created directly on `S3`. - - Backups **on disk** can also be manually triggered when kobo-docker is running by executing the the following commands: - - ``` - $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml exec kobocat /srv/src/kobocat/docker/backup_media.bash - $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec mongo bash /kobo-docker-scripts/backup-to-disk.bash - $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec -e PGUSER=kobo postgres bash /kobo-docker-scripts/backup-to-disk.bash - $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec redis_main bash /kobo-docker-scripts/backup-to-disk.bash - ``` - -1. **Restore backups** - - Commands should be run within containers. - - - MongoDB: `mongorestore --archive= --gzip` - - PostgreSQL: `pg_restore -U kobo -d kobotoolbox -c ""` - - Redis: `gunzip && mv /data/enketo-main.rdb` - -1. **Maintenance mode** - - There is one composer file `docker-compose.maintenance.yml` can be used to put `KoboToolbox` in maintenance mode. - Like front-end or back-end containers, a `docker-compose.maintenance.yml.sample` file is provided to help you to customize your settings. - First, copy `docker-compose.maintenance.yml.sample` to `docker-compose.maintenance.yml`. - - There are 4 variables that can be customized in `docker-compose.maintenance.override.yml`: - - - `ETA` e.g. `2 hours` - - `DATE_STR` e.g. `Monday, November 26 at 02:00 GMT` - - `DATE_ISO` e.g. `20181126T02` - - `EMAIL` e.g. `support@example.com` - - NGINX container has to be stopped before launching the maintenance container. - - **Start** - - ``` - docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml stop nginx - docker-compose -f docker-compose.maintenance.yml -f docker-compose.maintenance.override.yml up -d - ``` - - **Stop** - - ``` - docker-compose -f docker-compose.maintenance.yml -f docker-compose.maintenance.override.yml down - docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml up -d nginx - ``` - -## Troubleshooting - -- ### Basic troubleshooting - You can confirm that your containers are running with `docker ps`. - To inspect the log output from: - - - the frontend containers, execute `docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml logs -f` - - the primary backend containers, execute `docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml logs -f` - - the secondary backend container, execute `docker-compose -f docker-compose.backend.secondary.yml -f docker-compose.backend.secondary.override.yml logs -f` - - For a specific container use e.g. `docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml logs -f redis_main`. - - The documentation for Docker can be found at https://docs.docker.com. - -- ### Django debugging - Developers can use [PyDev](http://www.pydev.org/)'s [remote, graphical Python debugger](http://www.pydev.org/manual_adv_remote_debugger.html) to debug Python/Django code. To enable for the `kpi` container: - - 1. Specify the mapping(s) between target Python source/library paths on the debugging machine to the locations of those files/directories inside the container by customizing and uncommenting the `KPI_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS` variable in [`envfiles/kpi.txt`](./envfiles/kpi.txt). - 1. Share the source directory of the PyDev remote debugger plugin into the container by customizing (taking care to note the actual location of the version-numbered directory) and uncommenting the relevant `volumes` entry in your `docker-compose.yml`. - 1. To ensure PyDev shows you the same version of the code as is being run in the container, share your live version of any target Python source/library files/directories into the container by customizing and uncommenting the relevant `volumes` entry in your `docker-compose.yml`. - 1. Start the PyDev remote debugger server and ensure that no firewall or other settings will prevent the containers from connecting to your debugging machine at the reported port. - 1. Breakpoints can be inserted with: `import pydevd; pydevd.settrace('${DEBUGGING_MACHINE_IP}')`. - - Remote debugging in the `kobocat` container can be accomplished in a similar manner. - - -## Redis performance -Please take a look at [https://www.techandme.se/performance-tips-for-redis-cache-server/](https://www.techandme.se/performance-tips-for-redis-cache-server/) -to get rid of Warning message when starting redis containers +1. Build `docker compose build --pull` +2. Start postgres `docker compose up postgres` this ensures it has time to initialize +3. Run Django database migrations `docker compose run --rm kpi scripts/migrate.sh` +4. Make user `docker compose run --rm kpi ./manage.py createsuperuser` +5. Edit `/etc/hosts` and add `127.0.0.1 kf.kobo.local ee.kobo.local` +6. Run `npm i` in the kpi directory. diff --git a/README.old.md b/README.old.md new file mode 100644 index 00000000..b2dd95dc --- /dev/null +++ b/README.old.md @@ -0,0 +1,217 @@ +# kobo-docker + +1. [Introduction](#introduction) +1. [_Two PostgreSQL databases are now required_](#two-postgresql-databases-are-now-required) +1. [Important notice when upgrading from commit `5c2ef02` (March 4, 2019) or earlier](#important-notice-when-upgrading-from-commit-5c2ef02-march-4-2019-or-earlier) +1. [Important notice when upgrading from commit between `51aeccb` (March 11, 2019) and `2.022.44` (November 17, 2022)](#important-notice-when-upgrading-from-commit-between-51aeccb-march-11-2019-and-202244-november-17-2022) +1. [Architecture](#architecture) +1. [Setup procedure](#setup-procedure) +1. [Usage](#usage) + - Start/stop + - Backups + - Restore backups + - Maintenance +1. [Troubleshooting](#troubleshooting) +1. [Redis performance](#redis-performance) + + +## Introduction + +kobo-docker is used to run a copy of the [KoboToolbox](http://www.kobotoolbox.org) survey data collection platform on a machine of your choosing. It relies on [Docker](https://docker.com) to separate the different parts of KoBo into different containers (which can be thought of as lighter-weight virtual machines) and [Docker Compose](https://docs.docker.com/compose/) to configure, run, and connect those containers. + + +## _Two PostgreSQL databases are now required_ + +Prior to release [`2.020.18`](https://github.com/kobotoolbox/kobo-docker/releases/tag/2.020.18), +[KPI](https://github.com/kobotoolbox/kpi) and +[KoBoCAT](https://github.com/kobotoolbox/kobocat) both shared a common PostgreSQL +database. They now each have their own, separate databases. +**If you are upgrading an existing single-database installation, you must follow [these instructions](https://community.kobotoolbox.org/t/upgrading-to-separate-databases-for-kpi-and-kobocat/7202)** to migrate the KPI tables to a new database and adjust your configuration appropriately. +This assumes your last upgrade was **more recent** than March 4, 2019. If not, you must [upgrade your databases](#important-notice-when-upgrading-from-commit-5c2ef02-march-4-2019-or-earlier) before proceeding. + +If you do not want to upgrade at this time, please use the [`shared-database-obsolete`](https://github.com/kobotoolbox/kobo-docker/tree/shared-database-obsolete) branch instead. + + +## Important notice when upgrading from commit [`5c2ef02` (March 4, 2019)](https://github.com/kobotoolbox/kobo-docker/commit/5c2ef0273339bee5c374830f72e52945947042a8) or earlier + +Running current releases of KoboToolbox requires you to upgrade your PostgreSQL and MongoDB databases. Please follow [these instructions](./doc/March-2019-Upgrade.md). + +If you do not, the application may not start or your data may not be visible. + +## Important notice when upgrading from commit between [`51aeccb` (March 11, 2019)](https://github.com/kobotoolbox/kobo-docker/commit/51aeccb91316d640f30e28190b936ae48d56c468) and [`2.022.44` (November 17, 2022)](https://github.com/kobotoolbox/kobo-docker/releases/tag/2.022.44) + +Running current releases of KoboToolbox requires you to upgrade your PostgreSQL and MongoDB databases. Please follow [these instructions](./doc/November-2022-Upgrade.md). + +If you do not, the application may not start or your data may not be visible. + +## Architecture + +Below is a diagram (made with [Lucidchart](https://www.lucidchart.com)) of the containers that make up a running kobo-docker system and their connections. + +![Diagram of Docker Containers](./doc/container-diagram.svg) + +### Secure your installation +This version of kobo-docker does **not** expose backend container ports, but [previous versions did](https://github.com/kobotoolbox/kobo-docker/pull/280), relying on a firewall to prevent unauthorized access. You should always verify that your database ports (by default 5432, 27017, 6379, 6380) are not accessible to the public. + +If you want to use kobo-docker with separate front-end and back-end servers, you will need to expose ports, and **you MUST use a firewall**. The firewall is required to allow only the `frontend` containers to access PostgreSQL, Redis, and MongoDB. + + +## Setup procedure + +This procedure has been simplified by using [kobo-install](https://github.com/kobotoolbox/kobo-install ""). +Please use it to install kobo-docker. + +Already have an existing installation? Please see below. + +1. **Migrating from RabbitMQ to Redis as the Celery (asynchronous task) broker** + + The easiest way is to rely on [kobo-install](https://github.com/kobotoolbox/kobo-install) to generate the correct environment files. + + If you want to change it manually, edit: + + - `kobo-env/envfiles/kpi.txt` + ```diff + - KPI_BROKER_URL=amqp://kpi:kpi@rabbit.[internal domain name]:5672/kpi + + KPI_BROKER_URL=redis://redis-main.[internal domain name]:6389/1 + ``` + - `kobo-env/envfiles/kobocat.txt` + ```diff + - KOBOCAT_BROKER_URL=amqp://kobocat: kobocat@rabbit.[internal domain name]:5672/kobocat + + KOBOCAT_BROKER_URL=redis://redis-main.[internal domain name]:6389/2 + ``` + +1. **Load balancing and redundancy** + + 1. Load balancing + kobo-docker has two different composer files. One for `frontend` and one for `backend`. + + 1. `frontend`: + - NGINX + - [KoBoCAT](https://github.com/kobotoolbox/kobocat) + - [KPI](https://github.com/kobotoolbox/kpi) + - [Enketo Express](https://github.com/enketo/enketo-express) + + 1. `backend`: + - PostgreSQL + - MongoDB + - Redis + + Docker-compose for `frontend` can be started on its own server, same thing for `backend`. Users can start as many front-end servers they want. A load balancer can spread the traffic between front-end servers. + kobo-docker uses (private) domain names between `frontend` and `backend`. + It's fully customizable in configuration files. Once again, [kobo-install](https://github.com/kobotoolbox/kobo-install) does simplify the job by creating the configuration files for you. + + 1. Redundancy + `Backend` containers not redundant yet. Only `PostgreSQL` can be configured in `Primary/Secondary` mode where `Secondary` is a real-time read-only replica. + + This is a diagram that shows how kobo-docker can be used for a load-balanced/(almost) redundant solution. + + _NB: The diagram is based on AWS infrastructure, but it's not required to host your environment there._ + + ![aws diagram](./doc/aws-diagram.svg) + +## Usage +It's recommended to create `*.override.yml` docker-compose files to customize your environment. It makes easier to update. +Samples are provided. Remove `.sample` extension and update them to match your environment. + +- `docker-compose.frontend.override.yml` +- `docker-compose.backend.primary.override.yml` +- `docker-compose.backend.secondary.override.yml` (if a PostgreSQL replica is used) + +1. **Start/start containers** + + ``` + # Start + $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml up -d + $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml up -d + + # Stop + $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml stop + $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml stop + ``` + +1. **Backups** + + Automatic, periodic backups of KoBoCAT media, MongoDB, PostgreSQL and Redis can be individually enabled by uncommenting (and optionally customizing) the `*_BACKUP_SCHEDULE` variables in your envfiles. + + - `deployments/envfiles/databases.txt` (MongoDB, PostgreSQL, Redis) + - `deployments/envfiles/kobocat.txt` (KoBoCat media) + + When enabled, timestamped backups will be placed in backups/kobocat, backups/mongo, backups/postgres and backups/redis respectively. + + If `AWS` credentials and `AWS S3` backup bucket name are provided, the backups are created directly on `S3`. + + Backups **on disk** can also be manually triggered when kobo-docker is running by executing the the following commands: + + ``` + $kobo-docker> docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml exec kobocat /srv/src/kobocat/docker/backup_media.bash + $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec mongo bash /kobo-docker-scripts/backup-to-disk.bash + $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec -e PGUSER=kobo postgres bash /kobo-docker-scripts/backup-to-disk.bash + $kobo-docker> docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml exec redis_main bash /kobo-docker-scripts/backup-to-disk.bash + ``` + +1. **Restore backups** + + Commands should be run within containers. + + - MongoDB: `mongorestore --archive= --gzip` + - PostgreSQL: `pg_restore -U kobo -d kobotoolbox -c ""` + - Redis: `gunzip && mv /data/enketo-main.rdb` + +1. **Maintenance mode** + + There is one composer file `docker-compose.maintenance.yml` can be used to put `KoboToolbox` in maintenance mode. + Like front-end or back-end containers, a `docker-compose.maintenance.yml.sample` file is provided to help you to customize your settings. + First, copy `docker-compose.maintenance.yml.sample` to `docker-compose.maintenance.yml`. + + There are 4 variables that can be customized in `docker-compose.maintenance.override.yml`: + + - `ETA` e.g. `2 hours` + - `DATE_STR` e.g. `Monday, November 26 at 02:00 GMT` + - `DATE_ISO` e.g. `20181126T02` + - `EMAIL` e.g. `support@example.com` + + NGINX container has to be stopped before launching the maintenance container. + + **Start** + + ``` + docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml stop nginx + docker-compose -f docker-compose.maintenance.yml -f docker-compose.maintenance.override.yml up -d + ``` + + **Stop** + + ``` + docker-compose -f docker-compose.maintenance.yml -f docker-compose.maintenance.override.yml down + docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml up -d nginx + ``` + +## Troubleshooting + +- ### Basic troubleshooting + You can confirm that your containers are running with `docker ps`. + To inspect the log output from: + + - the frontend containers, execute `docker-compose -f docker-compose.frontend.yml -f docker-compose.frontend.override.yml logs -f` + - the primary backend containers, execute `docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml logs -f` + - the secondary backend container, execute `docker-compose -f docker-compose.backend.secondary.yml -f docker-compose.backend.secondary.override.yml logs -f` + + For a specific container use e.g. `docker-compose -f docker-compose.backend.primary.yml -f docker-compose.backend.primary.override.yml logs -f redis_main`. + + The documentation for Docker can be found at https://docs.docker.com. + +- ### Django debugging + Developers can use [PyDev](http://www.pydev.org/)'s [remote, graphical Python debugger](http://www.pydev.org/manual_adv_remote_debugger.html) to debug Python/Django code. To enable for the `kpi` container: + + 1. Specify the mapping(s) between target Python source/library paths on the debugging machine to the locations of those files/directories inside the container by customizing and uncommenting the `KPI_PATH_FROM_ECLIPSE_TO_PYTHON_PAIRS` variable in [`envfiles/kpi.txt`](./envfiles/kpi.txt). + 1. Share the source directory of the PyDev remote debugger plugin into the container by customizing (taking care to note the actual location of the version-numbered directory) and uncommenting the relevant `volumes` entry in your `docker-compose.yml`. + 1. To ensure PyDev shows you the same version of the code as is being run in the container, share your live version of any target Python source/library files/directories into the container by customizing and uncommenting the relevant `volumes` entry in your `docker-compose.yml`. + 1. Start the PyDev remote debugger server and ensure that no firewall or other settings will prevent the containers from connecting to your debugging machine at the reported port. + 1. Breakpoints can be inserted with: `import pydevd; pydevd.settrace('${DEBUGGING_MACHINE_IP}')`. + + Remote debugging in the `kobocat` container can be accomplished in a similar manner. + + +## Redis performance +Please take a look at [https://www.techandme.se/performance-tips-for-redis-cache-server/](https://www.techandme.se/performance-tips-for-redis-cache-server/) +to get rid of Warning message when starting redis containers diff --git a/docker-compose.external-databases.yml b/docker-compose.external-databases.yml new file mode 100644 index 00000000..dd1ba750 --- /dev/null +++ b/docker-compose.external-databases.yml @@ -0,0 +1,23 @@ +services: + kpi: + extends: + file: docker-compose.yml + service: kpi + worker: + extends: + file: docker-compose.yml + service: worker + nginx: + extends: + file: docker-compose.yml + service: nginx + enketo_express: + extends: + file: docker-compose.yml + service: enketo_express + +networks: + kobo-fe-network: + driver: bridge + kobo-be-network: + driver: bridge diff --git a/docker-compose.frontend.yml b/docker-compose.frontend.yml index e9ad2c95..d92069ae 100644 --- a/docker-compose.frontend.yml +++ b/docker-compose.frontend.yml @@ -1,11 +1,11 @@ # NOTE: Generate `../kobo-env/` environment files using # https://github.com/kobotoolbox/kobo-install. You may manually customize the # files afterwards and stop using kobo-install if necessary. -version: '3' - services: kpi: &django - image: kobotoolbox/kpi:2.024.33c + extends: + file: docker-compose.yml + service: kpi init: true hostname: kpi env_file: @@ -44,19 +44,25 @@ services: worker: <<: *django - command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_worker.bash'] + command: ["bash", "/srv/src/kpi/docker/entrypoint_celery_kpi_worker.bash"] worker_low_priority: <<: *django - command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_low_priority_worker.bash'] + command: + [ + "bash", + "/srv/src/kpi/docker/entrypoint_celery_kpi_low_priority_worker.bash", + ] worker_kobocat: <<: *django - command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash'] + command: + ["bash", "/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash"] beat: <<: *django - command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash'] + command: + ["bash", "/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash"] nginx: image: nginx:1.26 @@ -69,12 +75,12 @@ services: - UWSGI_PASS_TIMEOUT=130 - TEMPLATED_VAR_REFS=$${PUBLIC_REQUEST_SCHEME} $${INTERNAL_DOMAIN_NAME} $${PUBLIC_DOMAIN_NAME} $${KOBOFORM_PUBLIC_SUBDOMAIN} $${KOBOCAT_PUBLIC_SUBDOMAIN} $${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN} volumes: - - ./.vols/static:/srv/www:ro - - ./log/nginx:/var/log/nginx - - ./.vols/kobocat_media_uploads:/media - - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro - - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh - - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts + - ./.vols/static:/srv/www:ro + - ./log/nginx:/var/log/nginx + - ./.vols/kobocat_media_uploads:/media + - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro + - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh + - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts restart: unless-stopped networks: kobo-fe-network: @@ -83,14 +89,16 @@ services: - nginx.docker.container enketo_express: - image: kobotoolbox/enketo-express-extra-widgets:6.2.2 - init: true + extends: + file: docker-compose.yml + service: enketo_express hostname: enketo_express env_file: - ../kobo-env/envfiles/domains.txt restart: unless-stopped volumes: - ../kobo-env/enketo_express/config.json:/srv/src/enketo_express/config/config.json + command: "${ENKETO_SRC_DIR}/setup/docker/start.sh" networks: kobo-fe-network: aliases: @@ -100,3 +108,5 @@ services: networks: kobo-fe-network: driver: bridge + kobo-be-network: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 33a5942e..d0e1ff6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: networks: - kobo-be-network redis: - image: redis + image: valkey/valkey networks: - kobo-be-network kpi: &django @@ -28,7 +28,7 @@ services: <<: *default-environment DJANGO_DEBUG: "true" # Remove? USE_X_FORWARDED_HOST: "true" - PYTHONUNBUFFERED: 1 # Move to Dockerfile? + PYTHONUNBUFFERED: 1 # Move to Dockerfile? DATABASE_URL: postgis://postgres:postgres@postgres:5432/postgres KC_DATABASE_URL: postgis://postgres:postgres@postgres:5432/kobocat SESSION_COOKIE_DOMAIN: .kobo.local @@ -62,11 +62,11 @@ services: ports: - 80:80 volumes: - - ./.vols/static:/srv/www:ro - - ./.vols/kobocat_media_uploads:/media - - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro - - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh - - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts + - ./.vols/static:/srv/www:ro + - ./.vols/kobocat_media_uploads:/media + - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro + - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh + - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts enketo_express: image: kobotoolbox/enketo-express-extra-widgets:6.2.2 init: true