From 90e69882df80309e74a1f84fdcddb920bd22be10 Mon Sep 17 00:00:00 2001 From: SebastienReuiller Date: Tue, 7 May 2024 11:47:04 +0200 Subject: [PATCH] =?UTF-8?q?refactor:=20Simplication=20de=20l'environnement?= =?UTF-8?q?=20Docker=20pour=20le=20d=C3=A9veloppement=20(#1169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/linting_testing.yml | 4 +- .github/workflows/review-app-creation.yml | 2 +- Makefile | 16 ++-- README.md | 37 +++------ docker-compose.yml | 23 ++---- docker/dev/django/Dockerfile | 94 +++++++---------------- docker/dev/django/entrypoint.sh | 8 +- docker/dev/postgres/psql_init.sh | 19 ----- env.docker_default.local | 29 ++++--- scripts/delete_docker.sh | 3 - start_docker.sh | 69 ----------------- 11 files changed, 66 insertions(+), 238 deletions(-) delete mode 100755 docker/dev/postgres/psql_init.sh delete mode 100755 scripts/delete_docker.sh delete mode 100755 start_docker.sh diff --git a/.github/workflows/linting_testing.yml b/.github/workflows/linting_testing.yml index 41f0f8738..c95e184c9 100644 --- a/.github/workflows/linting_testing.yml +++ b/.github/workflows/linting_testing.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [3.10.6] + python-version: [3.11.9] env: DJANGO_SETTINGS_MODULE: config.settings.test @@ -47,7 +47,7 @@ jobs: - name: Set up python uses: actions/setup-python@v4 with: - python-version: 3.10.6 + python-version: 3.11.9 #---------------------------------------------- # install & configure poetry diff --git a/.github/workflows/review-app-creation.yml b/.github/workflows/review-app-creation.yml index a0f3be5df..213a28500 100644 --- a/.github/workflows/review-app-creation.yml +++ b/.github/workflows/review-app-creation.yml @@ -19,7 +19,7 @@ env: CONFIGURATION_ADDON: ${{ secrets.CLEVER_REVIEW_APPS_CONFIGURATION_ADDON }} S3_ADDON: ${{ secrets.CLEVER_REVIEW_APPS_S3_ADDON }} BRANCH: ${{ github.head_ref }} - PYTHON_VERSION: "3.10" + PYTHON_VERSION: "3.11" jobs: diff --git a/Makefile b/Makefile index f9d3cd140..b84cc4219 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Global tasks. # ============================================================================= -PYTHON_VERSION := python3.9 +PYTHON_VERSION := python3.11 .PHONY: shell_on_django_container shell_on_django_container_as_root shell_on_postgres_container # DOCKER commands @@ -8,18 +8,15 @@ PYTHON_VERSION := python3.9 # Django shell_on_django_container: - docker exec -ti bitoubi_django /bin/bash - -shell_on_django_container_as_root: - docker exec -ti --user root bitoubi_django /bin/bash + docker compose exec -ti app /bin/bash # Postgres shell_on_postgres_container: - docker exec -ti bitoubi_postgres /bin/bash + docker compose exec -ti db /bin/bash # Itou theme update_itou_theme: scripts/upload_itou_theme.sh - docker exec bitoubi_django /bin/sh -c "./scripts/upload_itou_theme.sh" + docker compose exec app /bin/sh -c "./scripts/upload_itou_theme.sh" # After migrate populate_db: @@ -27,8 +24,7 @@ populate_db: ls -d lemarche/fixtures/django/* | xargs django-admin loaddata populate_db_container: - # docker exec -ti bitoubi_postgres bash -c "pg_restore -d marche --if-exists --clean --no-owner --no-privileges lemarche/perimeters/management/commands/data/perimeters_20220104.sql" - docker exec -ti bitoubi_django bash -c "ls -d lemarche/fixtures/django/* | xargs django-admin loaddata" + docker compose exec -ti app bash -c "ls -d lemarche/fixtures/django/* | xargs django-admin loaddata" # Deployment # ============================================================================= @@ -37,7 +33,7 @@ deploy_prod: scripts/deploy_prod.sh ./scripts/deploy_prod.sh test_container: - docker exec -ti bitoubi_django django-admin test --settings=config.settings.test $(TARGET) --noinput --failfast --parallel + docker compose exec -ti app django-admin test --settings=config.settings.test $(TARGET) --noinput --failfast --parallel test: django-admin test --settings=config.settings.test $(TARGET) --noinput --failfast --parallel diff --git a/README.md b/README.md index 66f72946d..6f7a87b14 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,10 @@ Voici un tableau explicatif de la nomenclature utilisée dans le code (par rappo ## Installation Étapes d'une installation en local à des fins de développement. -L'environnement fourni permet de fonctionner de 3 manières différentes : +L'environnement fourni permet de fonctionner de deux manières différentes : 1. Poetry + Postgres (sans Docker) -2. Dockerfile + Postgres -3. docker-compose (installe tout l'environnement nécessaire) +2. Docker + Docker Compose (installe tout l'environnement nécessaire) ### Poetry (sans Docker) @@ -70,9 +69,7 @@ $ env PYTHONPATH=./lemarche:./lemarche/c4_directory poetry run python manage.py ### Docker -L'application utilise un Dockerfile multistage, permettant de fonctionner en "Dev" et "Prod" avec le même [Dockerfile](./Dockerfile). - -Pour l'environnement de développement, un `docker-compose` est fourni (voir ci-dessous). +Pour l'environnement de développement, un ficher `docker-compose.yml` est fourni et utilisable avec le plugin [Docker Compose](https://docs.docker.com/compose/). Pour la configuration Django, vérifiez le fichier [config/settings/dev.py](./config/settings/dev.py). @@ -82,41 +79,27 @@ Pour un déploiement local **avec Docker**, dupliquez le fichier `env.docker_def > :information_source: pour accéder à l'environnemnt depuis une autre machine, pensez à définir la variable d'environnemnt `CURRENT_HOST` dans le fichier d'environnement -#### Lancement docker-compose +#### Lancement Docker Compose Après création du fichier `env.docker.local` : ```bash # Démarrage - > docker-compose up + > docker compose up # Après démarrage, le serveur est disponible sur http://localhost:8880/ # Se connecter au containeur django - > docker exec -it bitoubi_django /bin/bash + > docker compose exec -it app /bin/bash # ou > make shell_on_django_container # Re-création de l'environnement (en cas de modification) - > docker-compose down - > docker-compose build --no-cache - > docker-compose up --force-recreate + > docker compose down + > docker compose build --no-cache + > docker compose up --force-recreate # Effacement complet des images dockers - > ./scripts/delete_docker.sh -``` - -#### Lancement Dockerfile - -Le script [start_docker.sh](./start_docker.sh) permet de lancer les environnements en local, en mode **dev** ou **prod** : - -```bash - > ./start_docker.sh -h - --p|--prod run full docker (Prod config) --d|--dev run dev docker (Dev config and local mounts) - -# Pour lancer l'environnement de développement -> ./start_docker.sh --dev + > docker compose down -v ``` ## Utilisation diff --git a/docker-compose.yml b/docker-compose.yml index 90b46d749..f16cee179 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,6 @@ -version: "3.7" - services: - postgres: - container_name: bitoubi_postgres - env_file: - - env.docker.local - # The default memory setting (64M) is not enough anymore due to the size of the database we import + db: + # The default memory setting (64M) is not enough anymore due to the size of the database we import # https://stackoverflow.com/questions/56839975/docker-shm-size-dev-shm-resizing-shared-memory shm_size: 1g build: @@ -13,30 +8,24 @@ services: dockerfile: ./docker/dev/postgres/Dockerfile volumes: - postgres_data:/var/lib/postgresql/data - - ./docker/dev/postgres/psql_init.sh:/docker-entrypoint-initdb.d/postgres-init.sh - restart: always + env_file: + - env.docker.local ports: - "${POSTGRESQL_PORT:-5432}:5432" - - bitoubi: - container_name: bitoubi_django - restart: always + app: build: context: . - target: dev dockerfile: ./docker/dev/django/Dockerfile command: ./docker/dev/django/entrypoint.sh volumes: - ./lemarche:/app/lemarche - ./config:/app/config - environment: - - DEBUG=true env_file: - env.docker.local ports: - "8880:8880" depends_on: - - postgres + - db volumes: postgres_data: diff --git a/docker/dev/django/Dockerfile b/docker/dev/django/Dockerfile index 9089f7a57..90b7c91d8 100644 --- a/docker/dev/django/Dockerfile +++ b/docker/dev/django/Dockerfile @@ -1,106 +1,66 @@ -# ---------------------------------------------------- -# Base-image -# ---------------------------------------------------- -FROM python:3.10-slim-buster as common-base -# Django directions: https://blog.ploetzli.ch/2020/efficient-multi-stage-build-django-docker/ -# Pip on docker : https://pythonspeed.com/articles/multi-stage-docker-python/ -# https://blog.mikesir87.io/2018/07/leveraging-multi-stage-builds-single-dockerfile-dev-prod/ -# https://pythonspeed.com/articles/base-image-python-docker-images/ +FROM python:3.11.9-slim-bookworm # Default environment: Dev ARG ENV=dev -ENV APP_DIR="/app" - -ENV PYTHONFAULTHANDLER=1 \ +ENV APP_DIR="/app" \ + HOST=0.0.0.0 \ + PORT=8000 \ + PYTHONPATH=. \ + PYTHONFAULTHANDLER=1 \ PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ PYTHONHASHSEED=random \ PIP_NO_CACHE_DIR=off \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 \ POETRY_VERSION=1.5.1 \ + POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_VIRTUALENVS_CREATE=1 \ + POETRY_CACHE_DIR=/opt/.cache \ NODE_VERSION=15 -ENV HOST=0.0.0.0 \ - PORT=8000 +# Add new user to run the whole thing as non-root. +RUN set -ex \ + && addgroup app \ + && adduser --ingroup app --home ${APP_DIR} --disabled-password app; -WORKDIR /app +WORKDIR $APP_DIR COPY install-packages.sh . RUN ./install-packages.sh -# ---------------------------------------------------- -# Install dependencies -# ---------------------------------------------------- -FROM common-base AS dependencies -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" - -RUN pip install poetry==$POETRY_VERSION -RUN pip install -I uwsgi - -# apt-get install build-essential -y -COPY poetry.lock pyproject.toml /app/ +ENV VIRTUAL_ENV=$APP_DIR/.venv \ + PATH="$APP_DIR/.venv/bin:$PATH" -RUN poetry config virtualenvs.create false && \ - poetry config virtualenvs.path /opt/venv && \ - poetry install $(test $ENV == "prod" && echo "--no-dev") --no-interaction --no-ansi +# Install Python dependencies +COPY poetry.lock pyproject.toml $APP_DIR +RUN pip install poetry==$POETRY_VERSION && pip install -I uwsgi +RUN poetry install --no-interaction --no-ansi --no-root -# ---------------------------------------------------- -# Build project -# ---------------------------------------------------- -FROM common-base AS app-run -COPY --from=dependencies /opt/venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" \ - VIRTUALENV="/opt/venv" \ - PYTHONPATH="$PYTHONPATH:/app/lemarche:/app/config" COPY ./lemarche ./lemarche COPY ./config ./config COPY ./manage.py ./manage.py -COPY ./pyproject.toml ./pyproject.toml COPY ./docker ./docker COPY ./scripts ./scripts -# ---------------------------------------------------- -# Run Dev -# ---------------------------------------------------- -FROM app-run AS dev -ENV DJANGO_SETTINGS_MODULE="config.settings.dev" \ - ENV="dev" \ - DEBUG="True" - RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/issue && cat /etc/motd' \ >> /etc/bash.bashrc \ ; echo "\ ===================================================================\n\ - = Bitoubi Dev Docker container =\n\ + = Le Marché de l'inclusion Dev Docker container =\n\ ===================================================================\n\ \n\ - (c) plateforme de l'Inclusion\n\ + (c) Plateforme de l'inclusion\n\ \n\ Source directory is /app \n\ - + \ Run App with :\n\ > python ./manage.py runserver \$HOST:\$PORT\n\ \n\ "\ > /etc/motd -CMD ["bash"] - -# ---------------------------------------------------- -# Run Dev -# ---------------------------------------------------- -FROM app-run AS prod -ENV DJANGO_SETTINGS_MODULE="config.settings.prod" \ - ENV="prod" \ - DEBUG="False" +USER app -CMD [".docker/dev/django/entrypoint.sh"] - -# # For some _real_ performance, at cost of ease of use: -# FROM python:3.9-alpine as prod -# COPY --from=dependencies /opt/venv /opt/venv -# ENV PATH="/opt/venv/bin:$PATH" -# COPY . . -# RUN apk add python3-dev build-base linux-headers pcre-dev -# RUN pip install uwsgi +CMD ["bash"] diff --git a/docker/dev/django/entrypoint.sh b/docker/dev/django/entrypoint.sh index 3c4c2ba94..beb323b62 100755 --- a/docker/dev/django/entrypoint.sh +++ b/docker/dev/django/entrypoint.sh @@ -1,9 +1,8 @@ #!/usr/bin/env bash -# Getting static files for Admin panel hosting! set -e echo "" echo "================================================" -echo "= Bitoubi Api Dev docker-compose =" +echo "= Le marché de l'inclusion Dev Docker Compose =" echo "================================================" echo "" @@ -12,11 +11,6 @@ while ! pg_isready -h $POSTGRESQL_ADDON_HOST -p $POSTGRESQL_ADDON_PORT; do sleep 1 done - -# ./manage.py collectstatic --noinput -# ./manage.py compress --force - ./manage.py migrate ./manage.py runserver 0.0.0.0:8880 -# gunicorn config.wsgi:application -w 2 -b :8880 --reload diff --git a/docker/dev/postgres/psql_init.sh b/docker/dev/postgres/psql_init.sh deleted file mode 100755 index 97a9433cd..000000000 --- a/docker/dev/postgres/psql_init.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -# Initialization script. -# https://hub.docker.com/_/postgres/#initialization-scripts - -export BASE_DIR=$(dirname "$BASH_SOURCE") - -# The PostgreSQL user should be able to create extensions. -# Only the PostgreSQL superuser role provides that permission. -psql -v ON_ERROR_STOP=0 --username "$POSTGRES_USER" <<-EOSQL - - \c postgres; - - CREATE USER $POSTGRESQL_ADDON_USER WITH ENCRYPTED PASSWORD '$POSTGRESQL_ADDON_PASSWORD'; - CREATE DATABASE $POSTGRESQL_ADDON_DB OWNER $POSTGRESQL_ADDON_USER; - ALTER USER $POSTGRESQL_ADDON_USER CREATEDB SUPERUSER; - -EOSQL diff --git a/env.docker_default.local b/env.docker_default.local index eb445c9c2..1a89c6220 100755 --- a/env.docker_default.local +++ b/env.docker_default.local @@ -1,6 +1,5 @@ -# Config for BITOUBI_POSTGRES +# Config for postgres container # ########################### -PYTHONPATH=. # Avoid a log error when starting the itou_postgres container: # > Role "root" does not exist. @@ -9,27 +8,28 @@ PYTHONPATH=. # https://www.postgresql.org/docs/current/libpq-envars.html PGUSER=postgres -# PostgreSQL +# PostgreSQL - for db/user creation POSTGRES_DB=lemarche -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres # ggignore -POSTGRES_HOST=postgres +POSTGRES_USER=lemarche_user +POSTGRES_PASSWORD=lemarche_password # ggignore +POSTGRES_HOST=db -# Config for BITOUBI_POSTGRES +# Config for app container with the same variable names as the production # ########################### -POSTGRESQL_ADDON_HOST=postgres +# PostgreSQL +POSTGRESQL_ADDON_HOST=db POSTGRESQL_ADDON_DB=lemarche POSTGRESQL_ADDON_USER=lemarche_user -POSTGRESQL_ADDON_PASSWORD=postgres ##ggignore +POSTGRESQL_ADDON_PASSWORD=lemarche_password ##ggignore POSTGRESQL_ADDON_PORT=5432 -# Config for BITOUBI_DJANGO -# ######################## # Django +DJANGO_SETTINGS_MODULE=config.settings.dev +ENV=dev +DEBUG=True SECRET_KEY=[YOUR_SECRET] TRACKER_HOST=localhost - # Mailjet / Anymail # no need in local, emails will be printed to the console (console.EmailBackend) # MAILJET_API_KEY= @@ -43,21 +43,18 @@ MAILJET_MASTER_API_SECRET= MAILJET_NEWSLETTER_CONTACT_LIST_BUYER_ID= # MTCAPTCHA -# ######################## MTCAPTCHA_PRIVATE_KEY= MTCAPTCHA_PUBLIC_KEY= # OPENAI -# ######################## OPENAI_ORG= OPENAI_API_BASE= OPENAI_API_BASE= OPENAI_MODEL= # ELASTICSEARCH -# ######################## ELASTICSEARCH_HOST= ELASTICSEARCH_USERNAME= ELASTICSEARCH_PASSWORD= ELASTICSEARCH_INDEX_SIAES= -ELASTICSEARCH_MIN_SCORE= +ELASTICSEARCH_MIN_SCORE=0.9 diff --git a/scripts/delete_docker.sh b/scripts/delete_docker.sh deleted file mode 100755 index 3fed26198..000000000 --- a/scripts/delete_docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker-compose down -docker rmi itou-marche_bitoubi itou-marche_postgres -docker volume rm itou-marche_postgres_data \ No newline at end of file diff --git a/start_docker.sh b/start_docker.sh deleted file mode 100755 index 4bc0705e2..000000000 --- a/start_docker.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Docker environment settings : https://vsupalov.com/docker-arg-env-variable-guide/ -docker_dev() { - docker build \ - --target dev \ - -t "c4_api" -f ./Dockerfile . \ - --build-arg ENV=DEV \ - && docker run --rm -it \ - -p 8000:8000 \ - --env-file=env.docker_dev.local \ - -e DEBUG="true" \ - --name c4_api \ - -v `pwd`/lemarche:/app/lemarche \ - -v `pwd`/config:/app/config \ - -v `pwd`/static:/app/static \ - c4_api -} - -docker_compose() { - docker-compose up --force-recreate -} - -docker_prod() { - docker build \ - --target prod \ - -t "c4_api" -f ./Dockerfile . \ - --build-arg ENV=PROD \ - && docker run --rm -it \ - -p 8000:8000 \ - --env-file=env.docker_prod.local \ - -e DEBUG="false" \ - --name c4_api \ - c4_api -} - -usage() { - echo " --p|--prod run full docker (Prod config) --d|--dev run dev docker (Dev config and local mounts) -" -} - -while [ "$1" != "" ]; do - case $1 in - -p | --prod ) - echo "Running full docker" - docker_prod - exit - ;; - -d | --dev ) - echo "Running dev docker" - docker_dev - exit - ;; - -c | --compose ) - echo "Running dev docker-compose" - docker_compose - exit - ;; - * ) - usage - exit 1 - ;; - esac - shift -done -usage -exit 1 -