diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5f97c55..50f029c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt -r dev-requirements.txt + pip install -r requirements.txt -r requirements.dev.txt - name: Linting checks run: | # stop the build if there are any Python syntax errors diff --git a/.gitignore b/.gitignore index 82989a87..12c16764 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,8 @@ *.rdb environments/ -!environments/docker.dev +!environments/dev-docker-postgres.env +!environments/dev-docker-mysql.env .coverage tags diff --git a/Makefile b/Makefile index b52e806f..e64ec2ff 100644 --- a/Makefile +++ b/Makefile @@ -7,19 +7,31 @@ help: ## Show this help menu @echo "Usage: make [TARGET ...]" @echo "" @grep --no-filename -E '^[a-zA-Z_%-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf "%-10s %s\n", $$1, $$2}' + awk 'BEGIN {FS = ":.*?## "}; {printf "%-25s %s\n", $$1, $$2}' .PHONY: dc-start dc-start: dc-stop ## Start dev docker server - @docker compose -f docker-compose.yml up --build --scale adminer=0 -d; + @docker compose -f docker-compose-postgres.yml up --build --scale adminer=0 -d; .PHONY: dc-start-adminer dc-start-adminer: dc-stop ## Start dev docker server (with adminer) - @docker compose -f docker-compose.yml up --build -d; + @docker compose -f docker-compose-postgres.yml up --build -d; .PHONY: dc-stop dc-stop: ## Stop dev docker server - @docker compose -f docker-compose.yml stop; + @docker compose -f docker-compose-postgres.yml stop; + +.PHONY: dc-start-mysql +dc-start-mysql: dc-stop ## Start dev docker server using MySQL + @docker compose -f docker-compose-mysql.yml up --build --scale adminer=0 -d; + +.PHONY: dc-start-adminer-mysql +dc-start-adminer-mysql: dc-stop ## Start dev docker server using MySQL (with adminer) + @docker compose -f docker-compose-mysql.yml up --build -d; + +.PHONY: dc-stop-mysql +dc-stop-mysql: ## Stop dev docker server using MySQL + @docker compose -f docker-compose-mysql.yml stop; VENV = venv VENV_PYTHON = $(VENV)/bin/python @@ -36,7 +48,7 @@ venv: $(VENV_PYTHON) ## Create a Python virtual environment .PHONY: deps deps: ## Install Python requirements in virtual environment $(PYTHON) -m pip install --upgrade pip - $(PYTHON) -m pip install --no-cache-dir -r requirements.txt -r dev-requirements.txt + $(PYTHON) -m pip install --no-cache-dir -r requirements.txt -r requirements.dev.txt .PHONY: checks checks: tests ruff mypy bandit ## Run all checks (unit tests, ruff, mypy, bandit) @@ -80,5 +92,5 @@ logs: ## Follow Flask logs docker logs shhh-app-1 -f -n 10 .PHONY: db-logs -db-logs: ## Follow Postgre logs +db-logs: ## Follow database logs docker logs shhh-db-1 -f -n 10 diff --git a/README.md b/README.md index 7c442e72..debffb5d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ You can find in this repo everything you need to host the app yourself. Or you can **one-click deploy to Heroku** using the below button. It will generate a fully configured private instance of Shhh immediately (using your own server running Flask behind Gunicorn and Nginx, -and your own Postgres database). +and your own PostgreSQL database). [![Deploy][heroku-shield]][heroku] (see [here](#initiate-the-database-tables) to initiate the db tables after deploying on Heroku) @@ -83,9 +83,16 @@ Once the container image has finished building and has started, you can access: * Shhh at -* Adminer at (if launched with `dc-start-adminer`) +* Adminer at (if launched with `dc-start-adminer`) -_You can find the development database credentials from the env file at [/environments/docker.dev](https://github.com/smallwat3r/shhh/blob/master/environments/docker.dev)._ +_You can find the development database credentials from the env file at [/environments/dev-docker-postgres.env](https://github.com/smallwat3r/shhh/blob/master/environments/dev-docker-postgres.env)._ + +You have also the option to use `MySQL` instead of `PostgreSQL`, using these commands: +```sh +make dc-start-mysql # to start the app +make dc-start-adminer-mysql # to start the app with adminer (SQL editor) +make dc-stop-mysql # to stop the app +``` #### Initiate the database tables @@ -158,10 +165,24 @@ Bellow is the list of environment variables used by Shhh. #### Mandatory * `FLASK_ENV`: the environment config to load (`testing`, `dev-local`, `dev-docker`, `heroku`, `production`). -* `POSTGRES_HOST`: Postgresql hostname +* `DB_HOST`: Database hostname +* `DB_USER`: Database username +* `DB_PASSWORD`: Database password +* `DB_NAME`: Database name +* `DB_ENGINE`: Database engine to use (ex: `postgresql+psycopg2`, `mysql+pymysql`) + +Depending if you can use PostgreSQL or MySQL you might also need to set (these need to match the values +you've specified as `DB_NAME`, `DB_PASSWORD` and `DB_NAME` above): + * `POSTGRES_USER`: Postgresql username * `POSTGRES_PASSWORD`: Postgresql password -* `POSTGRES_DB`: Database name +* `POSTGRES_DB`: Postgresql database name + +or + +* `MYSQLUSER`: MySQL username +* `MYSQL_PASSWORD`: MySQL password +* `MYSQL_DATABASE`: MySQL database name #### Optional * `SHHH_HOST`: This variable can be used to specify a custom hostname to use as the @@ -175,10 +196,6 @@ asleep (for instance this happens often on Heroku free plans). The default retry * `SHHH_DB_LIVENESS_SLEEP_INTERVAL`: This variable manages the interval in seconds between the database liveness retries. The default value is 1 second. -## Acknowledgements - -Special thanks: [@AustinTSchaffer](https://github.com/AustinTSchaffer), [@kleinfelter](https://github.com/kleinfelter) - ## License See [LICENSE](https://github.com/smallwat3r/shhh/blob/master/LICENSE) file. diff --git a/docker-compose-mysql.yml b/docker-compose-mysql.yml new file mode 100644 index 00000000..16fd39ef --- /dev/null +++ b/docker-compose-mysql.yml @@ -0,0 +1,29 @@ +version: '3.2' +services: + db: + image: mysql:8.2 + env_file: + - ./environments/dev-docker-mysql.env + ports: + - 3306:3306 + app: + build: + context: . + dockerfile: dockerfiles/alpine.dev.mysql.Dockerfile + image: shhh + depends_on: + - db + env_file: + - ./environments/dev-docker-mysql.env + ports: + - 8081:8081 + volumes: + - .:/opt/shhh:delegated + adminer: + image: adminer + depends_on: + - db + ports: + - 8082:8080 +volumes: + mysql: diff --git a/docker-compose-postgres.yml b/docker-compose-postgres.yml new file mode 100644 index 00000000..98a763e6 --- /dev/null +++ b/docker-compose-postgres.yml @@ -0,0 +1,29 @@ +version: '3.2' +services: + db: + image: postgres:15.4-alpine3.18 + env_file: + - ./environments/dev-docker-postgres.env + ports: + - 5432:5432 + app: + build: + context: . + dockerfile: dockerfiles/alpine.dev.Dockerfile + image: shhh + depends_on: + - db + env_file: + - ./environments/dev-docker-postgres.env + ports: + - 8081:8081 + volumes: + - .:/opt/shhh:delegated + adminer: + image: adminer + depends_on: + - db + ports: + - 8082:8080 +volumes: + postgres: diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 4206e714..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Describes the docker-compose configuration for development environments. -# These settings are not secure and are not intended for production use. -# Please override this configuration when configuring a production instance. -# This config launch Flask (gunicorn) / Postgres / Adminer - -version: '3.2' - -services: - db: - image: postgres:15.4-alpine3.18 - env_file: - - ./environments/docker.dev - volumes: - - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql - ports: - - 5432:5432 - - app: - build: - context: . - dockerfile: alpine.dev.Dockerfile - image: shhh - depends_on: - - db - env_file: - - ./environments/docker.dev - ports: - - 8081:8081 - volumes: - - .:/opt/shhh:delegated - - adminer: - image: adminer - depends_on: - - db - ports: - - 8082:8080 - -volumes: - postgres: diff --git a/alpine.dev.Dockerfile b/dockerfiles/alpine.dev.Dockerfile similarity index 95% rename from alpine.dev.Dockerfile rename to dockerfiles/alpine.dev.Dockerfile index da30c707..41b540d6 100644 --- a/alpine.dev.Dockerfile +++ b/dockerfiles/alpine.dev.Dockerfile @@ -29,13 +29,13 @@ ENV TZ UTC WORKDIR /opt/shhh -ENV GROUP=app USER=shhh UID=12345 GID=23456 +ARG GROUP=app USER=shhh UID=1001 GID=1001 RUN addgroup --gid "$GID" "$GROUP" \ && adduser --uid "$UID" --disabled-password --gecos "" \ --ingroup "$GROUP" "$USER" -USER "$USER" +USER $USER ENV PATH="/home/$USER/.local/bin:${PATH}" ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 diff --git a/dockerfiles/alpine.dev.mysql.Dockerfile b/dockerfiles/alpine.dev.mysql.Dockerfile new file mode 100644 index 00000000..5f237d0d --- /dev/null +++ b/dockerfiles/alpine.dev.mysql.Dockerfile @@ -0,0 +1,51 @@ +# This dockerfile runs the application with the bare Flask +# server. As it's for development only purposes. +# +# When using Gunicorn in a more prod-like config, multiple +# workers would require to use the --preload option, else +# the scheduler would spawn multiple scheduler instances. +# +# Note it would not be comptatible with Gunicorn --reload +# flag, which is useful to reload the app on change, for +# development purposes. +# +# Example: CMD gunicorn -b :8081 -w 3 wsgi:app --preload +# +# To use Gunicorn, please use: alpine.gunicorn.Dockerfile + +FROM python:3.12-alpine3.18 + +RUN apk update \ + && apk add --no-cache \ + gcc \ + g++ \ + libffi-dev \ + musl-dev \ + mariadb-dev \ + yarn \ + && python -m pip install --upgrade pip + +ENV TZ UTC + +WORKDIR /opt/shhh + +ARG GROUP=app USER=shhh UID=1001 GID=1001 + +RUN addgroup --gid "$GID" "$GROUP" \ + && adduser --uid "$UID" --disabled-password --gecos "" \ + --ingroup "$GROUP" "$USER" + +USER $USER +ENV PATH="/home/$USER/.local/bin:${PATH}" + +ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 + +COPY requirements.mysql.txt . +RUN pip install --no-cache-dir --no-warn-script-location \ + --user -r requirements.mysql.txt + +COPY --chown=$USER:$GROUP . . + +RUN yarn install --modules-folder=shhh/static/vendor + +CMD flask run --host=0.0.0.0 --port 8081 --reload diff --git a/alpine.gunicorn.Dockerfile b/dockerfiles/alpine.gunicorn.Dockerfile similarity index 92% rename from alpine.gunicorn.Dockerfile rename to dockerfiles/alpine.gunicorn.Dockerfile index 1087b73b..0190a646 100644 --- a/alpine.gunicorn.Dockerfile +++ b/dockerfiles/alpine.gunicorn.Dockerfile @@ -14,13 +14,13 @@ ENV TZ UTC WORKDIR /opt/shhh -ENV GROUP=app USER=shhh UID=12345 GID=23456 +ARG GROUP=app USER=shhh UID=1001 GID=1001 RUN addgroup --gid "$GID" "$GROUP" \ && adduser --uid "$UID" --disabled-password --gecos "" \ --ingroup "$GROUP" "$USER" -USER "$USER" +USER $USER ENV PATH="/home/$USER/.local/bin:${PATH}" ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 diff --git a/environments/docker.dev b/environments/docker.dev deleted file mode 100644 index 5ede1d3e..00000000 --- a/environments/docker.dev +++ /dev/null @@ -1,35 +0,0 @@ -# dev-docker env -# env file for local development purposes only -# please do not use this env file in production - -# [MANDATORY] - -FLASK_ENV=dev-docker -FLASK_DEBUG=1 - -POSTGRES_HOST=db -POSTGRES_USER=shhh -POSTGRES_PASSWORD=dummypassword -POSTGRES_PORT=5432 -POSTGRES_DB=shhh - -PGDATA=/data/postgres - -# [OPTIONAL] - -# This variable can be used to specify a custom hostname to use as the -# domain URL when Shhh creates a secret (ex: https://). If not -# set, the hostname defaults to request.url_root, which should be fine in -# most cases. -SHHH_HOST= - -# Default max secret length -SHHH_SECRET_MAX_LENGTH= - -# Number of tries to reach the database before performing a read or write operation. It -# could happens that the database is not reachable or is asleep (for instance this happens -# often on Heroku free plans). The default retry number is 5. -SHHH_DB_LIVENESS_RETRY_COUNT= - -# Sleep interval in seconds between database liveness retries. The default value is 1 second. -SHHH_DB_LIVENESS_SLEEP_INTERVAL= diff --git a/postgres/init.sql b/postgres/init.sql deleted file mode 100644 index 5f1c6c15..00000000 --- a/postgres/init.sql +++ /dev/null @@ -1,2 +0,0 @@ -SELECT 'CREATE DATABASE shhh' -WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'shhh')\gexec diff --git a/dev-requirements.txt b/requirements.dev.txt similarity index 100% rename from dev-requirements.txt rename to requirements.dev.txt diff --git a/requirements.mysql.txt b/requirements.mysql.txt new file mode 100644 index 00000000..c3edcc96 --- /dev/null +++ b/requirements.mysql.txt @@ -0,0 +1,34 @@ +APScheduler==3.10.4 +blinker==1.7.0 +certifi==2023.7.22 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cryptography==41.0.5 +cssmin==0.2.0 +Flask==3.0.0 +Flask-APScheduler==1.13.1 +Flask-Assets==2.1.0 +Flask-SQLAlchemy==3.1.1 +gunicorn==21.2.0 +htmlmin==0.1.12 +idna==3.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +jsmin==3.0.1 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +packaging==23.2 +pycparser==2.21 +PyMySQL==1.1.0 +python-dateutil==2.8.2 +pytz==2023.3.post1 +requests==2.31.0 +six==1.16.0 +SQLAlchemy==2.0.23 +typing_extensions==4.8.0 +tzlocal==5.2 +urllib3==2.1.0 +webargs==8.3.0 +webassets==2.0 +Werkzeug==3.0.1 diff --git a/shhh/config.py b/shhh/config.py index 1ee6464e..6ab2fdd5 100644 --- a/shhh/config.py +++ b/shhh/config.py @@ -9,19 +9,19 @@ class DefaultConfig: DEBUG = True - # Postgres connection. - POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") - POSTGRES_USER = os.environ.get("POSTGRES_USER") - POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD") - POSTGRES_PORT = os.environ.get("POSTGRES_PORT", 5432) - POSTGRES_DB = os.environ.get("POSTGRES_DB", "shhh") + DB_HOST = os.environ.get("DB_HOST", "localhost") + DB_USER = os.environ.get("DB_USER") + DB_PASSWORD = os.environ.get("DB_PASSWORD") + DB_PORT = os.environ.get("DB_PORT", 5432) + DB_NAME = os.environ.get("DB_NAME", "shhh") + DB_ENGINE = os.environ.get("DB_ENGINE", "postgresql+psycopg2") # SqlAlchemy SQLALCHEMY_ECHO = False SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_DATABASE_URI = ( - f"postgresql+psycopg2://{POSTGRES_USER}:{POSTGRES_PASSWORD}" - f"@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}") + f"{DB_ENGINE}://{DB_USER}:{DB_PASSWORD}" + f"@{DB_HOST}:{DB_PORT}/{DB_NAME}") # # Shhh optional custom configurations @@ -78,7 +78,7 @@ class TestConfig(DefaultConfig): DEBUG = False TESTING = True - SQLALCHEMY_DATABASE_URI = "sqlite://" + SQLALCHEMY_DATABASE_URI = "sqlite://" # in memory SHHH_HOST = "http://test.test" SHHH_SECRET_MAX_LENGTH = 20 @@ -100,7 +100,7 @@ class ProductionConfig(DefaultConfig): class HerokuConfig(ProductionConfig): - """Heroku configuration (heroku).""" + """Heroku configuration (heroku). Only support PostgreSQL.""" # SQLAlchemy 1.4 removed the deprecated postgres dialect name, the name # postgresql must be used instead. This URL is automatically set on