From 848c180f9459604f65a476d0a8f9776270306f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Capil=C3=A9?= Date: Wed, 27 Nov 2024 15:13:10 -0300 Subject: [PATCH] FIX: Slow startup and performance on NFS or slow volume mounts #188 --- README.md | 11 +++++ core/Dockerfile | 1 + core/files/entrypoint_nginx.sh | 47 +++++++++++++------ .../misp-docker/initialisation.envars.json | 3 ++ docker-compose.yml | 1 + template.env | 2 + 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ea319da..74879fd 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,17 @@ If it is a setting controlled by an environment variable which is meant to overr - `./gnupg`: `/var/www/MISP/.gnupg/` - If you need to automatically run additional steps each time the container starts, create a new file `files/customize_misp.sh`, and replace the variable `${CUSTOM_PATH}` inside `docker-compose.yml` with its parent path. +#### Using slow disks as volume mounts + +Using a slow disk as the mounted volume or a volume with high latency like NFS, EFS or S3 might significantly increase the startup time and downgrade the performance of the service. To address this we will mount the bare minimum that needs to be persisted. + +- Remove the `/var/www/MISP/app/files/` volume mount. +- Add the following volume mounts instead: + - `./img/`: `/var/www/MISP/app/files/img` + - `./terms`: `/var/www/MISP/app/files/terms` + - `./attachments`: `/var/www/MISP/app/attachments` +- Set the environment variable `ATTACHMENTS_DIR` to the above folder location (it is important that it doesn't replace the `/var/www/MISP/app/files/` folder). + ## Installing custom root CA certificates Custom root CA certificates can be mounted under `/usr/local/share/ca-certificates` and will be installed during the `misp-core` container start. diff --git a/core/Dockerfile b/core/Dockerfile index d090a9e..57ec171 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -253,6 +253,7 @@ EOF RUN <<-EOF cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist + echo "${CORE_COMMIT:-${CORE_TAG}}" > /var/www/MISP/app/files/VERSION cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; diff --git a/core/files/entrypoint_nginx.sh b/core/files/entrypoint_nginx.sh index 76502b2..5c2659a 100755 --- a/core/files/entrypoint_nginx.sh +++ b/core/files/entrypoint_nginx.sh @@ -138,6 +138,15 @@ EOT } update_misp_data_files(){ + # If $MISP_APP_FILES_PATH was not changed since the build, skip file updates there + FILES_VERSION= + MISP_APP_FILES_PATH=/var/www/MISP/app/files + if [ -f ${MISP_APP_FILES_PATH}/VERSION ]; then + FILES_VERSION=$(cat ${MISP_APP_FILES_PATH}/VERSION) + fi + if [ $FILES_VERSION = ${CORE_COMMIT:-${CORE_TAG}} ]; then + return 0; + fi for DIR in $(ls /var/www/MISP/app/files.dist); do if [ "$DIR" = "certs" ] || [ "$DIR" = "img" ] || [ "$DIR" == "taxonomies" ] ; then echo "... rsync -azh \"/var/www/MISP/app/files.dist/$DIR\" \"/var/www/MISP/app/files/\"" @@ -150,21 +159,29 @@ update_misp_data_files(){ } enforce_misp_data_permissions(){ - echo "... chown -R www-data:www-data /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + - # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory - echo "... chmod -R 0550 files /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 550 -type f -exec chmod 0550 {} + - # Directories are also writable, because there seems to be a requirement to add new files every once in a while - echo "... chmod -R 0770 directories /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 770 -type d -exec chmod 0770 {} + - # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit) - echo "... chmod -R u+w,g+w /var/www/MISP/app/tmp" && chmod -R u+w,g+w /var/www/MISP/app/tmp - - echo "... chown -R www-data:www-data /var/www/MISP/app/files" && find /var/www/MISP/app/files \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + - # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory - echo "... chmod -R 0550 files /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 550 -type f -exec chmod 0550 {} + - # Directories are also writable, because there seems to be a requirement to add new files every once in a while - echo "... chmod -R 0770 directories /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 770 -type d -exec chmod 0770 {} + - # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit) - echo "... chmod -R u+w,g+w /var/www/MISP/app/files" && chmod -R u+w,g+w /var/www/MISP/app/files + # If $MISP_APP_FILES_PATH was not changed since the build, skip file updates there + FILES_VERSION= + MISP_APP_FILES_PATH=/var/www/MISP/app/files + if [ -f ${MISP_APP_FILES_PATH}/VERSION ]; then + FILES_VERSION=$(cat ${MISP_APP_FILES_PATH}/VERSION) + fi + if [ $FILES_VERSION != ${CORE_COMMIT:-${CORE_TAG}} ]; then + echo "... chown -R www-data:www-data /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + + # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory + echo "... chmod -R 0550 files /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 550 -type f -exec chmod 0550 {} + + # Directories are also writable, because there seems to be a requirement to add new files every once in a while + echo "... chmod -R 0770 directories /var/www/MISP/app/tmp" && find /var/www/MISP/app/tmp -not -perm 770 -type d -exec chmod 0770 {} + + # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit) + echo "... chmod -R u+w,g+w /var/www/MISP/app/tmp" && chmod -R u+w,g+w /var/www/MISP/app/tmp + + echo "... chown -R www-data:www-data /var/www/MISP/app/files" && find /var/www/MISP/app/files \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + + # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory + echo "... chmod -R 0550 files /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 550 -type f -exec chmod 0550 {} + + # Directories are also writable, because there seems to be a requirement to add new files every once in a while + echo "... chmod -R 0770 directories /var/www/MISP/app/files" && find /var/www/MISP/app/files -not -perm 770 -type d -exec chmod 0770 {} + + # We make 'files' and 'tmp' (logs) directories and files user and group writable (we removed the SGID bit) + echo "... chmod -R u+w,g+w /var/www/MISP/app/files" && chmod -R u+w,g+w /var/www/MISP/app/files + fi echo "... chown -R www-data:www-data /var/www/MISP/app/Config" && find /var/www/MISP/app/Config \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory diff --git a/core/files/etc/misp-docker/initialisation.envars.json b/core/files/etc/misp-docker/initialisation.envars.json index b635093..5bc5b5e 100644 --- a/core/files/etc/misp-docker/initialisation.envars.json +++ b/core/files/etc/misp-docker/initialisation.envars.json @@ -12,6 +12,9 @@ "MISP.contact": { "default_value": "${SETTING_CONTACT}" }, + "MISP.attachments_dir": { + "default_value": "${ATTACHMENTS_DIR}" + }, "Security.encryption_key": { "default_value": "${ENCRYPTION_KEY}", "command_args": "-f" diff --git a/docker-compose.yml b/docker-compose.yml index c94ab52..e79d108 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,6 +114,7 @@ services: - "ADMIN_ORG=${ADMIN_ORG}" - "ADMIN_ORG_UUID=${ADMIN_ORG_UUID}" - "GPG_PASSPHRASE=${GPG_PASSPHRASE}" + - "ATTACHMENTS_DIR=${ATTACHMENTS_DIR:-/var/www/MISP/app/files}" # OIDC authentication settings - "OIDC_ENABLE=${OIDC_ENABLE}" - "OIDC_PROVIDER_URL=${OIDC_PROVIDER_URL}" diff --git a/template.env b/template.env index 77bc823..a9c6048 100644 --- a/template.env +++ b/template.env @@ -53,6 +53,8 @@ ENABLE_DB_SETTINGS= ENCRYPTION_KEY= # enable background updates. defaults to false ENABLE_BACKGROUND_UPDATES= +# use a different attachments_dir. configure this folder to use persistent storage +ATTACHMENTS_DIR=/var/www/MISP/app/files # defines the FQDN of the mail sub-system (defaults to 'mail') # SMTP_FQDN=