Merge pull request #308 from uselagoon/mysql_images

Makefile

@@ -237,7 +237,9 @@ versioned-images := 		php-8.1-fpm \
 							ruby-3.1 \
 							ruby-3.2 \
 							ruby-3.3 \
-							opensearch-2
+							opensearch-2 \
+							mysql-8.0 \
+							mysql-8.4
 
 # default-versioned-images are images that formerly had no versioning, and are made backwards-compatible.
 # the below versions are the ones that map to the unversioned namespace
@@ -319,6 +321,7 @@ build/mariadb-10.11-drupal: build/mariadb-10.11
 build/ruby-3.1 build/ruby-3.2 build/ruby-3.3: build/commons
 build/opensearch-2: build/commons
 build/mongo-4: build/commons 
+build/mysql-8.0 build/mysql-8.4: build/commons
 
 #######
 ####### Building Images

helpers/TESTING_service_images_dockercompose.md

@@ -26,6 +26,8 @@ docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp:/
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-5:3306 -timeout 1m
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-6:3306 -timeout 1m
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mariadb-10-11:3306 -timeout 1m
+docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mysql-8-0:3306 -timeout 1m
+docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://mysql-8-4:3306 -timeout 1m
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-11:5432 -timeout 1m
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-12:5432 -timeout 1m
 docker run --rm --net all-images_default jwilder/dockerize dockerize -wait tcp://postgres-13:5432 -timeout 1m
@@ -53,6 +55,8 @@ docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-5
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-6
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mariadb-10-11
+docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mysql-8-0
+docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mysql-8-4
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep mongo-4
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep postgres-11
 docker ps --filter label=com.docker.compose.project=all-images | grep Up | grep postgres-12
@@ -222,6 +226,32 @@ docker compose exec -T mongo-4 sh -c "mongo < /tmp/dbstats.sql" | grep "db" | gr
 docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mongo?service=mongo-4" | grep "SERVICE_HOST="
 docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mongo?service=mongo-4" | grep "LAGOON_TEST_VAR=all"
 
+# mysql-8-0 should be version 8.0 client
+docker compose exec -T mysql-8-0  sh -c "mysql -V" | grep "8.0"
+
+# mysql-8-0  should be version 8.0 server
+docker compose exec -T mysql-8-0  sh -c "mysql -e 'SHOW variables;'" | grep "version" | grep "8.0"
+
+# mysql-8-0  should use default credentials
+docker compose exec -T mysql-8-0  sh -c "mysql -D lagoon -u lagoon --password=lagoon -e 'SHOW databases;'" | grep lagoon
+
+# mysql-8-0  should be able to read/write data
+docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-0" | grep "SERVICE_HOST=8.0"
+docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-0" | grep "LAGOON_TEST_VAR=all-images"
+
+# mysql-8-4 should be version 8.4 client
+docker compose exec -T mysql-8-4  sh -c "mysql -V" | grep "8.4"
+
+# mysql-8-4  should be version 8.4 server
+docker compose exec -T mysql-8-4  sh -c "mysql -e 'SHOW variables;'" | grep "version" | grep "8.4"
+
+# mysql-8-4  should use default credentials
+docker compose exec -T mysql-8-4  sh -c "mysql -D lagoon -u lagoon --password=lagoon -e 'SHOW databases;'" | grep lagoon
+
+# mysql-8-4  should be able to read/write data
+docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-4" | grep "SERVICE_HOST=8.4"
+docker compose exec -T commons sh -c "curl -kL http://internal-services-test:3000/mariadb?service=mysql-8-4" | grep "LAGOON_TEST_VAR=all-images"
+
 # postgres-11 should be version 11 client
 docker compose exec -T postgres-11 bash -c "psql --version" | grep "psql" | grep "11."
 

helpers/services-docker-compose.yml

@@ -46,6 +46,18 @@ services:
       - "28017"
     << : *default-user # uses the defined user from top
 
+  mysql-8-0:
+    image: uselagoon/mysql-8.0:latest
+    ports:
+      - "3306"
+    << : *default-user # uses the defined user from top
+
+  mysql-8-4:
+    image: uselagoon/mysql-8.4:latest
+    ports:
+      - "3306"
+    << : *default-user # uses the defined user from top
+
   postgres-11:
     image: uselagoon/postgres-11:latest
     ports:

images/mysql/8.0.Dockerfile

@@ -0,0 +1,70 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM mysql:8.0.37-oracle
+
+LABEL org.opencontainers.image.authors="The Lagoon Authors" maintainer="The Lagoon Authors"
+LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-images" repository="https://github.com/uselagoon/lagoon-images"
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/wait-for /bin/
+COPY --from=commons /home /home
+
+RUN fix-permissions /etc/passwd \
+    && mkdir -p /home
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+ENV BACKUPS_DIR="/var/lib/mysql/backup"
+
+ENV MYSQL_DATABASE=lagoon \
+    MYSQL_USER=lagoon \
+    MYSQL_PASSWORD=lagoon \
+    MYSQL_ROOT_PASSWORD=Lag00n
+
+RUN microdnf install -y epel-release \
+    && microdnf update -y \
+    && microdnf install -y \
+        gettext \
+        net-tools \
+        pwgen \
+        tini \
+        wget; \
+    rm -rf /var/lib/mysql/* /etc/mysql/ /etc/my.cnf*; \
+    curl -sSL https://raw.githubusercontent.com/major/MySQLTuner-perl/master/mysqltuner.pl -o mysqltuner.pl
+
+COPY entrypoints/ /lagoon/entrypoints/
+COPY mysql-backup.sh /lagoon/
+COPY my.cnf /etc/mysql/my.cnf
+
+RUN for i in /var/run/mysqld /run/mysqld /var/lib/mysql /etc/mysql/conf.d /docker-entrypoint-initdb.d /home; \
+    do mkdir -p $i; chown mysql $i; /bin/fix-permissions $i; \
+    done
+
+COPY root/usr/share/container-scripts/mysql/readiness-probe.sh /usr/share/container-scripts/mysql/readiness-probe.sh
+RUN /bin/fix-permissions /usr/share/container-scripts/mysql/ \
+    && /bin/fix-permissions /etc/mysql
+
+RUN touch /var/log/mariadb-slow.log && /bin/fix-permissions /var/log/mariadb-slow.log \
+    && touch /var/log/mariadb-queries.log && /bin/fix-permissions /var/log/mariadb-queries.log
+
+# We cannot start mysql as root, we add the user mysql to the group root and
+# change the user of the Docker Image to this user.
+RUN usermod -a -G root mysql
+USER mysql
+ENV USER_NAME mysql
+
+WORKDIR /var/lib/mysql
+EXPOSE 3306
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/lagoon/entrypoints.bash"]
+CMD ["mysqld"]

images/mysql/8.4.Dockerfile

@@ -0,0 +1,70 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/commons as commons
+FROM mysql:8.4.0-oracle
+
+LABEL org.opencontainers.image.authors="The Lagoon Authors" maintainer="The Lagoon Authors"
+LABEL org.opencontainers.image.source="https://github.com/uselagoon/lagoon-images" repository="https://github.com/uselagoon/lagoon-images"
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+# Copy commons files
+COPY --from=commons /lagoon /lagoon
+COPY --from=commons /bin/fix-permissions /bin/ep /bin/docker-sleep /bin/wait-for /bin/
+COPY --from=commons /home /home
+
+RUN fix-permissions /etc/passwd \
+    && mkdir -p /home
+
+ENV TMPDIR=/tmp \
+    TMP=/tmp \
+    HOME=/home \
+    # When Bash is invoked via `sh` it behaves like the old Bourne Shell and sources a file that is given in `ENV`
+    ENV=/home/.bashrc \
+    # When Bash is invoked as non-interactive (like `bash -c command`) it sources a file that is given in `BASH_ENV`
+    BASH_ENV=/home/.bashrc
+
+ENV BACKUPS_DIR="/var/lib/mysql/backup"
+
+ENV MYSQL_DATABASE=lagoon \
+    MYSQL_USER=lagoon \
+    MYSQL_PASSWORD=lagoon \
+    MYSQL_ROOT_PASSWORD=Lag00n
+
+RUN microdnf install -y epel-release \
+    && microdnf update -y \
+    && microdnf install -y \
+        gettext \
+        net-tools \
+        pwgen \
+        tini \
+        wget; \
+    rm -rf /var/lib/mysql/* /etc/mysql/ /etc/my.cnf*; \
+    curl -sSL https://raw.githubusercontent.com/major/MySQLTuner-perl/master/mysqltuner.pl -o mysqltuner.pl
+
+COPY entrypoints/ /lagoon/entrypoints/
+COPY mysql-backup.sh /lagoon/
+COPY my.cnf /etc/mysql/my.cnf
+
+RUN for i in /var/run/mysqld /run/mysqld /var/lib/mysql /etc/mysql/conf.d /docker-entrypoint-initdb.d /home; \
+    do mkdir -p $i; chown mysql $i; /bin/fix-permissions $i; \
+    done
+
+COPY root/usr/share/container-scripts/mysql/readiness-probe.sh /usr/share/container-scripts/mysql/readiness-probe.sh
+RUN /bin/fix-permissions /usr/share/container-scripts/mysql/ \
+    && /bin/fix-permissions /etc/mysql
+
+RUN touch /var/log/mariadb-slow.log && /bin/fix-permissions /var/log/mariadb-slow.log \
+    && touch /var/log/mariadb-queries.log && /bin/fix-permissions /var/log/mariadb-queries.log
+
+# We cannot start mysql as root, we add the user mysql to the group root and
+# change the user of the Docker Image to this user.
+RUN usermod -a -G root mysql
+USER mysql
+ENV USER_NAME mysql
+
+WORKDIR /var/lib/mysql
+EXPOSE 3306
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/lagoon/entrypoints.bash"]
+CMD ["mysqld"]

images/mysql/entrypoints/100-myql-logging.bash

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ -n "$MYSQL_LOG_SLOW" ]; then
+  echo "MYSQL_LOG_SLOW set, logging to /var/log/mysql-slow.log"
+  cat <<EOF > /etc/mysql/conf.d/log-slow.cnf
+[mysqld]
+log_output=file
+slow_query_log = 1
+slow_query_log_file = /var/log/mysql-slow.log
+long_query_time = ${MYSQL_LONG_QUERY_TIME:-10}
+log_slow_rate_limit = ${MYSQL_LOG_SLOW_RATE_LIMIT:-1}
+EOF
+fi
+
+
+if [ -n "$MYSQL_LOG_QUERIES" ]; then
+  echo "MYSQL_LOG_QUERIES set, logging to /var/log/mysql-queries.log"
+  cat <<EOF > /etc/mysql/conf.d/log-queries.cnf
+
+[mysqld]
+general-log
+log-output=file
+general-log-file=/var/log/mysql-queries.log
+EOF
+fi

images/mysql/entrypoints/150-mysql-performance.bash

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+if [ "$LAGOON_ENVIRONMENT_TYPE" == "production" ]; then
+  # only set if not already defined
+  if [ -z ${MYSQL_INNODB_BUFFER_POOL_SIZE+x} ]; then
+    export MYSQL_INNODB_BUFFER_POOL_SIZE=1024M
+  fi
+  if [ -z ${MYSQL_INNODB_LOG_FILE_SIZE+x} ]; then
+    export MYSQL_INNODB_LOG_FILE_SIZE=256M
+  fi
+fi
+
+if [ -n "$MYSQL_PERFORMANCE_SCHEMA" ]; then
+  echo "Enabling performance schema"
+  cat <<EOF > /etc/mysql/conf.d/performance-schema.cnf
+[mysqld]
+performance_schema=ON
+performance-schema-instrument='stage/%=ON'
+performance-schema-consumer-events-stages-current=ON
+performance-schema-consumer-events-stages-history=ON
+performance-schema-consumer-events-stages-history-long=ON
+EOF
+
+fi

images/mysql/entrypoints/200-mysql-envplate.bash

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+/bin/ep /etc/mysql/*