From e2786afa58106a9a006ee5aa224cc806ed1bb917 Mon Sep 17 00:00:00 2001 From: Danyal Faheem <138459282+Danyal-Faheem@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:00:48 +0500 Subject: [PATCH] fix: upgrade MySQL from 5.7 to 8.1 first and then to 8.4 (#1149) * fix: upgrade MySQL from 5.7 to 8.1 first and then to 8.4 This is required when upgrading from Tutor v15 to v18 directly MySQL does not allow direct upgrades from v5.7 to v8.4 * fix: run MySQL 8.1 as a separate container during upgrade from Olive to Redwood (#1140) We do this because MySQL 8.1 does not have the --mysql-native-password option We have this option turned on for backward compatibility --- ..._danyal.faheem_mysql_upgrade_5_7_to_8_4.md | 2 + ...eem_run_mysql_8_1_as_separate_container.md | 1 + tutor/commands/upgrade/common.py | 33 +++- tutor/commands/upgrade/compose.py | 60 ++++++- tutor/commands/upgrade/k8s.py | 146 +++++++++++++++++- 5 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md create mode 100644 changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md diff --git a/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md b/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md new file mode 100644 index 0000000000..01275c273d --- /dev/null +++ b/changelog.d/20240726_202449_danyal.faheem_mysql_upgrade_5_7_to_8_4.md @@ -0,0 +1,2 @@ +- [Bugfix] Do not directly upgrade MySQL from v5.7 to v8.4 when upgrading from quince as MySQL does not allow that. First, upgrade to v8.1 and then to v8.4. (by @Danyal-Faheem) + This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security. \ No newline at end of file diff --git a/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md b/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md new file mode 100644 index 0000000000..a4b5403c75 --- /dev/null +++ b/changelog.d/20241018_122745_danyal.faheem_run_mysql_8_1_as_separate_container.md @@ -0,0 +1 @@ +- [Bugfix] Run MySQL 8.1 as a separate container during upgrade from Olive to Redwood as it crashed otherwise due to the `--mysql-native-password` option not being present. (by @Danyal-Faheem) \ No newline at end of file diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index e641a77001..783be6435d 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -1,10 +1,13 @@ from __future__ import annotations +from typing import Optional + import click +from packaging import version from tutor import config as tutor_config from tutor import fmt, plugins -from tutor.types import Config +from tutor.types import Config, get_typed def upgrade_from_lilac(config: Config) -> None: @@ -60,6 +63,34 @@ def get_mongo_upgrade_parameters( return mongo_version, admin_command +def get_intermediate_mysql_upgrade(config: Config) -> Optional[str]: + """ + Checks if a MySQL upgrade is needed based on the Tutor version and MySQL setup. + + This method ensures that MySQL is running and determines if the upgrade + process should proceed based on the Tutor version. It is intended for upgrades + from Tutor version 15 to version 18 or later. Manual upgrade steps are not + required for versions 16 or 17. + + Returns: + Optional[str]: The docker image of MySQL to upgrade to or None if not applicable + """ + if not get_typed(config, "RUN_MYSQL", bool): + fmt.echo_info( + "You are not running MySQL (RUN_MYSQL=false). It is your " + "responsibility to upgrade your MySQL instance to v8.4. There is " + "nothing left to do to upgrade from Olive." + ) + return None + image_tag = get_typed(config, "DOCKER_IMAGE_MYSQL", str).split(":")[-1] + # If latest image, we assign a constant value to invalidate the condition + # as we know that the latest image will always be greater than 8.1.0 + target_version = ( + version.Version("8.1.1") if image_tag == "latest" else version.parse(image_tag) + ) + return "docker.io/mysql:8.1.0" if target_version > version.parse("8.1.0") else None + + PALM_RENAME_ORA2_FOLDER_COMMAND = """ if stat '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' 2> /dev/null; then echo "Renaming ora2 folder..." diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index df54c71d8b..4f8f7773d8 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -4,8 +4,9 @@ from tutor import config as tutor_config from tutor import env as tutor_env +from tutor import hooks from tutor import fmt -from tutor.commands import compose +from tutor.commands import compose, jobs from tutor.types import Config from . import common as common_upgrade @@ -158,6 +159,63 @@ def upgrade_from_olive(context: click.Context, config: Config) -> None: upgrade_mongodb(context, config, "4.2.17", "4.2") upgrade_mongodb(context, config, "4.4.22", "4.4") + intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade( + config + ) + if not intermediate_mysql_docker_image: + return + + click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}")) + + # We start up a mysql-8.1 container to build data dictionary to preserve + # the upgrade order of 5.7 -> 8.1 -> 8.4 + # Use the mysql-8.1 context so that we can clear these filters later on + with hooks.Contexts.app("mysql-8.1").enter(): + hooks.Filters.ENV_PATCHES.add_items( + [ + ( + "local-docker-compose-services", + """ +mysql-8.1: + extends: mysql + image: docker.io/mysql:8.1.0 + command: > + mysqld + --character-set-server=utf8mb3 + --collation-server=utf8mb3_general_ci + --binlog-expire-logs-seconds=259200 + """, + ), + ( + "local-docker-compose-jobs-services", + """ +mysql-8.1-job: + image: docker.io/mysql:8.1.0 + depends_on: {{ [("mysql-8.1", RUN_MYSQL)]|list_if }} + """, + ), + ] + ) + hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-8.1")) + + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("mysql-8.1", tutor_env.read_core_template_file("jobs", "init", "mysql.sh")) + ) + + tutor_env.save(context.obj.root, config) + + # Run the init command to make sure MySQL is ready for connections + context.invoke(jobs.initialise, limit="mysql-8.1") + context.invoke(compose.stop, services=["mysql-8.1"]) + + # Clear the filters added for mysql-8.1 as we don't need them anymore + hooks.clear_all(context="app:mysql-8.1") + + # Save environment and run init for mysql 8.4 to make sure MySQL is ready + tutor_env.save(context.obj.root, config) + context.invoke(jobs.initialise, limit="mysql") + context.invoke(compose.stop, services=["mysql"]) + def upgrade_from_quince(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Quince")) diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index db18835704..e547bf4b80 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -2,7 +2,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt +from tutor import fmt, hooks from tutor.commands import k8s from tutor.commands.context import Context from tutor.types import Config @@ -39,7 +39,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None: running_release = "olive" if running_release == "olive": - upgrade_from_olive(context.obj, config) + upgrade_from_olive(context, config) running_release = "palm" if running_release == "palm": @@ -148,11 +148,11 @@ def upgrade_from_maple(context: Context, config: Config) -> None: ) -def upgrade_from_olive(context: Context, config: Config) -> None: +def upgrade_from_olive(context: click.Context, config: Config) -> None: # Note that we need to exec because the ora2 folder is not bind-mounted in the job # services. k8s.kubectl_apply( - context.root, + context.obj.root, "--selector", "app.kubernetes.io/name=lms", ) @@ -165,6 +165,144 @@ def upgrade_from_olive(context: Context, config: Config) -> None: upgrade_mongodb(config, "4.2.17", "4.2") upgrade_mongodb(config, "4.4.22", "4.4") + intermediate_mysql_docker_image = common_upgrade.get_intermediate_mysql_upgrade( + config + ) + if not intermediate_mysql_docker_image: + return + + click.echo(fmt.title(f"Upgrading MySQL to {intermediate_mysql_docker_image}")) + + # We start up a mysql-8.1 container to build data dictionary to preserve + # the upgrade order of 5.7 -> 8.1 -> 8.4 + # Use the mysql-8.1 context so that we can clear these filters later on + with hooks.Contexts.app("mysql-8.1").enter(): + hooks.Filters.ENV_PATCHES.add_items( + [ + ( + "k8s-deployments", + """ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql-81 + labels: + app.kubernetes.io/name: mysql-81 +spec: + selector: + matchLabels: + app.kubernetes.io/name: mysql-81 + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: mysql-81 + spec: + securityContext: + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + fsGroupChangePolicy: "OnRootMismatch" + containers: + - name: mysql-81 + image: docker.io/mysql:8.1.0 + args: + - "mysqld" + - "--character-set-server=utf8mb3" + - "--collation-server=utf8mb3_general_ci" + - "--binlog-expire-logs-seconds=259200" + env: + - name: MYSQL_ROOT_PASSWORD + value: "{{ MYSQL_ROOT_PASSWORD }}" + ports: + - containerPort: 3306 + volumeMounts: + - mountPath: /var/lib/mysql + name: data + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: data + persistentVolumeClaim: + claimName: mysql + """, + ), + ( + "k8s-jobs", + """ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mysql-81-job + labels: + app.kubernetes.io/component: job +spec: + template: + spec: + restartPolicy: Never + containers: + - name: mysql-81 + image: docker.io/mysql:8.1.0 + """, + ), + ] + ) + hooks.Filters.ENV_PATCHES.add_item( + ( + "k8s-services", + """ +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql-81 + labels: + app.kubernetes.io/name: mysql-81 +spec: + type: ClusterIP + ports: + - port: 3306 + protocol: TCP + selector: + app.kubernetes.io/name: mysql-81 + """, + ) + ) + hooks.Filters.CONFIG_DEFAULTS.add_item(("MYSQL_HOST", "mysql-81")) + + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("mysql-81", tutor_env.read_core_template_file("jobs", "init", "mysql.sh")) + ) + + tutor_env.save(context.obj.root, config) + + # Run the init command to make sure MySQL is ready for connections + k8s.kubectl_apply( + context.obj.root, + "--selector", + "app.kubernetes.io/name=mysql-81", + ) + k8s.wait_for_deployment_ready(config, "mysql-81") + context.invoke(k8s.do.commands["init"], limit="mysql-8.1") + context.invoke(k8s.stop, names=["mysql-81"]) + + # Clear the filters added for mysql-8.1 as we don't need them anymore + hooks.clear_all(context="app:mysql-8.1") + + # Save environment and run init for mysql 8.4 to make sure MySQL is ready + tutor_env.save(context.obj.root, config) + k8s.kubectl_apply( + context.obj.root, + "--selector", + "app.kubernetes.io/name=mysql", + ) + k8s.wait_for_deployment_ready(config, "mysql") + context.invoke(k8s.do.commands["init"], limit="mysql") + context.invoke(k8s.stop, names=["mysql"]) + def upgrade_from_quince(config: Config) -> None: click.echo(fmt.title("Upgrading from Quince"))