[ATMOSPHERE-545] Add playbook and roles for mariadb migration to pxc

playbooks/migrate_mariadb.yml

@@ -0,0 +1,32 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This playbook is used to migrate from OpenStack Ansible to Atmosphere, you
+# should run it with the following manner:
+#
+# ansible-playbook \
+#   -i inventory/hosts.ini \
+#   -i /opt/openstack-ansible/inventory/dynamic_inventory.py \
+#   vexxhost.atmosphere.migrate_from_osa
+
+- name: Migrate Mariadb to PXC
+  hosts: controllers[0]
+  become: true
+  tasks:
+    - name: Migrate Heat
+      ansible.builtin.import_role:
+        name: heat
+        tasks_from: migrate_mariadb
+      tags:
+        - heat

roles/heat/tasks/migrate_mariadb.yml

@@ -0,0 +1,52 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Check If the service uses legacy mariadb and PXC is fresh
+  ansible.builtin.include_role:
+    name: migrate_db_from_mariadb
+    tasks_from: pre_check
+  vars:
+    migrate_db_from_mariadb_openstack_service: heat
+    migrate_db_from_mariadb_pxc_namespace: "{{ heat_helm_release_namespace }}"
+    migrate_db_from_mariadb_databases:
+      heat: heat
+
+- name: Scale down heat service
+  kubernetes.core.k8s_scale:
+    api_version: v1
+    kind: Deployment
+    namespace: "{{ heat_helm_release_namespace }}"
+    name: "{{ item }}"
+    replicas: 0
+    wait_timeout: 60
+  retries: 3
+  delay: 1
+  loop:
+    - heat-api
+    - heat-cfn
+    - heat-engine
+
+- name: Migrate the database
+  when: _service_uses_mariadb
+  ansible.builtin.include_role:
+    name: migrate_db_from_mariadb
+    tasks_from: migrate_db
+  vars:
+    migrate_db_from_mariadb_mariadb_namespace: "{{ heat_helm_release_namespace }}"
+    migrate_db_from_mariadb_databases:
+      heat: heat
+
+- name: Run deployment flow
+  ansible.builtin.import_tasks:
+    file: main.yml

roles/migrate_db_from_mariadb/README.md

@@ -0,0 +1,13 @@
+# `migrate_db_from_mariadb`
+
+This is a role that is designed to migrate the database of service
+from Openstack Helm Mariadb to Atmosphare Percona XtraDB Cluster(PXC)
+deployment. It will take care of the following:
+
+* Check if Openstack service uses Mariadb from etc secret of service.
+If it uses PXC already, stop.
+* Ensure that the database does not exist in the PXC database already.
+If exists, stop.
+* Shut off all of the containers that are using the database (This step is not included in the role. Need to be done outside of this role scope.)
+* Dump the database from the OpenStack Helm Mariadb database
+* Restore the database into the PXC database

roles/migrate_db_from_mariadb/defaults/main.yml

@@ -0,0 +1,30 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Namespace where the Percona XtraDB Cluster is deployed
+# migrate_db_from_mariadb_pxc_namespace:
+
+# Namespace where the mariadb is deployed
+# migrate_db_from_mariadb_mariadb_namespace:
+
+# Openstack service name who owns databases
+# migrate_db_from_mariadb_openstack_service: keystone
+
+# Mariadb service name
+migrate_db_from_mariadb_mariadb_service_name: mariadb
+
+# Mapping of databases to migrate, where key is Mariadb database name and value is
+# Atmosphere database name
+# migrate_db_from_mariadb_databases:
+#   magnum_service: magnum

roles/migrate_db_from_mariadb/meta/main.yml

@@ -0,0 +1,24 @@
+# Copyright (c) 2022 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+galaxy_info:
+  author: VEXXHOST, Inc.
+  description: Ansible role for migrating databases from Openstack Helm Mariadb to PXC
+  license: Apache-2.0
+  min_ansible_version: 5.5.0
+  standalone: false
+  platforms:
+    - name: Ubuntu
+      versions:
+        - focal

roles/migrate_db_from_mariadb/tasks/main.yml

@@ -0,0 +1,19 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Check If the service uses legacy mariadb and PXC is fresh
+  ansible.builtin.include_tasks: pre_check.yml
+
+- name: Migrate database
+  ansible.builtin.include_tasks: migrate_db.yml

roles/migrate_db_from_mariadb/tasks/migrate_db.yml

@@ -0,0 +1,54 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Copy data from Mariadb to PXC
+  when: _service_uses_mariadb
+  run_once: true
+  block:
+    - name: Get the Kuberentes service for Mariadb
+      run_once: true
+      kubernetes.core.k8s_info:
+        kind: Service
+        name: "{{ migrate_db_from_mariadb_mariadb_service_name }}"
+        namespace: "{{ migrate_db_from_mariadb_mariadb_namespace }}"
+      register: _mariadb_service
+
+    - name: Install the mysql client
+      ansible.builtin.package:
+        name: mysql-client
+        state: latest # noqa: package-latest
+
+    - name: Dump the database
+      run_once: true
+      community.mysql.mysql_db:
+        login_host: "{{ _mariadb_service.resources[0].spec.clusterIP }}"
+        login_user: "{{ _service_mariadb_username }}"
+        login_password: "{{ _service_mariadb_password }}"
+        state: dump
+        name: "{{ item.key }}"
+        dump_extra_args: --skip_add_locks --set-gtid-purged=OFF
+        skip_lock_tables: true
+        target: "/root/{{ item.key }}-{{ ansible_date_time.epoch }}.sql"
+      loop: "{{ migrate_db_from_mariadb_databases | dict2items }}"
+
+    - name: Restore the database
+      run_once: true
+      community.mysql.mysql_db:
+        login_host: "{{ _pxc_service.resources[0].spec.clusterIP }}"
+        login_user: root
+        login_password: "{{ openstack_helm_endpoints.oslo_db.auth.admin.password }}"
+        state: import
+        name: "{{ item.value }}"
+        target: "/root/{{ item.key }}-{{ ansible_date_time.epoch }}.sql"
+      loop: "{{ migrate_db_from_mariadb_databases | dict2items }}"

roles/migrate_db_from_mariadb/tasks/pre_check.yml

@@ -0,0 +1,73 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Check if the service uses Mariadb
+  run_once: true
+  block:
+    - name: Get the Openstack service secret
+      kubernetes.core.k8s_info:
+        api_version: v1
+        kind: Secret
+        name: "{{ migrate_db_from_mariadb_openstack_service }}-etc"
+        namespace: openstack
+      register: _service_etc_secret
+
+    - name: Fetch db connection string secret data
+      ansible.builtin.set_fact:
+        _service_db_conn_string: "{{ _service_etc_secret.resources[0].data.values() | map('b64decode') | select('search', 'mysql\\+pymysql') }}"
+
+    - name: Check if the service uses legacy MariaDB
+      ansible.builtin.set_fact:
+        _service_uses_mariadb: >-
+          {{
+            _service_db_conn_string
+            | select('search', migrate_db_from_mariadb_mariadb_service_name)
+            | list
+            | length > 0
+          }}
+
+    - name: Fetch the mariadb username and password
+      when: _service_uses_mariadb
+      ansible.builtin.set_fact:
+        _service_mariadb_credential: "{{ _service_db_conn_string | map('split', 'mysql+pymysql://') | list | map('last') |list | map('split', '@' + migrate_db_from_mariadb_mariadb_service_name) | list | map('first') }}"
+
+    - name: Fetch the mariadb username and password
+      when: _service_uses_mariadb
+      ansible.builtin.set_fact:
+        _service_mariadb_username: "{{ _service_mariadb_credential | map('split', ':') | map('first') | join('') }}"
+        _service_mariadb_password: "{{ _service_mariadb_credential | map('split', ':') | map('last') | join('') }}"
+
+- name: Check if the database exists in PXC already
+  when: _service_uses_mariadb
+  run_once: true
+  block:
+    - name: Get the Kuberentes service for Percona XtraDB Cluster
+      run_once: true
+      kubernetes.core.k8s_info:
+        kind: Service
+        name: "{{ openstack_helm_endpoints.oslo_db.hosts.default }}"
+        namespace: "{{ migrate_db_from_mariadb_pxc_namespace }}"
+      register: _pxc_service
+
+    - name: Fail if the databases already exist
+      run_once: true
+      check_mode: true
+      community.mysql.mysql_db:
+        login_host: "{{ _pxc_service.resources[0].spec.clusterIP }}"
+        login_user: root
+        login_password: "{{ openstack_helm_endpoints.oslo_db.auth.admin.password }}"
+        name: "{{ item.value }}"
+      register: _pxc_db_check
+      failed_when: _pxc_db_check is not changed
+      loop: "{{ migrate_db_from_mariadb_databases | dict2items }}"