feat: add multi_copy mode (#85)

* Add unarchive based copy method to upload_helm_chart role

Synchronize module does not work when for some reason there is no
ssh to the target, which can happen when using custom ansible
connection plugins to containerised targets.

This patch adds the multi_copy action plugin which is a wrapper
around the unarchive module, copying many files in one task.

* Use multi_copy instead of synchronize to copy cluster api providers

This works on targets that do not have ssh enabled, which synchronize
does not.

* Add molecule test for upload_helm_chart 'copy' method

This patch also extends the molecule test to cover the case
where a new version of a chart is uploaded which should result
in a file being removed from the destination.

---------

Co-authored-by: Jonathan Rosser <[email protected]>

molecule/upload-helm-chart/converge.yml

@@ -17,3 +17,5 @@
   become: true
   roles:
     - role: test-role
+      vars:
+        test_relative_path : "chart-v1/"

molecule/upload-helm-chart/roles/test-role/meta/main.yml

@@ -1,9 +1,12 @@
 dependencies:
   - role: vexxhost.kubernetes.upload_helm_chart
     vars:
-      upload_helm_chart_src: "./files/chart/"
-      upload_helm_chart_dest: /usr/local/src/chart-1
+      # with rsync
+      upload_helm_chart_src: "{{ test_relative_path }}"
+      upload_helm_chart_dest: /usr/local/src/chart-one
   - role: vexxhost.kubernetes.upload_helm_chart
     vars:
-      upload_helm_chart_src: "../../charts/foo/"
-      upload_helm_chart_dest: /usr/local/src/chart-2
+      # with multi_copy
+      upload_helm_chart_src: "{{ test_relative_path }}"
+      upload_helm_chart_dest: /usr/local/src/chart-two
+      upload_helm_chart_method: copy

molecule/upload-helm-chart/side_effect.yml

@@ -0,0 +1,21 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Side effect
+  hosts: all
+  become: true
+  roles:
+    - role: test-role
+      vars:
+        test_relative_path: "chart-v2/"

molecule/upload-helm-chart/verify.yml

@@ -0,0 +1,30 @@
+- name: Verify
+  hosts: all
+  become: true
+  tasks:
+    - name: Stat file 1
+      ansible.builtin.stat:
+        path: /usr/local/src/chart-one/.gitkeep
+      register: stat_1
+
+    - name: Stat file 2
+      ansible.builtin.stat:
+        path: /usr/local/src/chart-two/.gitkeep
+      register: stat_2
+
+    - name: Stat removed file 1
+      ansible.builtin.stat:
+        path: /usr/local/src/chart-one/file
+      register: stat_r1
+
+    - name: Stat removed file 2
+      ansible.builtin.stat:
+        path: /usr/local/src/chart-two/file
+      register: stat_r2
+
+    - ansible.builtin.assert:
+        that:
+          - stat_1.stat.exists
+          - stat_2.stat.exists
+          - not stat_r1.stat.exists
+          - not stat_r2.stat.exists

plugins/action/multi_copy.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2023 BBC R&D
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.errors import AnsibleActionFail
+from ansible.plugins.action import ActionBase
+
+import os
+import shutil
+from tempfile import TemporaryDirectory
+from zipfile import ZipFile, ZipInfo
+
+
+class ActionModule(ActionBase):
+    def run(self, tmp=None, task_vars=None):
+        super(ActionModule, self).run(tmp, task_vars)
+
+        task_vars = task_vars or {}
+
+        # Check required arguments
+        for required_arg in ["src", "dest"]:
+            if required_arg not in self._task.args:
+                raise AnsibleActionFail("Missing '{}' argument.".format(required_arg))
+
+        with TemporaryDirectory() as tmp_dir:
+          zip_file_name = os.path.join(tmp_dir, "multi_copy.zip")
+          source = self._find_needle('files', self._task.args['src'])
+          shutil.make_archive(zip_file_name, 'gztar', source)
+
+          # Upload and extract the files
+          unarchive_args = {
+              "src": zip_file_name + ".tar.gz",
+          }
+          for attr_arg in ["dest", "mode", "group", "owner", "attributes", "list_files"]:
+              if attr_arg in self._task.args:
+                  unarchive_args[attr_arg] = self._task.args[attr_arg]
+          unarchive_result = self._execute_action_plugin(
+              name='ansible.builtin.unarchive',
+              args=unarchive_args,
+              task_vars=task_vars,
+          )
+
+          return unarchive_result
+
+    def _execute_action_plugin(self, name, args, task_vars):
+        task = self._task.copy()
+        task.args = args
+
+        action_plugin = self._shared_loader_obj.action_loader.get(
+            name,
+            task=task,
+            connection=self._connection,
+            play_context=self._play_context,
+            loader=self._loader,
+            templar=self._templar,
+            shared_loader_obj=self._shared_loader_obj,
+        )
+
+        return action_plugin.run(task_vars=task_vars)

roles/cluster_api/tasks/main.yml

@@ -21,12 +21,12 @@
     group: root
 
 - name: Copy over all provider configuration to the remote system
-  ansible.posix.synchronize:
-    src: providers/
+  vexxhost.kubernetes.multi_copy:
+    src: "{{ role_path }}/files/providers"
     dest: "{{ cluster_api_provider_path }}"
-    archive: false
-    recursive: true
-    checksum: true
+    mode: "0755"
+    owner: root
+    group: root
 
 - name: Get a list of all Cluster API providers
   run_once: true

roles/upload_helm_chart/defaults/main.yml

@@ -0,0 +1,15 @@
+# Copyright (c) 2023 BBC R&D
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+upload_helm_chart_method: synchronize

roles/upload_helm_chart/tasks/copy.yml

@@ -0,0 +1,51 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Copy without using synchronize module
+  run_once: true
+  block:
+    - name: Ensure destination dir
+      ansible.builtin.file:
+        path: "{{ upload_helm_chart_dest }}"
+        state: directory
+        mode: "0755"
+        owner: root
+        group: root
+
+    - name: Upload helm chart with multi_copy
+      vexxhost.kubernetes.multi_copy:
+        src: "{{ upload_helm_chart_src }}"
+        dest: "{{ upload_helm_chart_dest }}"
+        owner: root
+        group: root
+        mode: "0755"
+        list_files: true
+      register: multi_copy
+      when: upload_helm_chart_method == 'copy'
+
+    - name: Find destination files
+      ansible.builtin.find:
+        paths: "{{ upload_helm_chart_dest }}"
+        recurse: true
+        hidden: true
+      register: destination_files
+
+    - name: Delete any destination files not in the sources
+      vars:
+        s: "{{ multi_copy.files | map('regex_replace', '\\./',  '/') }}"
+        d: "{{ destination_files.files | map(attribute='path') | map('regex_replace', upload_helm_chart_dest | realpath, '') }}"
+      ansible.builtin.file:
+        path: "{{ upload_helm_chart_dest ~ item }}"
+        state: absent
+      with_items: "{{ d | difference(s) }}"

roles/upload_helm_chart/tasks/main.yml

@@ -1,12 +1,2 @@
-- name: Upload Helm chart
-  run_once: true
-  # NOTE(mnaser): https://github.com/ansible-collections/ansible.posix/issues/381
-  #               https://github.com/ansible-collections/ansible.posix/pull/433
-  vexxhost.kubernetes.synchronize:
-    src: "{{ upload_helm_chart_src }}"
-    dest: "{{ upload_helm_chart_dest }}"
-    archive: false
-    recursive: true
-    checksum: true
-    delete: true
-    use_ssh_args: true
+- name: Import help chart upload method tasks
+  ansible.builtin.import_tasks: "{{ upload_helm_chart_method }}.yml"

roles/upload_helm_chart/tasks/synchronize.yml

@@ -0,0 +1,26 @@
+# Copyright (c) 2023 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Upload Helm chart
+  run_once: true
+  # NOTE(mnaser): https://github.com/ansible-collections/ansible.posix/issues/381
+  #               https://github.com/ansible-collections/ansible.posix/pull/433
+  vexxhost.kubernetes.synchronize:
+    src: "{{ upload_helm_chart_src }}"
+    dest: "{{ upload_helm_chart_dest }}"
+    archive: false
+    recursive: true
+    checksum: true
+    delete: true
+    use_ssh_args: true