Add support for installing Ansible collections on IAG servers (#110)

* Update IAG Ansible version
Add support for installing Ansible collections on IAG servers

* Fix bug when IAG Ansible collection list is empty

* Add size check when installing IAG packages

* Rename install-ansible-collections to install-ansible-collections-offline

roles/common_vars/defaults/main/main.yml

@@ -21,17 +21,20 @@ rpms_path: "{{ packages_path }}/rpms"
 wheels_path: "{{ packages_path }}/wheels"
 archives_path: "{{ packages_path }}/archives"
 adapters_path: "{{ packages_path }}/adapters"
+ansible_collections_path: "{{ packages_path }}/ansible_collections"
 
 packages_download_root_control_node: "{{ playbook_dir }}/files"
 packages_download_dir_control_node: "{{ packages_download_root_control_node }}/{{ packages_path }}"
 rpms_download_dir_control_node: "{{ packages_download_dir_control_node }}/rpms"
 wheels_download_dir_control_node: "{{ packages_download_dir_control_node }}/wheels"
 archives_download_dir_control_node: "{{ packages_download_dir_control_node }}/archives"
 adapters_download_dir_control_node: "{{ packages_download_dir_control_node }}/adapters"
+ansible_collections_download_dir_control_node: "{{ packages_download_dir_control_node }}/ansible_collections"
 
 packages_download_root_target_node: /var/tmp
 packages_download_dir_target_node: "{{ packages_download_root_target_node }}/{{ packages_path }}"
 rpms_download_dir_target_node: "{{ packages_download_dir_target_node }}/rpms"
 wheels_download_dir_target_node: "{{ packages_download_dir_target_node }}/wheels"
 archives_download_dir_target_node: "{{ packages_download_dir_target_node }}/archives"
 adapters_download_dir_target_node: "{{ packages_download_dir_target_node }}/adapters"
+ansible_collections_download_dir_target_node: "{{ packages_download_dir_target_node }}/ansible_collections"

roles/gateway/defaults/main.yml

@@ -20,6 +20,9 @@ iag_enable_git: true
 # Location of IAG assets
 iag_install_dir: /opt/automation-gateway
 
+# Location of IAG Ansible collections
+iag_ansible_collections_path: "{{ iag_install_dir }}/ansible/collections"
+
 # Location of IAG data
 iag_data_dir: /var/lib/automation-gateway
 

roles/gateway/tasks/download-ansible-collections.yml

@@ -0,0 +1,23 @@
+# Copyright (c) 2024, Itential, Inc
+# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt)
+---
+- name: Create Ansible collections download directory
+  ansible.builtin.file:
+    path: "{{ ansible_collections_download_dir_target_node }}"
+    state: directory
+    mode: '0755'
+
+- name: Download Ansible collections
+  ansible.builtin.command:
+    chdir: "{{ ansible_collections_download_dir_target_node }}"
+    cmd: "{{ iag_pkgs_temp_dir.path }}/offline_install/bin/ansible-galaxy collection download {{ item }}"
+  with_items: "{{ iag_ansible_collections }}"
+  changed_when: true
+
+- name: Copy Ansible collections to control node
+  ansible.builtin.import_role:
+    name: offline
+    tasks_from: fetch-packages
+  vars:
+    src_dir: "{{ ansible_collections_download_dir_target_node }}/collections"
+    dest_dir: "{{ ansible_collections_download_dir_control_node }}"

roles/gateway/tasks/download-packages-python.yml

@@ -12,6 +12,14 @@
     download_dir: "{{ rpms_download_dir_target_node }}"
   tags: download_gateway_python_packages
 
+- name: Copy Gateway Python rpms to control node
+  ansible.builtin.import_role:
+    name: offline
+    tasks_from: fetch-packages
+  vars:
+    src_dir: "{{ rpms_download_dir_target_node }}"
+    dest_dir: "{{ rpms_download_dir_control_node }}"
+
 - name: Install Python
   ansible.builtin.include_role:
     name: python

roles/gateway/tasks/download-packages.yml

@@ -1,30 +1,32 @@
 # Copyright (c) 2024, Itential, Inc
 # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt)
 ---
+- name: Validation steps
+  tags: always
+  block:
+    - name: Validate distribution major version
+      ansible.builtin.fail:
+        msg: Download not supported for version 7
+      when: ansible_distribution_major_version == "7"
 
-- name: Validate distribution major version
-  ansible.builtin.fail:
-    msg: Download not supported for version 7
-  when: ansible_distribution_major_version == "7"
+    - name: Validate offline_install variable
+      ansible.builtin.fail:
+        msg: offline_install must be set to false for download
+      when:
+        - offline_install is defined
+        - offline_install
 
-- name: Validate offline_install variable
-  ansible.builtin.fail:
-    msg: offline_install must be set to false for download
-  when:
-    - offline_install is defined
-    - offline_install
-
-- name: Include release vars
-  ansible.builtin.include_vars:
-    file: "{{ item }}"
-  with_first_found:
-    - "{{ iag_release }}-{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version }}.yml"
-    - "release-undefined.yml"
+    - name: Include release vars
+      ansible.builtin.include_vars:
+        file: "{{ item }}"
+      with_first_found:
+        - "{{ iag_release }}-{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version }}.yml"
+        - "release-undefined.yml"
 
-- name: Check for valid IAG release
-  ansible.builtin.fail:
-    msg: "Deployer does not support installing Gateway version {{ iag_release }} on {{ ansible_distribution }}-{{ ansible_distribution_major_version }}"
-  when: invalid_iag_release is defined
+    - name: Check for valid IAG release
+      ansible.builtin.fail:
+        msg: "Deployer does not support installing Gateway version {{ iag_release }} on {{ ansible_distribution }}-{{ ansible_distribution_major_version }}"
+      when: invalid_iag_release is defined
 
 - name: Download Gateway OS rpms
   ansible.builtin.import_role:
@@ -36,12 +38,24 @@
     download_dir: "{{ rpms_download_dir_target_node }}"
   tags: download_gateway_packages
 
+- name: Copy rpms to control node
+  ansible.builtin.import_role:
+    name: offline
+    tasks_from: fetch-packages
+  vars:
+    src_dir: "{{ rpms_download_dir_target_node }}"
+    dest_dir: "{{ rpms_download_dir_control_node }}"
+
 - name: Install Gateway rpms
   ansible.builtin.dnf:
     name: "{{ item }}"
     state: present
   with_items: "{{ gateway_packages }}"
   register: gateway_packages_installed
+  when:
+    - gateway_packages is defined
+    - gateway_packages is iterable
+    - gateway_packages | length > 0
   tags: install_gateway_packages
 
 - name: Download Python packages
@@ -62,14 +76,6 @@
     src_dir: "{{ archives_download_dir_target_node }}"
     dest_dir: "{{ archives_download_dir_control_node }}"
 
-- name: Copy rpms to control node
-  ansible.builtin.import_role:
-    name: offline
-    tasks_from: fetch-packages
-  vars:
-    src_dir: "{{ rpms_download_dir_target_node }}"
-    dest_dir: "{{ rpms_download_dir_control_node }}"
-
 - name: Uninstall Gateway packages
   ansible.builtin.dnf:
     name: "{{ item }}"

roles/gateway/tasks/download-python-dependencies.yml

@@ -68,6 +68,17 @@
     extra_args: --log /var/log/pip.log
   when: iag_enable_ansible | bool
 
+- name: Download Ansible collections
+  tags: download_ansible_collections
+  block:
+    - name: Download Ansible collections
+      ansible.builtin.include_tasks:
+        file: download-ansible-collections.yml
+      when:
+        - iag_ansible_collections is defined
+        - iag_ansible_collections is iterable
+        - iag_ansible_collections | length > 0
+
 - name: Copy IAG wheel file to target node
   ansible.builtin.copy:
     src: "{{ iag_whl_file }}"

roles/gateway/tasks/install-ansible-collections-offline.yml

@@ -0,0 +1,33 @@
+# Copyright (c) 2024, Itential, Inc
+# GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt)
+---
+- name: Create temporary working directory
+  ansible.builtin.tempfile:
+    state: directory
+  register: install_collections_temp_dir
+
+- name: Copy collections to target node
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "{{ install_collections_temp_dir.path }}/{{ item | basename }}"
+    mode: '0644'
+  with_fileglob:
+    - "{{ ansible_collections_download_dir_control_node }}/*.tar.gz"
+    - "{{ ansible_collections_download_dir_control_node }}/requirements.yml"
+
+- name: Install collections
+  ansible.builtin.command:
+    cmd: "ansible-galaxy collection install -p {{ iag_ansible_collections_path }} -r requirements.yml"
+    chdir: "{{ install_collections_temp_dir.path }}"
+  register: install_result
+  changed_when: '"Nothing to do" not in install_result.stdout'
+  failed_when: install_result.failed
+  when:
+    - iag_ansible_collections is defined
+    - iag_ansible_collections is iterable
+    - iag_ansible_collections | length > 0
+
+- name: Remove temporary working directory
+  ansible.builtin.file:
+    path: "{{ install_collections_temp_dir.path }}"
+    state: absent

roles/gateway/tasks/main.yml

@@ -1,18 +1,20 @@
 # Copyright (c) 2024, Itential, Inc
 # GNU General Public License v3.0+ (see LICENSE or https://www.gnu.org/licenses/gpl-3.0.txt)
 ---
-- name: Include release vars
-  ansible.builtin.include_vars:
-    file: "{{ item }}"
-  with_first_found:
-    - "{{ iag_release }}-{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version }}.yml"
-    - "release-undefined.yml"
+- name: Validation steps
   tags: always
-
-- name: Check for valid IAG release
-  ansible.builtin.fail:
-    msg: "Deployer does not support installing Gateway version {{ iag_release }} on {{ ansible_distribution }}-{{ ansible_distribution_major_version }}"
-  when: invalid_iag_release is defined
+  block:
+    - name: Include release vars
+      ansible.builtin.include_vars:
+        file: "{{ item }}"
+      with_first_found:
+        - "{{ iag_release }}-{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version }}.yml"
+        - "release-undefined.yml"
+
+    - name: Check for valid IAG release
+      ansible.builtin.fail:
+        msg: "Deployer does not support installing Gateway version {{ iag_release }} on {{ ansible_distribution }}-{{ ansible_distribution_major_version }}"
+      when: invalid_iag_release is defined
 
 - name: Install base OS packages
   ansible.builtin.include_role:
@@ -29,7 +31,11 @@
     name: "{{ item }}"
     state: present
   with_items: "{{ gateway_packages }}"
-  when: not offline_install
+  when:
+    - not offline_install
+    - gateway_packages is defined
+    - gateway_packages is iterable
+    - gateway_packages | length > 0
   tags: install_gateway_packages
 
 - name: Install Gateway packages (offline)
@@ -64,11 +70,10 @@
     - "{{ iag_properties_location }}"
     - "{{ iag_data_dir }}"
     - "{{ iag_install_dir }}"
-    - "{{ iag_install_dir }}/ansible/collections"
+    - "{{ iag_ansible_collections_path }}"
     - "{{ iag_install_dir }}/ansible/inventory"
     - "{{ iag_install_dir }}/ansible/modules"
     - "{{ iag_install_dir }}/ansible/roles"
-    - "{{ iag_install_dir }}/ansible/collections"
     - "{{ iag_install_dir }}/ansible/playbooks"
     - "{{ iag_install_dir }}/ansible/plugins/modules"
     - "{{ iag_install_dir }}/ansible/scripts"
@@ -104,13 +109,32 @@
 
 # Need to install ansible within the virtual environment
 - name: Install Ansible
+  tags: install_ansible
   when: iag_enable_ansible | bool
   block:
-    - name: Install Ansible into the python virtual environment
-      ansible.builtin.pip:
-        name: "{{ iag_ansible_version }}"
-        virtualenv: "{{ iag_install_dir }}/venv"
+    - name: Install Ansible and Ansible collections
       when: not offline_install
+      block:
+        - name: Install Ansible into the python virtual environment
+          ansible.builtin.pip:
+            name: "{{ iag_ansible_version }}"
+            virtualenv: "{{ iag_install_dir }}/venv"
+
+        - name: Install collections
+          ansible.builtin.command:
+            cmd: "ansible-galaxy collection install -p {{ iag_ansible_collections_path }} {{  iag_ansible_collections | join(' ') }}"
+          register: install_result
+          changed_when: '"Nothing to do" not in install_result.stdout'
+          failed_when: install_result.failed
+          when:
+            - iag_ansible_collections is defined
+            - iag_ansible_collections is iterable
+            - iag_ansible_collections | length > 0
+
+    - name: Install Ansible collections (offline)
+      ansible.builtin.include_tasks:
+        file: install-ansible-collections-offline.yml
+      when: offline_install
 
     - name: Create Ansible config file
       ansible.builtin.template:

roles/gateway/tasks/upgrade-iag.yml

@@ -6,11 +6,42 @@
     state: directory
   register: workingdir
 
-- name: Copy IAG to host
+- name: Copy IAG archive from local
   ansible.builtin.copy:
     src: "{{ iag_whl_file }}"
     dest: "{{ workingdir.path }}/{{ iag_whl_file | basename }}"
     mode: '0644'
+  when:
+    - iag_whl_file is defined
+    - iag_archive_download_url is not defined
+
+- name: Download IAG archive from repository
+  when:
+    - iag_archive_download_url is defined
+    - iag_whl_file is not defined
+  block:
+    - name: Download IAG archive from repository
+      ansible.builtin.get_url:
+        url: "{{ iag_archive_download_url }}"
+        dest: "{{ workingdir.path }}/"
+        mode: '0644'
+        # Sets the appropriate header based on the repository type:
+        # - For JFrog: Uses the "X-JFrog-Art-Api" header with the API key if "repository_api_key" is defined and "jfrog" is part of the download URL.
+        # - For Nexus: Uses a default header ("Accept: application/octet-stream") since Nexus doesn't support API key authentication.
+        headers: >-
+          {%- if repository_api_key is defined and iag_archive_download_url is search("jfrog") -%}
+            {"X-JFrog-Art-Api": "{{ repository_api_key }}", "Accept": "application/octet-stream"}
+          {%- else -%}
+            {"Accept": "application/octet-stream"}
+          {%- endif -%}
+        url_username: "{{ repository_username | default(omit) }}"
+        url_password: "{{ repository_password | default(omit) }}"
+        validate_certs: true
+      register: download_result
+
+    - name: Set iag_whl_file destination from download
+      ansible.builtin.set_fact:
+        iag_whl_file: "{{ download_result.dest }}"
 
 - name: Upgrade IAG
   ansible.builtin.pip:

roles/gateway/templates/ansible.cfg.j2

@@ -1,2 +1,2 @@
 [defaults]
-collections_path={{ iag_install_dir }}/ansible/collections:/usr/share/ansible/collections
+collections_path={{ iag_ansible_collections_path }}:/usr/share/ansible/collections

roles/gateway/vars/2023.1-redhat-8.yml

@@ -25,3 +25,6 @@ base_python_dependencies:
   - wheel==0.42.0
 
 iag_ansible_version: ansible==7.7.0
+
+iag_ansible_collections:
+  - ansible.netcommon:5.1.0

roles/gateway/vars/2023.1-redhat-9.yml

@@ -23,3 +23,6 @@ app_python_dependencies:
   - ansible-pylibssh
 
 iag_ansible_version: ansible==7.7.0
+
+iag_ansible_collections:
+  - ansible.netcommon:5.1.0

roles/gateway/vars/2023.1-rocky-8.yml

@@ -25,3 +25,6 @@ base_python_dependencies:
   - wheel==0.42.0
 
 iag_ansible_version: ansible==7.7.0
+
+iag_ansible_collections:
+  - ansible.netcommon:5.1.0

roles/gateway/vars/2023.1-rocky-9.yml

@@ -23,3 +23,6 @@ app_python_dependencies:
   - ansible-pylibssh
 
 iag_ansible_version: ansible==7.7.0
+
+iag_ansible_collections:
+  - ansible.netcommon:5.1.0

roles/gateway/vars/2023.2-redhat-8.yml

@@ -37,4 +37,4 @@ app_python_dependencies:
   - nornir-utils==0.2.0
   - pygnmi==0.8.9
 
-iag_ansible_version: ansible==7.7.0
+iag_ansible_version: ansible==8.7.0

roles/gateway/vars/2023.2-redhat-9.yml

@@ -32,4 +32,4 @@ app_python_dependencies:
   - nornir-utils==0.2.0
   - pygnmi==0.8.9
 
-iag_ansible_version: ansible==7.7.0
+iag_ansible_version: ansible==8.7.0