Added changes to support and enable installation of SODA on Ubuntu20.04

.github/workflows/ci_pull_request.yml

@@ -5,7 +5,12 @@ name: SODA Installer CI
 # Controls when the action will run. 
 # Triggers the workflow on push or pull request events but only for the main branch
 # Allows you to run this workflow manually from the Actions tab
-on: [pull_request, workflow_dispatch]
+on: 
+  pull_request:
+    branches:
+      - main
+      - ubuntu2004-experimental
+  workflow_dispatch:
 
 jobs:
   # This workflow contains a single job called "build"
@@ -16,7 +21,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.13.x]
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -36,12 +41,12 @@ jobs:
         run: |
             echo "Installed Ansible Version in Runtime: "
             ansible --version
-        
-      - name: Remove latest ansible on Ubuntu 18.04
+
+      - name: Remove latest ansible on Ubuntu 20.04
         run: |
             echo "uninstall with pipx"
             pipx uninstall ansible-core
-        if: ${{ matrix.os == 'ubuntu-18.04' }}
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
 
       # Change the permissions of the ansible installer
       - name: 'Make ansible installer executable and run install ansible on OS: ${{ matrix.os }}'
@@ -62,15 +67,6 @@ jobs:
           GOPATH: "/home/runner/work/installer/"
         if: ${{ always() }}
 
-      - name: 'Make the CI check executable and initiate the testing with SODA repository'
-        run: |
-          sudo chmod +x ./ci/ci_check.sh
-          ./ci/ci_check.sh repository
-        shell: bash
-        env:
-          GOPATH: "/home/runner/work/installer/"
-        if: ${{ always() }}
-
       - name: 'Make the CI check executable and initiate the testing with SODA release'
         run: |
           sudo chmod +x ./ci/ci_check.sh

.github/workflows/ci_push.yml

@@ -5,8 +5,12 @@ name: SODA Installer CI
 # Controls when the action will run. 
 # Triggers the workflow on push or pull request events but only for the main branch
 # Allows you to run this workflow manually from the Actions tab
-on: [push, workflow_dispatch]
-
+on:
+  push:
+    branches:
+      - main
+      - ubuntu2004-experimental
+  workflow_dispatch:
 jobs:
   # This workflow contains a single job called "build"
   build:
@@ -16,7 +20,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.13.x]
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -36,13 +40,13 @@ jobs:
         run: |
             echo "Installed Ansible Version in Runtime: "
             ansible --version
-        
-      - name: Remove latest ansible on Ubuntu 18.04
+
+      - name: Remove latest ansible on Ubuntu 20.04
         run: |
             echo "uninstall with pipx"
             pipx uninstall ansible-core
-        if: ${{ matrix.os == 'ubuntu-18.04'}}
-
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        
       # Change the permissions of the ansible installer
       - name: 'Make ansible installer executable and run install ansible on OS: ${{ matrix.os }}'
         run: |
@@ -62,15 +66,6 @@ jobs:
           GOPATH: "/home/runner/work/installer/"
         if: ${{ always() }}
 
-      - name: 'Make the CI check executable and initiate the testing with SODA repository'
-        run: |
-          sudo chmod +x ./ci/ci_check.sh
-          ./ci/ci_check.sh repository
-        shell: bash
-        env:
-          GOPATH: "/home/runner/work/installer/"
-        if: ${{ always() }}
-
       - name: 'Make the CI check executable and initiate the testing with SODA release'
         run: |
           sudo chmod +x ./ci/ci_check.sh

ansible/group_vars/common.yml

@@ -52,7 +52,12 @@ ansible_os_family: Debian
 # Install as systemd process or standalone application: true / false
 # true: Will install api, dock, controller and etcd processes as a systemd services and enable them. Recommended for production use
 # false: Will install api, dock, controller and etcd as standalone processes. Recommended for development use.
-install_as_systemd: true
+install_as_systemd: false
+
+# Install  etcd as systemd process or standalone application: true / false
+# true: Will install etcd as a systemd service and enable it. Recommended for production use
+# false: Will install etcd as standalone process. Recommended for development use and on Ubuntu 20.04.
+install_etcd_as_systemd: false
 
 # delete all source packages
 source_purge: true

ansible/roles/cleaner/tasks/main.yml

@@ -24,15 +24,15 @@
 # Kill all the systemd processes and disable services
 - name: kill osdslet and osdsdock and osdsapiserver etcd daemon service
   shell: killall osdslet osdsdock osdsapiserver etcd
-  when: install_from != "container" and install_as_systemd == false
+  when: install_from != "container" and install_as_systemd == false and install_etcd_as_systemd == false
   ignore_errors: true
   become: true
   tags: hotpot
 
 # Kill all the systemd processes and disable services
 - name: kill osdslet and osdsdock and osdsapiserver etcd daemon service
   shell: killall osdslet osdsdock osdsapiserver etcd
-  when: install_from != "container" and install_as_systemd == true
+  when: install_from != "container" and install_as_systemd == true and install_etcd_as_systemd == true
   notify:
     - Stop Controller service
     - Disable Controller service
@@ -79,7 +79,7 @@
     state: absent
     force: yes
   ignore_errors: yes
-  when: install_as_systemd == false and ( database_purge is undefined or database_purge == true )
+  when: install_etcd_as_systemd == false and ( database_purge is undefined or database_purge == true )
   tags: hotpot
 
 - name: clean etcd db
@@ -88,7 +88,7 @@
     state: absent
     force: yes
   ignore_errors: yes
-  when: install_as_systemd == false and ( database_purge is undefined or database_purge == true )
+  when: install_etcd_as_systemd == false and ( database_purge is undefined or database_purge == true )
   tags: hotpot
 
 - name: stop all gelato services

ansible/roles/common/tasks/main.yml

@@ -27,7 +27,7 @@
   become: yes
 
 - name: run the equivalent of "apt-get update" as a separate step
-  when: ansible_distribution == "Ubuntu" and (ansible_lsb.major_release|int >=16 and ansible_lsb.major_release|int <=18)
+  when:  ansible_facts['distribution'] == 'Ubuntu' and (ansible_facts['distribution_major_version']|int >=16 and ansible_facts['distribution_major_version']|int <=20)
   apt:
     update_cache: yes
 
@@ -38,9 +38,9 @@
   with_items:
     - make
     - gcc
-    - python-pip
+    - python3-pip
     - open-iscsi
-  when: ansible_distribution == "Ubuntu" and (ansible_lsb.major_release|int >=16 and ansible_lsb.major_release|int <=18)
+  when: ansible_facts['distribution'] == 'Ubuntu' and (ansible_facts['distribution_major_version']|int >=16 and ansible_facts['distribution_major_version']|int <=20)
 
 - name: Install system packages on CentoS or RHEL
   when: (ansible_distribution == "CentOS" or ansible_distribution == "RedHat")
@@ -63,7 +63,7 @@
   with_items:
     - librados-dev
     - librbd-dev
-  when: ansible_distribution == "Ubuntu" and (ansible_lsb.major_release|int >=16 and ansible_lsb.major_release|int <=18)
+  when: ansible_facts['distribution'] == 'Ubuntu' and (ansible_facts['distribution_major_version']|int >=16 and ansible_facts['distribution_major_version']|int <=20)
 
 - name: create opensds work directory if it doesn't exist
   file:

ansible/roles/dashboard-installer/scenarios/container.yml

@@ -33,10 +33,10 @@
       OPENSDS_S3_URL: "http://{{ host_ip }}:8090"
       OPENSDS_S3_HOST: "{{ host_ip }}" 
       OPENSDS_S3_PORT: "8090"
-      SODA_PROMETHEUS_PORT: "{{ prometheus_port }}"
-      SODA_ALERTMANAGER_PORT: "{{ alertmanager_port }}"
-      SODA_GRAFANA_PORT: "{{ grafana_port }}"
-      STORAGE_SERVICE_PLAN_ENABLED: "{{ enable_storage_service_plans }}"
+      SODA_PROMETHEUS_PORT: "{{ prometheus_port|quote }}"
+      SODA_ALERTMANAGER_PORT: "{{ alertmanager_port|quote }}"
+      SODA_GRAFANA_PORT: "{{ grafana_port|quote }}"
+      STORAGE_SERVICE_PLAN_ENABLED: "{{ enable_storage_service_plans|string }}"
       SODA_ALERTMANAGER_URL: "http://{{ host_ip }}:{{ alertmanager_port }}"
   when: gelato_ha != true
 
@@ -56,9 +56,9 @@
       OPENSDS_S3_URL: "http://{{ gelato_ha_s3_ip }}:{{ gelato_ha_s3_port }}"
       OPENSDS_S3_HOST: "{{ gelato_ha_s3_ip }}"
       OPENSDS_S3_PORT: "{{ gelato_ha_s3_port }}"
-      SODA_PROMETHEUS_PORT: "{{ prometheus_port }}"
-      SODA_ALERTMANAGER_PORT: "{{ alertmanager_port }}"
-      SODA_GRAFANA_PORT: "{{ grafana_port }}"
-      STORAGE_SERVICE_PLAN_ENABLED: "{{ enable_storage_service_plans }}"
+      SODA_PROMETHEUS_PORT: "{{ prometheus_port|quote }}"
+      SODA_ALERTMANAGER_PORT: "{{ alertmanager_port|quote}}"
+      SODA_GRAFANA_PORT: "{{ grafana_port|quote}}"
+      STORAGE_SERVICE_PLAN_ENABLED: "{{ enable_storage_service_plans|string }}"
       SODA_ALERTMANAGER_URL: "http://{{ host_ip }}:{{ alertmanager_port }}"      
   when: gelato_ha == true

ansible/roles/delfin-installer/scenarios/rabbitmq.yml

@@ -82,14 +82,11 @@
 
 - name: Install RabbitMQ package
   apt: 
-    name: "{{ item }}"
+    name: ['erlang-base', 'erlang-asn1', 'erlang-crypto', 'erlang-eldap', 'erlang-ftp', 'erlang-inets', 'erlang-mnesia', 'erlang-os-mon', 'erlang-parsetools', 'erlang-public-key', 'erlang-runtime-tools', 'erlang-snmp', 'erlang-ssl', 'erlang-syntax-tools', 'erlang-tftp', 'erlang-tools', 'erlang-xmerl', 'apt-transport-https', 'rabbitmq-server']
     update_cache: yes
     install_recommends: yes
     allow_unauthenticated: yes
     state: present
-  with_items:
-    - apt-transport-https 
-    - rabbitmq-server
   become: yes
   when:
     - rabbitmqservice.stat.exists is undefined or rabbitmqservice.stat.exists == false

ansible/roles/gelato-ha-uninstaller/scenarios/clean_gelato_ha.yml

@@ -32,9 +32,9 @@
   shell: "{{ item }}"
   become_user: "{{ k8s_user }}"
   with_items:
-    - "kubectl patch pvc mongo-0-pv-claim -p '\''{{patch_param }}'\'' --type=merge -n soda-multi-cloud"
-    - "kubectl patch pvc mongo-1-pv-claim -p '\''{{patch_param }}'\'' --type=merge -n soda-multi-cloud"
-    - "kubectl patch pvc mongo-2-pv-claim -p '\''{{patch_param }}'\'' --type=merge -n soda-multi-cloud"
+    - 'kubectl patch pvc mongo-0-pv-claim -p "\""{{patch_param }}"\"" --type=merge -n soda-multi-cloud'
+    - 'kubectl patch pvc mongo-1-pv-claim -p "\""{{patch_param }}"\"" --type=merge -n soda-multi-cloud'
+    - 'kubectl patch pvc mongo-2-pv-claim -p "\""{{patch_param }}"\"" --type=merge -n soda-multi-cloud'
   ignore_errors: yes
 
 - name: Pause for 2 seconds
@@ -92,6 +92,6 @@
   file:
     state: absent
     path: "{{ gelato_work_dir }}/multi-cloud/"
-  ignore_error: yes
+    ignore_error: yes
   become: yes
 

ansible/roles/ha-ip-update/tasks/main.yml

@@ -22,7 +22,6 @@
 
 # Same IP is used for both the services. Only Port is different (predefined)
 - name: replace the gelato_ha_api_ip and gelato_ha_s3_ip in the gelato-ha.yml
-  hosts: controllers
   replace:
     path: "{{ role_path }}/../../group_vars/gelato-ha.yml"
     regexp: "{{ item.regexp }}"

ansible/roles/osdsdb/scenarios/etcd.yml

@@ -47,20 +47,20 @@
   become: true
   args:
     chdir: "{{ etcd_dir }}"
-  when: install_as_systemd == false and service_etcd_status.rc != 0
+  when: install_etcd_as_systemd == false and service_etcd_status.rc != 0
 
 # Enable etcd service as systemd process
 - name: check for etcd service source file existed
   stat:
     path: "{{ etcd_dir }}/soda-etcd.service"
   register: etcd_service_source_exists
-  when: install_as_systemd == true
+  when: install_etcd_as_systemd == true
 
 - name: Check if etcd service source file exists
   debug:
     var: etcd_service_source_exists
     verbosity: 2
-  when: install_as_systemd == true
+  when: install_etcd_as_systemd == true
 
 - name: Create the etcd service file
   copy:
@@ -74,6 +74,8 @@
       Type=notify
       Restart=always
       RestartSec=5s
+      User=root
+      Group=root
       LimitNOFILE=40000
       TimeoutStartSec=0
 
@@ -82,20 +84,20 @@
       [Install]
       WantedBy=multi-user.target
   when:
-    - install_as_systemd == true and ( service_etcd_status.rc != 0 and etcd_service_source_exists.stat.exists is undefined or etcd_service_source_exists.stat.exists == false )  
+    - install_etcd_as_systemd == true and ( service_etcd_status.rc != 0 and etcd_service_source_exists.stat.exists is undefined or etcd_service_source_exists.stat.exists == false )  
   become: true
 
 - name: check for etcd service file existed
   stat:
     path: "/etc/systemd/system/soda-etcd.service"
   register: etcd_service_exists
-  when: install_as_systemd == true
+  when: install_etcd_as_systemd == true
 
 - name: Check if etcd service file exists at systemd
   debug:
     var: etcd_service_exists
     verbosity: 2
-  when: install_as_systemd == true
+  when: install_etcd_as_systemd == true
 
 - name: Copy etcd service file to systemd
   copy:
@@ -107,9 +109,19 @@
     - Reload daemon
     - Start etcd service
   when:
-    - install_as_systemd == true and ( service_etcd_status.rc != 0 and etcd_service_source_exists.stat.exists is undefined or etcd_service_source_exists.stat.exists == false )  
+    - install_etcd_as_systemd == true and ( service_etcd_status.rc != 0 and etcd_service_source_exists.stat.exists is undefined or etcd_service_source_exists.stat.exists == false )  
   become: true
 
+- name: sleep for 5 seconds and wait for etcd to come up
+  wait_for:
+    host: "{{ etcd_host }}"
+    port: "{{ etcd_port }}"
+    state: started
+    delay: 5
+    timeout: 120
+  ignore_errors: true
+  when: ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version']|int >=20
+
 - name: check etcd cluster health
   shell: ./etcdctl --endpoints http://{{ etcd_host }}:{{ etcd_port }} cluster-health
   become: true

ansible/roles/osdsdock/scenarios/cinder_standalone.yml

@@ -18,7 +18,7 @@
     name: "{{ item }}"
     state: present
   with_items:
-    - python-pip
+    - python3-pip
     - lvm2
     - thin-provisioning-tools
     - libffi-dev

ansible/script/keystone.sh

@@ -116,7 +116,7 @@ wait_for_keystone () {
     do
         # get a token to check if keystone is working correctly or not.
         # keystone credentials such as OS_USERNAME must be set before.
-        python ${TOP_DIR}/ministone.py token_issue
+        python3 ${TOP_DIR}/ministone.py token_issue
         if [ "$?" == "0" ]; then
             return
         fi
@@ -188,7 +188,7 @@ install(){
         docker cp "$TOP_DIR/../../conf/keystone.policy.json" opensds-authchecker:/etc/keystone/policy.json
         keystone_credentials
         wait_for_keystone
-        python ${TOP_DIR}/ministone.py endpoint_bulk_update keystone "http://${HOST_IP}/identity"
+        python3 ${TOP_DIR}/ministone.py endpoint_bulk_update keystone "http://${HOST_IP}/identity"
     else
         create_user
         download_code
@@ -227,7 +227,7 @@ config_hotpot() {
         create_user_and_endpoint_for_hotpot
     else
         keystone_credentials
-        python ${TOP_DIR}/ministone.py endpoint_bulk_update "opensds$OPENSDS_VERSION" "http://${HOST_IP}:50040/$OPENSDS_VERSION/%(tenant_id)s"
+        python3 ${TOP_DIR}/ministone.py endpoint_bulk_update "opensds$OPENSDS_VERSION" "http://${HOST_IP}:50040/$OPENSDS_VERSION/%(tenant_id)s"
     fi
 }
 
@@ -237,7 +237,7 @@ config_gelato() {
         create_user_and_endpoint_for_gelato
     else
         keystone_credentials
-        python ${TOP_DIR}/ministone.py endpoint_bulk_update "multicloud$MULTICLOUD_VERSION" "http://${HOST_IP}:8089/v1beta/%(tenant_id)s"
+        python3 ${TOP_DIR}/ministone.py endpoint_bulk_update "multicloud$MULTICLOUD_VERSION" "http://${HOST_IP}:8089/v1beta/%(tenant_id)s"
     fi
 }