test(robot): pull backup from another longhorn

ref: longhorn/longhorn#9646

Signed-off-by: Chris <[email protected]>

e2e/keywords/longhorn.resource

@@ -82,3 +82,11 @@ Delete instance-manager of deployment ${deployment_id} volume
 
 Wait for Longhorn components all running
     wait_for_namespace_pods_running    longhorn-system
+
+Install Longhorn stable version
+    install_longhorn_system    is_stable_version=True
+
+Uninstall Longhorn stable version
+    ${backups_before_uninstall} =     list_all_backups    
+    uninstall_longhorn_system    is_stable_version=True
+    Set Test Variable    ${backups_before_uninstall}

e2e/keywords/volume.resource

@@ -91,6 +91,10 @@ Write data ${data_id} to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     write_volume_random_data   ${volume_name}    2048    ${data_id}
 
+Write data ${data_id} ${size} MB to volume ${volume_id}
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    write_volume_random_data   ${volume_name}    ${size}    ${data_id}
+
 Keep writing data to volume ${volume_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     keep_writing_data    ${volume_name}
@@ -299,6 +303,11 @@ Check volume ${volume_id} data is backup ${backup_id} created in another cluster
     ${backup_data} =    get_backup_data_from_backup_list    ${backups_before_uninstall}    ${backup_id}
     Should Be Equal    ${current_checksum}    ${backup_data}
 
+Create volume ${volume_id} from backup ${backup_id} in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    create_volume   ${volume_name}    fromBackup=${backup_url}
+
 Create DR volume ${volume_id} from backup ${backup_id} in another cluster
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}

e2e/libs/backup/rest.py

@@ -53,8 +53,13 @@ def get(self, backup_id, volume_name):
 
     def get_from_list(self, backup_list, backup_id):
         for backup in backup_list["items"]:
-            if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
-                return backup
+            try:
+                if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
+                    return backup
+            except KeyError as e:
+                logging(f"Missing key in backup metadata: {str(e)} for backup {backup['metadata']['name']}")
+            except Exception as e:
+                logging(f"Unexpected error accessing backup {backup['metadata']['name']}: {str(e)}")
         return None
 
     def get_by_snapshot(self, volume_name, snapshot_name):

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -6,11 +6,11 @@ class longhorn_deploy_keywords:
     def __init__(self):
         self.longhorn = LonghornDeploy()
 
-    def uninstall_longhorn_system(self):
-        self.longhorn.uninstall()
+    def uninstall_longhorn_system(self, is_stable_version=False):
+        self.longhorn.uninstall(is_stable_version)
 
     def check_longhorn_crd_removed(self):
         self.longhorn.check_longhorn_crd_removed()
 
-    def install_longhorn_system(self):
-        self.longhorn.install()
+    def install_longhorn_system(self, is_stable_version=False):
+        self.longhorn.install(is_stable_version)

e2e/libs/longhorn_deploy/base.py

@@ -19,7 +19,7 @@ def install(self):
         return NotImplemented
 
     @abstractmethod
-    def uninstall(self, longhorn_branch=None):
+    def uninstall(self, is_stable_version=False):
         return NotImplemented
 
     def check_longhorn_crd_removed(self):
@@ -29,17 +29,27 @@ def check_longhorn_crd_removed(self):
 
     def check_longhorn_uninstall_pod_log(self):
         logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
-        assert "error" not in logs
-        assert "level=fatal" not in logs
+        assert "level=error" not in logs, f"find string 'level=error' in uninstall log {logs}"
+        assert "level=fatal" not in logs, f"find string 'level=fatal' in uninstall log {logs}"
 
-    def install_longhorn(self):
+    def install_longhorn(self, is_stable_version=False):
         current_path=os.getcwd()
         full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH)
 
+        if is_stable_version is True:
+            cmd = ['bash', '-c', f'IS_INSTALL_STABLE_VERSION=true {full_path}']
+        else:
+            cmd = ['bash', full_path]
+
         try:
-            output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT)
+            output = subprocess.check_output(cmd, timeout=LONGHORN_INSTALL_TIMEOUT)
             logging(output)
         except subprocess.CalledProcessError as e:
-            logging(f"Error: {e.stderr}")
+            logging(f"Command failed with exit code {e.returncode}")
+            logging(f"stdout: {e.output}")
+            logging(f"stderr: {e.stderr}")
+            raise
         except subprocess.TimeoutExpired as e:
             logging(f"Command timed out after {e.timeout} seconds")
+            logging(f"stdout: {e.output}")
+            raise

e2e/libs/longhorn_deploy/longhorn_deploy.py

@@ -14,11 +14,11 @@ def __init__(self):
         elif self._method == "helm":
             self.longhorn = LonghornHelmChart()
 
-    def uninstall(self):
-        return self.longhorn.uninstall()
+    def uninstall(self, is_stable_version=False):
+        return self.longhorn.uninstall(is_stable_version)
 
     def check_longhorn_crd_removed(self):
         return self.longhorn.check_longhorn_crd_removed()
 
-    def install(self):
-        return self.longhorn.install()
+    def install(self, is_stable_version=False):
+        return self.longhorn.install(is_stable_version)

e2e/libs/longhorn_deploy/longhorn_helm_chart.py

@@ -8,7 +8,7 @@
 
 class LonghornHelmChart(Base):
 
-    def uninstall(self):
+    def uninstall(self, is_stable_version=False):
         control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
         control_plane_node = control_plane_nodes[0]
 
@@ -19,5 +19,5 @@ def uninstall(self):
         k8s.delete_namespace(namespace=LONGHORN_NAMESPACE)
         k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
 
-    def install(self):
-        self.install_longhorn()
+    def install(self, is_stable_version=False):
+        self.install_longhorn(is_stable_version)

e2e/libs/longhorn_deploy/longhorn_kubectl.py

@@ -9,8 +9,11 @@
 
 class LonghornKubectl(Base):
 
-    def uninstall(self):
-        longhorn_branch = os.getenv("LONGHORN_REPO_BRANCH")
+    def uninstall(self, is_stable_version=False):
+        env_var = "LONGHORN_STABLE_VERSION" if is_stable_version else "LONGHORN_REPO_BRANCH"
+        longhorn_branch = os.getenv(env_var)
+        if not longhorn_branch:
+           raise ValueError(f"Required environment variable {env_var} is not set")
 
         control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
         control_plane_node = control_plane_nodes[0]
@@ -30,5 +33,5 @@ def uninstall(self):
         assert res, "delete uninstallation components failed"
         k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
 
-    def install(self):
-        self.install_longhorn()
+    def install(self, is_stable_version=False):
+        self.install_longhorn(is_stable_version)

e2e/tests/negative/pull_backup_from_another_longhorn.robot

@@ -0,0 +1,98 @@
+*** Settings ***
+Documentation    Uninstallation Checks
+
+Test Tags    negative
+
+Resource    ../keywords/common.resource
+Resource    ../keywords/setting.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/persistentvolume.resource
+Resource    ../keywords/persistentvolumeclaim.resource
+Resource    ../keywords/workload.resource
+Resource    ../keywords/backup.resource
+Resource    ../keywords/snapshot.resource
+Resource    ../keywords/backupstore.resource
+Resource    ../keywords/longhorn.resource
+Library     ../libs/keywords/setting_keywords.py
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources
+
+*** Variables ***
+${LOOP_COUNT}    1
+${RETRY_COUNT}    300
+${RETRY_INTERVAL}    1
+${DATA_ENGINE}    v1
+
+*** Test Cases ***
+Pull backup created by another Longhorn system
+    [Documentation]    Pull backup created by another Longhorn system
+    ...    1. Install test version of Longhorn.
+    ...    2. Create volume, write data, and take backup.
+    ...    3. Uninstall Longhorn.
+    ...    4. Install test version of Longhorn.
+    ...    5. Restore the backup create in step 2 and verify the data.
+    ...    6. Uninstall Longhorn.
+    ...    7. Install previous version of Longhorn.
+    ...    8. Create volume, write data, and take backup.
+    ...    9. Uninstall Longhorn.
+    ...    10. Install test version of Longhorn.
+    ...    11. Restore the backup create in step 8 and verify the data.        
+    ...
+    ...    Important
+    ...    - This test case need have set environment variable manually first if not run on Jenkins
+    ...       - LONGHORN_INSTALL_METHOD : helm or manifest
+    ...       - LONGHORN_REPO_BRANCH (ex:master)
+    ...       - CUSTOM_LONGHORN_MANAGER_IMAGE (if not using master-head)
+    ...       - CUSTOM_LONGHORN_ENGINE_IMAGE (if not using master-head)
+    ...       - CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE (if not using master-head)
+    ...       - CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE (if not using master-head)
+    ...       - CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE (if not using master-head)
+    ...       - LONGHORN_STABLE_VERSION (ex:v1.6.3)
+    Given Set setting deleting-confirmation-flag to true
+    And Create volume 0 with    dataEngine=${DATA_ENGINE}
+    And Attach volume 0
+    And Wait for volume 0 healthy
+    And Write data 0 300 MB to volume 0
+    When Create backup 0 for volume 0    
+    Then Verify backup list contains no error for volume 0
+    And Verify backup list contains backup 0 of volume 0
+    Then Uninstall Longhorn
+    And Check Longhorn CRD removed
+
+    # Install current version then pull backup and verify data
+    Then Install Longhorn
+    And Set setting deleting-confirmation-flag to true
+    And Set backupstore
+    And Check backup synced from backupstore
+    And Create volume 1 from backup 0 in another cluster
+    And Wait for volume 1 detached
+    And Attach volume 1
+    And Wait for volume 1 healthy
+    Then Check volume 1 data is backup 0 created in another cluster
+    Then Uninstall Longhorn
+    And Check Longhorn CRD removed
+
+    # Install previous version and create backup
+    Then Install Longhorn stable version
+    And Set setting deleting-confirmation-flag to true
+    And Set backupstore
+    And Create volume 2 with    dataEngine=${DATA_ENGINE}
+    And Attach volume 2
+    And Wait for volume 2 healthy
+    And Write data 1 300 MB to volume 2
+    When Create backup 1 for volume 2
+    Then Verify backup list contains no error for volume 2
+    And Verify backup list contains backup 1 of volume 2
+    Then Uninstall Longhorn stable version
+    And Check Longhorn CRD removed
+
+     # Install current version then pull backup and verify data
+    Then Install Longhorn
+    And Set backupstore
+    And Check backup synced from backupstore
+    And Create volume 3 from backup 1 in another cluster
+    And Wait for volume 3 detached
+    And Attach volume 3
+    And Wait for volume 3 healthy
+    Then Check volume 3 data is backup 1 created in another cluster

e2e/utilities/longhorn-install.sh

@@ -13,6 +13,8 @@ source ../pipelines/utilities/longhorn_manifest.sh
 # create and clean tmpdir
 TMPDIR="/tmp/longhorn"
 LONGHORN_NAMESPACE="longhorn-system"
+LONGHORN_REPO_DIR="${TMPDIR}/longhorn"
+LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"}
 mkdir -p ${TMPDIR}
 rm -rf "${TMPDIR}/"
 
@@ -23,19 +25,48 @@ install_longhorn_by_chart(){
   wait_longhorn_status_running
 }
 
+install_longhorn_stable_by_chart(){
+  git clone --single-branch \
+            --branch "${LONGHORN_STABLE_VERSION}" \
+            "${LONGHORN_REPO_URI}" \
+            "${LONGHORN_REPO_DIR}"
+    helm upgrade --install longhorn "${LONGHORN_REPO_DIR}/chart/" --namespace "${LONGHORN_NAMESPACE}"
+    wait_longhorn_status_running
+}
+
+install_longhorn_stable_by_manifest(){
+  LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION}
+  LONGHORN_STABLE_MANIFEST_URL="https://raw.githubusercontent.com/longhorn/longhorn/${LONGHORN_STABLE_VERSION}/deploy/longhorn.yaml"  
+  kubectl apply -f "${LONGHORN_STABLE_MANIFEST_URL}"
+  wait_longhorn_status_running
+}
+
 install_longhorn(){
   create_longhorn_namespace
   install_backupstores
   if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then
-    LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"}
-    LONGHORN_REPO_DIR="${TMPDIR}/longhorn"
     install_longhorn_by_chart
   elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then
     generate_longhorn_yaml_manifest "${TF_VAR_tf_workspace}"
     install_longhorn_by_manifest "${TF_VAR_tf_workspace}/longhorn.yaml"
   fi
   setup_longhorn_ui_nodeport
+}
 
+install_longhorn_stable_version(){
+  create_longhorn_namespace
+  install_backupstores
+  if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then
+    install_longhorn_stable_by_chart
+  elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then
+    install_longhorn_stable_by_manifest
+  fi
+  setup_longhorn_ui_nodeport
 }
 
-install_longhorn
+IS_INSTALL_STABLE_VERSION="${IS_INSTALL_STABLE_VERSION:-false}"
+if [[ "${IS_INSTALL_STABLE_VERSION}" == "true" ]]; then
+  install_longhorn_stable_version
+else
+  install_longhorn
+fi

pipelines/utilities/run_longhorn_e2e_test.sh

@@ -42,6 +42,7 @@ run_longhorn_e2e_test(){
   yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}"
   yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}"
   yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_INSTALL_METHOD", "value": "'${LONGHORN_INSTALL_METHOD}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}"
+  yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_STABLE_VERSION", "value": "'${LONGHORN_STABLE_VERSION}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}"
 
   LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}`
 
@@ -106,6 +107,7 @@ run_longhorn_e2e_test_out_of_cluster(){
              -e CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE="${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}"\
              -e CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE="${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}"\
              -e LONGHORN_INSTALL_METHOD="${LONGHORN_INSTALL_METHOD}"\
+             -e LONGHORN_STABLE_VERSION="${LONGHORN_STABLE_VERSION}"\
              --mount source="vol-${IMAGE_NAME}",target=/tmp \
              "${LONGHORN_TESTS_CUSTOM_IMAGE}" "${ROBOT_COMMAND_ARGS[@]}"
   docker stop "${CONTAINER_NAME}"