diff --git a/test/appframework_az/c3/appframework_azure_suite_test.go b/test/appframework_az/c3/appframework_azure_suite_test.go deleted file mode 100644 index 313c7c4fc..000000000 --- a/test/appframework_az/c3/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurec3appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go deleted file mode 100644 index 97bcbea77..000000000 --- a/test/appframework_az/c3/appframework_azure_test.go +++ /dev/null @@ -1,3259 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApiV3.ClusterMasterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterMasterWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, masterappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterMasterRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterMasterRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) -}) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go deleted file mode 100644 index 503b3cce3..000000000 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ /dev/null @@ -1,3259 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurec3appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDirShc string - var azTestDirIdxc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - - if filePresentOnOperator { - // Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - //uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Azure - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to S3 - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on S3 location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Azure - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Download Technology add-on app from S3 - testcaseEnvInst.Log.Info("Download Technology add-on app from Axure") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Azure") - - // Create directory for file upload to S3 - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app to Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to S3 for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileListIdxc) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to S3 test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - - // Deploy C3 SVA - // Deploy the Cluster Master - testcaseEnvInst.Log.Info("Deploy Cluster Master") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shc.Spec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - // Delete ES app on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Download ES App from Azure - appVersion = "V2" - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 ES app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = esApp - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Azure (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Azure for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Azure", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure, azure_sanity: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - //Delete the file from az - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration,managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("smoke, c3azure, managerappframeworkazurec3, appframeworkazure: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - azTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with ClusterManagerRef - ############# VERIFICATION ################# - * Verify ClusterManagerRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with ClusterManagerRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with ClusterManagerRef - testcaseEnvInst.Log.Info("Deploy Standalone with ClusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with ClusterManagerRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) -}) diff --git a/test/appframework_az/m4/appframework_azure_suite_test.go b/test/appframework_az/m4/appframework_azure_suite_test.go deleted file mode 100644 index 6184e543b..000000000 --- a/test/appframework_az/m4/appframework_azure_suite_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azurem4appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/m4/appframework_azure_test.go b/test/appframework_az/m4/appframework_azure_test.go deleted file mode 100644 index 6bf739a7e..000000000 --- a/test/appframework_az/m4/appframework_azure_test.go +++ /dev/null @@ -1,2685 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4azure, integration, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("smoke, m4azure, masterappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - -}) diff --git a/test/appframework_az/m4/manager_appframework_azure_test.go b/test/appframework_az/m4/manager_appframework_azure_test.go deleted file mode 100644 index 657cb7501..000000000 --- a/test/appframework_az/m4/manager_appframework_azure_test.go +++ /dev/null @@ -1,2684 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azurem4appfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var azTestDirShc string - var azTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var azTestDirShcLocal string - var azTestDirIdxcLocal string - var azTestDirShcCluster string - var azTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to Azure for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to Azure for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Azure - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Azure (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Azure (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - azTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Azure (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster-wide install (cluster scope)", appVersion)) - azTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, azTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameIdxcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, azTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, azTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpecAzure(appSourceVolumeNameShcCluster, testenv.GetAzureEndpoint(ctx), testcaseEnvInst.GetIndexSecretName(), "azure", "blob")} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, azTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Azure (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcLocal, localappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Azure (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShcCluster, clusterappFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Cluster Manager") - azTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Search Head Cluster") - azTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Azure for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Azure for Deployer") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ Upload Disabled App ########### - * Download disabled app from az - * Delete the app from az - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDirIdxc, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, azTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, azTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("m4azure, integration, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload files to Azure for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Manager and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - azTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search head Cluster", appVersion)) - azTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4azure, managerappframeworkazurem4, appframeworkazure, azure_sanity: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from Azure - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Azure for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Azure for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Azure for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Azure for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), "Unable to upload big size to Azure test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("smoke, m4azure, managerappframeworkazurem4, appframeworkazure: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to Azure for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Azure for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirIdxc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Indexer Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Azure for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirShc, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, azTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, azTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - -}) diff --git a/test/appframework_az/s1/appframework_azure_suite_test.go b/test/appframework_az/s1/appframework_azure_suite_test.go deleted file mode 100644 index d3c06c3c4..000000000 --- a/test/appframework_az/s1/appframework_azure_suite_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package azures1appfw - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - AzureDataContainer = os.Getenv("TEST_CONTAINER") - AzureContainer = os.Getenv("INDEXES_CONTAINER") - AzureStorageAccount = os.Getenv("AZURE_STORAGE_ACCOUNT") - AzureAppDirV1 = testenv.AppLocationV1 - AzureAppDirV2 = testenv.AppLocationV2 - AzureAppDirDisabled = testenv.AppLocationDisabledApps - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - ctx := context.TODO() - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "azure" { - // Create a list of apps to upload to Azure - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Azure - containerName := "/test-data/appframework/v1apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Azure after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Azure - containerName = "/test-data/appframework/v2apps/" - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_az/s1/appframework_azure_test.go b/test/appframework_az/s1/appframework_azure_test.go deleted file mode 100644 index 7d23f92d4..000000000 --- a/test/appframework_az/s1/appframework_azure_test.go +++ /dev/null @@ -1,1991 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package azures1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var azTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - // Delete files uploaded to Azure - if !testcaseEnvInst.SkipTeardown { - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframeworkazure, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - appFrameworkSpecMC.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - azTestDir = "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - uploadedApps = nil - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on Azure - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to Azure - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - azTestDir = "azures1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework, azure_sanity: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING UP VERIFICATION ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING DOWN VERIFICATION ####### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on Azure - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to Azure App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for Standalone - azTestDir := "azures1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, integration, appframeworkazures1, appframework: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Azure - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from Azure - testcaseEnvInst.Log.Info("Download ES app from Azure") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to Azure - testcaseEnvInst.Log.Info("Upload ES app on Azure") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload ES app to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, azTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - - // Download ES App from Azure - containerName = "/" + AzureDataContainer + "/" + AzureAppDirV2 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV2, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to S3 for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on Azure for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download bigger amount of apps from Azure for this test") - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure - testcaseEnvInst.Log.Info("Upload bigger amount of apps to Azure for this test") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1azure, smoke, appframeworkazures1, appframework: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Azure - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - - //Delete apps on Azure for new Apps - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to Azure - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on Azure for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from Azure - testcaseEnvInst.Log.Info("Download the extra apps from Azure for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to Azure for first Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileListStandalone1) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Azure for second Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for 2nd Standalone") - azTestDirStandalone2 := "azures1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirStandalone2, appFileListStandalone2) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, azTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone2, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err = testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Azure for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Azure for Standalone") - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload more apps to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from Azure - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ Upload Disabled App ########### - * Download disabled app from Azure - * Delete the app from Azure - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Azure bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - testcaseEnvInst.Log.Info("Download disabled version of apps from Azure for this test") - testenv.DisableAppsOnAzure(ctx, downloadDirV1, appFileName, azTestDir) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from Azure - azFilepath := "/" + AzureContainer + "/" + filepath.Join(azTestDir, appFileName[0]) - azureBlobClient := &testenv.AzureBlobClient{} - err = azureBlobClient.DeleteFileOnAzure(ctx, azFilepath, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Azure test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Azure bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect Azure credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to Azure App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, _ := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"azure_sa_name": []byte(testenv.RandomDNSName(5)), "azure_sa_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on Azure - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Azure", appVersion)) - azureBlobClient := &testenv.AzureBlobClient{} - azureBlobClient.DeleteFilesOnAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to Azure as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from Azure - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to Azure for Standalone - testcaseEnvInst.Log.Info("Upload apps to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Azure for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to Azure for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV2, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to Azure for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Upload apps to Azure - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload apps to Azure test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integration, s1azure, appframeworkazures1, appframework: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from Azure during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from Azure - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - containerName := "/" + AzureDataContainer + "/" + AzureAppDirV1 - err := testenv.DownloadFilesFromAzure(ctx, testenv.GetAzureEndpoint(ctx), testenv.StorageAccountKey, testenv.StorageAccount, downloadDirV1, containerName, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to Azure for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to Azure for Standalone") - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), "Unable to upload big-size app to Azure test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smoke, s1azure, appframeworkazures1, appframework: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Azure for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to Azure for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to Azure for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Monitoring Console", appVersion)) - - azTestDirMC := "azures1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDirMC, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Monitoring Console", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, azTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to Azure for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Azure for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToAzure(ctx, testenv.StorageAccount, testenv.StorageAccountKey, downloadDirV1, azTestDir, appFileList) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Azure test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, azTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - }) - }) -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_suite_test.go b/test/appframework_gcp/c3/appframework_gcs_suite_test.go deleted file mode 100644 index 9aa061bad..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_suite_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package c3gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "c3appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "c3appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "c3appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "c3appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to Gcs - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to Gcs after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from Gcs - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files.") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files.") -}) diff --git a/test/appframework_gcp/c3/appframework_gcs_test.go b/test/appframework_gcp/c3/appframework_gcs_test.go deleted file mode 100644 index a56a97247..000000000 --- a/test/appframework_gcp/c3/appframework_gcs_test.go +++ /dev/null @@ -1,721 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - //"encoding/json" - "fmt" - "path/filepath" - //"strings" - //"time" - - //enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - //splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - //var gcsTestDirShcLocal string - //var gcsTestDirIdxcLocal string - //var gcsTestDirShcCluster string - //var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - testenv.SpecifiedTestTimeout = 5000 - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, appframeworkgcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Bucket for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCS for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, appframeworkgcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCS for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Master and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on GCS - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to GCS for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to GCS for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, masterappframeworkc3gcp, appframeworkgcp, c3_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS - * Create app source with local scope for C3 SVA (Cluster Master and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCS for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, masterappframeworkc3gcp, appframeworkgcp, c3_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to GCS for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCS for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCS for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterMasterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/c3/manager_appframework_test.go b/test/appframework_gcp/c3/manager_appframework_test.go deleted file mode 100644 index 0d1a81952..000000000 --- a/test/appframework_gcp/c3/manager_appframework_test.go +++ /dev/null @@ -1,3410 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package c3gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("c3appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - - var deployment *testenv.Deployment - var gcsTestDirShc string - var gcsTestDirIdxc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var appSourceNameIdxc string - var appSourceNameShc string - var uploadedApps []string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - testenv.SpecifiedTestTimeout = 100000 - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - // Delete files uploaded to Gcs - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, managerappframeworkc3gcp, appframeworkgcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ FINAL VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - XContext("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework and Image Upgrade", func() { - It(" c3gcp, managerappframeworkc3gcpt, appframeworkgcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then upgrade the image and apps", func() { - - //################## SETUP #################### - - // Download License File - downloadDir := "licenseFolder" - switch testenv.ClusterProvider { - case "eks": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from Gcs") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "azure": - licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) - Expect(err).To(Succeed(), "Unable to download license file from Azure") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - case "gcp": - licenseFilePath, err := testenv.DownloadLicenseFromGCPBucket() - Expect(err).To(Succeed(), "Unable to download license file from GCP") - // Create License Config Map - testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) - default: - fmt.Printf("Unable to download license file") - testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) - } - - // Upload V1 apps to Gcs for Monitoring Console - oldImage := "Refer to RELATED_SPLUNK_IMAGE_ENTERPRISE" - newImage := "splunk/splunk:latest" - - lm, err := deployment.DeployLicenseManager(ctx, deployment.GetName()) - cm, err := deployment.DeployClusterManager(ctx, deployment.GetName(), lm.GetName(), "", "") - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: cm.GetName(), - }, - }, - } - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - shcName := fmt.Sprintf("%s-shc", deployment.GetName()) - idxName := fmt.Sprintf("%s-idxc", deployment.GetName()) - shc, err := deployment.DeploySearchHeadCluster(ctx, shcName, cm.GetName(), lm.GetName(), "", mcName) - idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "") - - // Wait for License Manager to be in READY phase - testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Monitoring Console goes to Ready phase - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Verify no SH in disconnected status is present on CM - testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE IMAGE ################ - - // Update LM Image - - testcaseEnvInst.Log.Info("Upgrading the License Manager Image", "Current Image", oldImage, "New Image", newImage) - lm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, lm) - Expect(err).To(Succeed(), "Failed upgrade License Manager image") - - // Update CM image - - testcaseEnvInst.Log.Info("Upgrading the Cluster Manager Image", "Current Image", oldImage, "New Image", newImage) - cm.Spec.Image = newImage - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed upgrade Cluster Manager image") - - // Update MC image - - testcaseEnvInst.Log.Info("Upgrading the Monitoring Console Image", "Current Image", oldImage, "New Image", newImage) - mc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, mc) - Expect(err).To(Succeed(), "Failed upgrade Monitoring Console image") - - // Update SHC image - - testcaseEnvInst.Log.Info("Upgrading the Search Head Cluster Image", "Current Image", oldImage, "New Image", newImage) - shc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed upgrade Search Head Cluster image") - - // // Update IDXC image - - testcaseEnvInst.Log.Info("Upgrading the Indexer Cluster Image", "Current Image", oldImage, "New Image", newImage) - idxc.Spec.Image = newImage - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed upgrade Indexer Cluster image") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for License Manager to be in READY phase - testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - - // // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It(" c3gcp, managerappframeworkc3gcp, appframeworkgcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app source for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ########### INITIAL VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and also on Search Heads and Indexers pods - ############## DOWNGRADE APPS ############### - * Upload V1 apps on Gcs - * Wait for Monitoring Console and C3 pods to be ready - ########### FINAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Upload V2 apps to Gcs for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - // Monitoring Console AppFramework Spec - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### INITIAL VERIFICATIONS ########### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############## DOWNGRADE APPS ############### - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexers", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### FINAL VERIFICATIONS ############# - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps on Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app config and wait for pods to be ready - ########## INITIAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers - ############# SCALING UP ################### - * Scale up indexers and Search Heads - * Wait for C3 to be ready - ########## SCALING UP VERIFICATIONS ######### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on all Search Heads and Indexers pods - ############### SCALING DOWN ################ - * Scale down Indexers and Search Heads - * Wait for C3 to be ready - ######## SCALING DOWN VERIFICATIONS ######### - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - appFileList := testenv.GetAppFileList(appListV1) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATIONS ############ - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - time.Sleep(60 * time.Second) - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############# SCALING UP ################### - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-idxc" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to scale up Indexer Cluster") - - // Ensure Indexer Cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify New Indexer On Cluster Manager - indexerName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), scaledIndexerReplicas-1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Checking for New Indexer %s On Cluster Manager", indexerName)) - Expect(testenv.CheckIndexerOnCM(ctx, deployment, indexerName)).To(Equal(true)) - - // Ingest data on Indexers - for i := 0; i < int(scaledIndexerReplicas); i++ { - podName := fmt.Sprintf(testenv.IndexerPod, deployment.GetName(), i) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //########## SCALING UP VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Verify no pods reset by checking the pod age - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ################ - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scale down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer Cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Indexer Cluster go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchPod = fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), scaledSHReplicas-1) - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######## SCALING DOWN VERIFICATIONS ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It(" c3gcp, managerappframeworkc3gcp, appframeworkgcp, c3_mgr_gcp_sanity: can deploy a C3 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATIONS ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Indexers", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V2 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### Downgrade APPS ################ - * Upload V1 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V2" - appListLocal := appListV2[len(appListV2)/2:] - appListCluster := appListV2[:len(appListV2)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 60) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 60) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: "V2", CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - time.Sleep(2 * time.Minute) // FIXME adding sleep to see if verification succeedes - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V1 apps to Gcs (to be used for local install) - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA instance with App Framework enabled and install above 200MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create App Source for C3 SVA (Cluster Manager and Deployer) - * Add more apps than usual on Gcs for this test - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############### VERIFICATIONS ############### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Creating a bigger list of apps to be installed than the default one - appList := []string{"splunk_app_db_connect", "splunk_app_aws", "Splunk_TA_microsoft-cloudservices", "Splunk_ML_Toolkit", "Splunk_Security_Essentials"} - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from Gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from Gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Create consolidated list of app files - appList = append(appListV1, appList...) - appFileList = testenv.GetAppFileList(appList) - - // Upload app to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload app to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Create Single Site Indexer Cluster and Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############### VERIFICATIONS ############### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled for manual update", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to Gcs - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to Gcs for Indexer Cluster - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with App framework") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs for C3 - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Uploading %s apps to Gcs", appVersion)) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############## UPGRADE VERIFICATIONS ############ - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA and have apps installed and updated locally on Cluster Manager and Deployer for manual polling", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs - * Create app source with local scope for C3 SVA (Cluster Manager and Deployer) - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ############# INITIAL VERIFICATION ########## - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########### UPGRADE VERIFICATIONS ########### - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy C3 CRD - indexerReplicas := 3 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############## INITIAL VERIFICATION ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to Gcs - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########### UPGRADE VERIFICATIONS ########### - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on Gcs - * Wait for all C3 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "c3appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to Gcs (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "c3appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to Gcs (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "c3appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to Gcs test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on Gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on Gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to Gcs (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to Gcs (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework - ############## VERIFICATIONS ################ - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, _, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on Deployer - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from Gcs during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to Gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "c3appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big-size app to Gcs for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Cluster Manager") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to Gcs for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to Gcs for Deployer - testcaseEnvInst.Log.Info("Upload more apps to Gcs for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to Gcs test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - // Vivek need testing - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from Gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //######### INITIAL VERIFICATIONS ############# - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on Gcs bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from Gcs - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on Gcs test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from Gcs bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download all apps from Gcs - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to Gcs for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("c3, integration, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to Gcs for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search head Cluster", appVersion)) - gcsTestDirShc := "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to Gcs for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy C3 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Download big size apps from Gcs - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to Gcs for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Indexer Cluster") - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to Gcs for Search Head Cluster") - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to Gcs test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - shReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### VERIFICATIONS ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { - It(" c3gcp, managerappframeworkc3gcp, appframeworkgcp, c3_mgr_gcp_sanity: can deploy a C3 SVA with App Framework enabled and check isDeploymentInProgressFlag for CM and SHC CR's", func() { - - /* - Test Steps - ################## SETUP ################## - * Upload V1 apps to Gcs for Indexer Cluster and Search Head Cluster - * Prepare and deploy C3 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP #################### - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Indexer Cluster", appVersion)) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to Gcs for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to Gcs for Search Head Cluster", appVersion)) - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to Gcs test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for C3 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy C3 CRD - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster with Search Head Cluster") - indexerReplicas := 3 - cm, _, shc, err := deployment.DeploySingleSiteClusterWithGivenAppFrameworkSpec(ctx, deployment.GetName(), indexerReplicas, true, appFrameworkSpecIdxc, appFrameworkSpecShc, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Manager CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3: can deploy a C3 SVA and a Standalone, then add that Standalone as a Search Head to the cluster", func() { - - /* Test Steps - ################## SETUP ################### - * Deploy C3 CRD - * Deploy Standalone with clusterMasterRef - ############# VERIFICATION ################# - * Verify clusterMasterRef is present in Standalone's server.conf file - */ - //################## SETUP #################### - // Deploy C3 CRD - indexerReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - err := deployment.DeploySingleSiteCluster(ctx, deployment.GetName(), indexerReplicas, false, "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Create spec with clusterMasterRef for Standalone - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - } - - // Deploy Standalone with clusterMasterRef - testcaseEnvInst.Log.Info("Deploy Standalone with clusterManagerRef") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with clusterMasterRef") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Ensure that the Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############# VERIFICATION ################# - // Verify Standalone is configured as a Search Head for the Cluster Manager - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - Expect(testenv.CheckSearchHeadOnCM(ctx, deployment, standalonePodName)).To(Equal(true)) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Clustered deployment (C3 - clustered indexer, search head cluster)", func() { - It("integration, c3, managerappframeworkc3gcptodo, appframework: can deploy a C3 SVA and have ES app installed on Search Head Cluster", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to Gcs - * Upload TA add-on app to location for Indexer cluster - * Create App Source with 'ScopeClusterWithPreConfig' scope for C3 SVA - * Prepare and deploy C3 CRD with app framework and wait for pods to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Deployer and on Search Heads - * Verify TA add-on app is installed on indexers - ################## UPGRADE VERIFICATION ############# - * Update ES app on Gcs location - * Verify updated ES app is installed on Deployer and on Search Heads - */ - - //################## SETUP #################### - // Download ES app from Gcs - appVersion := "V1" - testcaseEnvInst.Log.Info("Download ES app from Gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Download Technology add-on app from Gcs - testcaseEnvInst.Log.Info("Download Technology add-on app from Gcs") - taApp := []string{"Splunk_TA_ForIndexers"} - appFileListIdxc := testenv.GetAppFileList(taApp) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileListIdxc) - Expect(err).To(Succeed(), "Unable to download ES app file from Gcs") - - // Create directory for file upload to Gcs - gcsTestDirShc = "c3appfw-shc-" + testenv.RandomDNSName(4) - gcsTestDirIdxc = "c3appfw-idxc-" + testenv.RandomDNSName(4) - - // Upload ES app to Gcs - testcaseEnvInst.Log.Info("Upload ES app to Gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to Gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload Technology add-on apps to Gcs for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Technology add-on app to Gcs for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileListIdxc, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Technology add-on app to Gcs test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for SHC - appSourceNameShc = "appframework-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopePremiumApps, appSourceNameShc, gcsTestDirShc, 180) - appFrameworkSpecShc.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - - // Create App framework Spec for Indexer Cluster - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 180) - - // Deploy C3 SVA - // Deploy the Cluster Manager - testcaseEnvInst.Log.Info("Deploy Cluster Manager") - cmSpec := enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecIdxc, - } - cm, err := deployment.DeployClusterManagerWithGivenSpec(ctx, deployment.GetName(), cmSpec) - Expect(err).To(Succeed(), "Unable to deploy Cluster Manager") - - // Deploy the Indexer Cluster - testcaseEnvInst.Log.Info("Deploy Single Site Indexer Cluster") - indexerReplicas := 3 - _, err = deployment.DeployIndexerCluster(ctx, deployment.GetName()+"-idxc", "", indexerReplicas, deployment.GetName(), "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster") - - // Deploy the Search Head Cluster - testcaseEnvInst.Log.Info("Deploy Search Head Cluster") - shSpec := enterpriseApi.SearchHeadClusterSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: deployment.GetName(), - }, - }, - Replicas: 3, - AppFrameworkConfig: appFrameworkSpecShc, - } - shc, err := deployment.DeploySearchHeadClusterWithGivenSpec(ctx, deployment.GetName()+"-shc", shSpec) - Expect(err).To(Succeed(), "Unable to deploy Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //######### INITIAL VERIFICATIONS ############# - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), int(shSpec.Replicas), false, 1) - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList, CrReplicas: int(shSpec.Replicas), CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - idxcPodNames := testenv.GeneratePodNameSlice(testenv.IndexerPod, deployment.GetName(), indexerReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: taApp, CrAppFileList: appFileListIdxc, CrReplicas: indexerReplicas, CrClusterPods: idxcPodNames} - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // //############### UPGRADE APPS ################ - // // Download ES App from Gcs - // appVersion = "V2" - // testcaseEnvInst.Log.Info("Download updated ES app from Gcs") - // err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - // Expect(err).To(Succeed(), "Unable to download ES app") - - // // Upload V2 ES app to Gcs for Search Head Cluster - // testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s ES app to Gcs for Search Head Cluster", appVersion)) - // uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - // Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s ES app to Gcs test directory for Search Head Cluster", appVersion)) - // uploadedApps = append(uploadedApps, uploadedFiles...) - - // // Check for changes in App phase to determine if next poll has been triggered - // testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // // Ensure that the Cluster Manager goes to Ready phase - // testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Indexers go to Ready phase - // testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // // Ensure Search Head Cluster go to Ready phase - // testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // // Verify RF SF is met - // testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // // Get Pod age to check for pod resets later - // splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // //############ FINAL VERIFICATIONS ############ - - // shcAppSourceInfo.CrAppVersion = appVersion - // shcAppSourceInfo.CrAppList = esApp - // shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - // allAppSourceInfo = []testenv.AppSourceInfo{shcAppSourceInfo} - // testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_suite_test.go b/test/appframework_gcp/m4/appframework_gcs_suite_test.go deleted file mode 100644 index 8f4a28249..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_suite_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package m4gcpappfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "m4appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGcsBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "m4appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "m4appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "m4appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - testenvInstance.Log.Info("logging download details", "bucket", testDataGcsBucket, "gcsAppDirV1", gcsAppDirV1, "downloadDirV1", downloadDirV1, "appFileList", appFileList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/m4/appframework_gcs_test.go b/test/appframework_gcp/m4/appframework_gcs_test.go deleted file mode 100644 index d5e2f16df..000000000 --- a/test/appframework_gcp/m4/appframework_gcs_test.go +++ /dev/null @@ -1,2703 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var gcsTestDirShc string - var gcsTestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var gcsTestDirShcLocal string - var gcsTestDirIdxcLocal string - var gcsTestDirShcCluster string - var gcsTestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, appframeworkgcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Master and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Master and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Master and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - gcsTestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - gcsTestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster master with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, gcsTestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, gcsTestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, gcsTestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "gcs", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, gcsTestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - ClusterMasterBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, ClusterMasterBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Master - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, appframeworkgcp, m4_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Master and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Master and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Master and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - gcsTestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Master - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - gcsTestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - gcsTestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Master - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from gcs - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - gcsFilepath := filepath.Join(gcsTestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Master and Deployer - * Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Master and Search Head Cluster - * Change status to 'ON' in config map for Cluster Master and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Master and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Master CR with latest config - cm = &enterpriseApiV3.ClusterMaster{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Master") - - // Set AppsRepoPollInterval for Cluster Master to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Master to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Master") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Master") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterMaster"] = strings.Replace(config.Data["ClusterMaster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Master") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Master and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Master and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterMaster"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, gcsTestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, gcsTestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, masterappframeworkm4, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - gcsTestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - gcsTestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Master and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, masterappframeworkm4, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Master - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterMasterPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, masterappframeworkm4gcp, appframeworkgcp, m4_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, gcsTestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, gcsTestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, gcsTestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterMasterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Master goes to Ready phase - testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/m4/manager_appframework_test.go b/test/appframework_gcp/m4/manager_appframework_test.go deleted file mode 100644 index 0e5c7e75b..000000000 --- a/test/appframework_gcp/m4/manager_appframework_test.go +++ /dev/null @@ -1,2702 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package m4gcpappfw - -import ( - "context" - "encoding/json" - "fmt" - "path/filepath" - "strings" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("m4appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var uploadedApps []string - var appSourceNameIdxc string - var appSourceNameShc string - var s3TestDirShc string - var s3TestDirIdxc string - var appSourceVolumeNameIdxc string - var appSourceVolumeNameShc string - var s3TestDirShcLocal string - var s3TestDirIdxcLocal string - var s3TestDirShcCluster string - var s3TestDirIdxcCluster string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - appSourceVolumeNameIdxc = "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc = "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCP - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, managerappframeworkm4gcp, appframeworkgcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and upgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############# UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for Monitoring Console and M4 pod to be ready - ########## UPGRADE VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - _ = testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify MC is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ########## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled, install apps and downgrade them", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V2 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload V2 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - ############ DOWNGRADE APPS ############### - * Downgrade apps in app sources - * Wait for Monitoring Console and M4 to be ready - ########## DOWNGRADE VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and downgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V2 version of apps to GCP for Monitoring Console - appVersion := "V2" - appFileList := testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console instance") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V2 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############# DOWNGRADE APPS ################ - // Delete V2 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V1 apps to GCP for Indexer Cluster - appVersion = "V1" - appFileList = testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATIONS ######## - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV1 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV1 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled, install apps, scale up clusters, install apps on new pods, scale down", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for M4 - * Create app source for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app config and wait for pods to be ready - ########### INITIAL VERIFICATIONS ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on Monitoring Console and also on Search Heads and Indexers pods - ############### SCALING UP ################ - * Scale up Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING UP VERIFICATIONS ######## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are copied and installed on new Search Heads and Indexers pods - ############### SCALING DOWN ############## - * Scale down Indexers and Search Head Cluster - * Wait for Monitoring Console and M4 to be ready - ######### SCALING DOWN VERIFICATIONS ###### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is sucessful - * Verify apps are still copied and installed on all Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // Ingest data on Indexers - for i := 1; i <= siteCount; i++ { - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), i, 0) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - } - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########### INITIAL VERIFICATIONS ######### - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################ - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale up Search Head Cluster - defaultSHReplicas := shc.Spec.Replicas - scaledSHReplicas := defaultSHReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale up Search Head Cluster") - - // Ensure Search Head Cluster scales up and go to ScalingUp phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp) - - // Get instance of current Indexer CR with latest config - idxcName := deployment.GetName() + "-" + "site1" - idxc := &enterpriseApi.IndexerCluster{} - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas := idxc.Spec.Replicas - scaledIndexerReplicas := defaultIndexerReplicas + 1 - testcaseEnvInst.Log.Info("Scale up Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale Up Indexer Cluster") - - // Ensure Indexer cluster scales up and go to ScalingUp phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingUp, idxcName) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ingest data on new Indexers - podName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - logFile := fmt.Sprintf("test-log-%s.log", testenv.RandomDNSName(3)) - testenv.CreateMockLogfile(logFile, 2000) - testenv.IngestFileViaMonitor(ctx, logFile, "main", podName, deployment) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data on newly added indexer - searchPod := fmt.Sprintf(testenv.SearchHeadPod, deployment.GetName(), 0) - indexerName := fmt.Sprintf(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, 1) - searchString := fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err := testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse := strings.Split(searchResultsResp, "\n")[0] - var searchResults map[string]interface{} - jsonErr := json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine := searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname := strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING UP VERIFICATIONS ######## - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // Listing the Search Head cluster pods to exclude them from the 'no pod reset' test as they are expected to be reset after scaling - shcPodNames = []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - shcPodNames = append(shcPodNames, testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1)...) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - - //############### SCALING DOWN ############## - // Get instance of current Search Head Cluster CR with latest config - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - - Expect(err).To(Succeed(), "Failed to get instance of Search Head Cluster") - - // Scale down Search Head Cluster - defaultSHReplicas = shc.Spec.Replicas - scaledSHReplicas = defaultSHReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Search Head Cluster", "Current Replicas", defaultSHReplicas, "New Replicas", scaledSHReplicas) - - // Update Replicas of Search Head Cluster - shc.Spec.Replicas = int32(scaledSHReplicas) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to scale down Search Head Cluster") - - // Ensure Search Head Cluster scales down and go to ScalingDown phase - testenv.VerifySearchHeadClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown) - - // Get instance of current Indexer CR with latest config - err = deployment.GetInstance(ctx, idxcName, idxc) - Expect(err).To(Succeed(), "Failed to get instance of Indexer Cluster") - defaultIndexerReplicas = idxc.Spec.Replicas - scaledIndexerReplicas = defaultIndexerReplicas - 1 - testcaseEnvInst.Log.Info("Scaling down Indexer Cluster", "Current Replicas", defaultIndexerReplicas, "New Replicas", scaledIndexerReplicas) - - // Update Replicas of Indexer Cluster - idxc.Spec.Replicas = int32(scaledIndexerReplicas) - err = deployment.UpdateCR(ctx, idxc) - Expect(err).To(Succeed(), "Failed to Scale down Indexer Cluster") - - // Ensure Indexer cluster scales down and go to ScalingDown phase - testenv.VerifyIndexerClusterPhase(ctx, deployment, testcaseEnvInst, enterpriseApi.PhaseScalingDown, idxcName) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure Indexer cluster go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Search for data from removed indexer - searchString = fmt.Sprintf("index=%s host=%s | stats count by host", "main", indexerName) - searchResultsResp, err = testenv.PerformSearchSync(ctx, searchPod, searchString, deployment) - Expect(err).To(Succeed(), "Failed to execute search '%s' on pod %s", searchPod, searchString) - - // Verify result. - searchResponse = strings.Split(searchResultsResp, "\n")[0] - jsonErr = json.Unmarshal([]byte(searchResponse), &searchResults) - Expect(jsonErr).To(Succeed(), "Failed to unmarshal JSON Search Results from response '%s'", searchResultsResp) - - testcaseEnvInst.Log.Info("Search results :", "searchResults", searchResults["result"]) - Expect(searchResults["result"]).ShouldNot(BeNil(), "No results in search response '%s' on pod %s", searchResults, searchPod) - - resultLine = searchResults["result"].(map[string]interface{}) - testcaseEnvInst.Log.Info("Sync Search results host count:", "count", resultLine["count"].(string), "host", resultLine["host"].(string)) - testHostname = strings.Compare(resultLine["host"].(string), indexerName) - Expect(testHostname).To(Equal(0), "Incorrect search result hostname. Expect: %s Got: %s", indexerName, resultLine["host"].(string)) - - //######### SCALING DOWN VERIFICATIONS ###### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, shcPodNames) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA and have apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - indexersPerSite := 1 - shReplicas := 3 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled for manual poll", func() { - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to GCP - * Create app source with manaul poll for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify apps are installed locally on Cluster Manager and Deployer - */ - - // ################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - volumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, volumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 0) - - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multi Site Indexer Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // ############### UPGRADE APPS ################ - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Monitoring Console - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - allPodNames := append(idxcPodNames, shcPodNames...) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPodNames, appListV1, true, "enabled", false, true) - - // ############ ENABLE MANUAL POLL ############ - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["MonitoringConsole"] = strings.Replace(config.Data["MonitoringConsole"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - time.Sleep(2 * time.Minute) - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll disabled after the check ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - // ############ VERIFY APPS UPDATED TO V2 ############# - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA and have apps installed and updated locally on Cluster Manager and Deployer via manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA (Cluster Manager and Deployer) - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the poll is triggered - ########## UPGRADE VERIFICATIONS ############ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are copied, installed and upgraded on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer cluster configured as multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATIONS ############ - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, managerappframeworkm4gcptodo, appframework: can deploy a m4 SVA with apps installed locally on Cluster Manager and Deployer, cluster-wide on Peers and Search Heads, then upgrade them via a manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Split Applist into clusterlist and local list - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster for local and cluster scope - * Create app sources for Cluster Manager and Deployer with local and cluster scope - * Prepare and deploy m4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Monitoring Console and on Search Heads and Indexers pods - ############### UPGRADE APPS ################ - * Upload V2 apps on GCP - * Wait for all m4 pods to be ready - ############ FINAL VERIFICATIONS ############ - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V2 apps are copied and upgraded on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Split Applist into 2 lists for local and cluster install - appVersion := "V1" - appListLocal := appListV1[len(appListV1)/2:] - appListCluster := appListV1[:len(appListV1)/2] - - // Upload appListLocal list of apps to GCP (to be used for local install) for Idxc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirIdxcLocal = "m4appfw-" + testenv.RandomDNSName(4) - localappFileList := testenv.GetAppFileList(appListLocal) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListLocal list of apps to GCP (to be used for local install) for Shc - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - s3TestDirShcLocal = "m4appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (local scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirIdxcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - clusterappFileList := testenv.GetAppFileList(appListCluster) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of apps to GCP (to be used for cluster-wide install) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster-wide install (cluster scope)", appVersion)) - s3TestDirShcCluster = "m4appfw-cluster-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps (cluster scope) to GCP test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameLocalIdxc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameLocalShc := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameClusterIdxc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameClusterShc := "appframework-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcLocal := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcLocal := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appSourceVolumeNameIdxcCluster := "appframework-test-volume-idxc-cluster-" + testenv.RandomDNSName(3) - appSourceVolumeNameShcCluster := "appframework-test-volume-shc-cluster-" + testenv.RandomDNSName(3) - - // Create App framework Spec for Cluster manager with scope local and append cluster scope - - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalIdxc, s3TestDirIdxcLocal, 0) - volumeSpecCluster := []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameIdxcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - appFrameworkSpecIdxc.VolList = append(appFrameworkSpecIdxc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec := enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameIdxcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster := []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterIdxc, s3TestDirIdxcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecIdxc.AppSources = append(appFrameworkSpecIdxc.AppSources, appSourceSpecCluster...) - - // Create App framework Spec for Search head cluster with scope local and append cluster scope - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShcLocal, enterpriseApi.ScopeLocal, appSourceNameLocalShc, s3TestDirShcLocal, 0) - volumeSpecCluster = []enterpriseApi.VolumeSpec{testenv.GenerateIndexVolumeSpec(appSourceVolumeNameShcCluster, testenv.GetGCPEndpoint(), testcaseEnvInst.GetIndexSecretName(), "aws", "s3", testenv.GetDefaultGCPRegion())} - - appFrameworkSpecShc.VolList = append(appFrameworkSpecShc.VolList, volumeSpecCluster...) - appSourceClusterDefaultSpec = enterpriseApi.AppSourceDefaultSpec{ - VolName: appSourceVolumeNameShcCluster, - Scope: enterpriseApi.ScopeCluster, - } - appSourceSpecCluster = []enterpriseApi.AppSourceSpec{testenv.GenerateAppSourceSpec(appSourceNameClusterShc, s3TestDirShcCluster, appSourceClusterDefaultSpec)} - appFrameworkSpecShc.AppSources = append(appFrameworkSpecShc.AppSources, appSourceSpecCluster...) - - // Create Single site Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - testcaseEnvInst.Log.Info("Deploy Single site Indexer Cluster with both Local and Cluster scope for apps installation") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Single Site Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameLocalIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcLocal, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - cmAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameClusterIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxcCluster, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoLocal := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameLocalShc, CrAppSourceVolumeName: appSourceVolumeNameShcLocal, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListLocal, CrAppFileList: localappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - shcAppSourceInfoCluster := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameClusterShc, CrAppSourceVolumeName: appSourceVolumeNameShcCluster, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListCluster, CrAppFileList: clusterappFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - clusterManagerBundleHash := testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - //############### UPGRADE APPS ################ - // Delete apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Redefine app lists as LDAP app isn't in V1 apps - appListLocal = appListV1[len(appListV1)/2:] - appListCluster = appListV1[:len(appListV1)/2] - - // Upload appListLocal list of V2 apps to GCP (to be used for local install) - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for local install (local scope)", appVersion)) - localappFileList = testenv.GetAppFileList(appListLocal) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcLocal, localappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for local install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload appListCluster list of V2 apps to GCP (to be used for cluster-wide install) - clusterappFileList = testenv.GetAppFileList(appListCluster) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for cluster install (cluster scope)", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShcCluster, clusterappFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for cluster-wide install", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // ############ ENABLE MANUAL POLL ############ - - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameClusterIdxc, clusterappFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // ########## Verify Manual Poll config map disabled after the poll is triggered ################# - - // Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info("Verify config map set back to off after poll trigger for app", "version", appVersion) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //########## UPGRADE VERIFICATION ############# - cmAppSourceInfoLocal.CrAppVersion = appVersion - cmAppSourceInfoLocal.CrAppList = appListLocal - cmAppSourceInfoLocal.CrAppFileList = localappFileList - cmAppSourceInfoCluster.CrAppVersion = appVersion - cmAppSourceInfoCluster.CrAppList = appListCluster - cmAppSourceInfoCluster.CrAppFileList = clusterappFileList - shcAppSourceInfoLocal.CrAppVersion = appVersion - shcAppSourceInfoLocal.CrAppList = appListLocal - shcAppSourceInfoLocal.CrAppFileList = localappFileList - shcAppSourceInfoCluster.CrAppVersion = appVersion - shcAppSourceInfoCluster.CrAppList = appListCluster - shcAppSourceInfoCluster.CrAppFileList = clusterappFileList - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfoLocal, cmAppSourceInfoCluster, shcAppSourceInfoLocal, shcAppSourceInfoCluster} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, clusterManagerBundleHash) - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4, add new apps to app source while install is in progress and have all apps installed locally on Cluster Manager and Deployer", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework - * Verify app installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Verify all apps are installed on Cluster Manager - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Cluster Manager", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), cmPod, appList, true, "enabled", false, false) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - time.Sleep(60 * time.Second) - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Deployer - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Deployer", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) - }) - }) - - Context("Single Site Indexer Cluster with Search Head Cluster (M4) and App Framework", func() { - It(" m4gcp, managerappframeworkm4gcp, appframeworkgcp, m4_mgr_gcp_sanity: can deploy a M4, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Create app sources for Cluster Manager and Deployer - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Cluster Manager and Deployer - * Upload more apps from GCP during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Cluster Manager and Deployer - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Monitoring Console", appVersion)) - s3TestDirMC := "m4appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Monitoring Console %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Prepare Monitoring Console spec with its own app source - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, s3TestDirMC, 60) - - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Download all test apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCP for Cluster Manager - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Cluster Manager") - s3TestDirIdxc = "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big-size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big-size app to GCP for Search Head Cluster") - s3TestDirShc = "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, mcName, "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCP for Cluster Manager - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCP for Cluster Manager") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Cluster Manager") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload more apps to GCP for Deployer - testcaseEnvInst.Log.Info("Upload more apps to GCP for Deployer") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCP test directory for Deployer") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify all apps are installed on indexers - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), indexersPerSite, true, siteCount) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) - - // Verify all apps are installed on Search Heads - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Search Heads", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), shcPodNames, appList, true, "enabled", false, true) - - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app install is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App installation is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled, install an app, then disable it by using a disabled version of the app and then remove it from app source", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ########## INITIAL VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - * Disable the app - * Delete the app from s3 - * Check for repo state in App Deployment Info - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATIONS ########## - idxcPodNames := testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames := testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCP bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, s3TestDirIdxc) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileName) - - // Ensure Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Wait for App state to update after config file change - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, idxcPodNames, testcaseEnvInst.GetName(), appName, "disabled", true) - - // Delete the file from GCP - s3Filepath := filepath.Join(s3TestDirIdxc, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGcsBucket, s3Filepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCP test directory", appFileName)) - - // Verify repo state is set to 2 (i.e app deleted from GCP bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind, appSourceNameIdxc, 2, appFileName[0]) - }) - }) - - Context("Multi Site Indexer Cluster with Search Head Cluster (M4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA, install apps via manual polling, switch to periodic polling, verify apps are not updated before the end of AppsRepoPollInterval, then updated after", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCP - * Create app source with local scope for M4 SVA, AppsRepoPollInterval=0 to set apps polling as manual - * Prepare and deploy M4 CRD with app framework and wait for pods to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify apps are installed locally on Cluster Manager and Deployer - * Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - ######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - * Set AppsRepoPollInterval to 180 seconds for Cluster Manager and Search Head Cluster - * Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - ############### UPGRADE APPS ################ - * Upgrade apps in app sources - * Wait for pods to be ready - ############ UPGRADE VERIFICATION ########## - * Verify apps are not updated before the end of AppsRepoPollInterval duration - * Verify apps are updated after the end of AppsRepoPollInterval duration - */ - - //################## SETUP #################### - // Upload V1 apps to GCP for Indexer Cluster - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 0) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 0) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - - // Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'OFF' in config map for Cluster Manager and Search Head Cluster") - config, _ := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: off") && strings.Contains(config.Data["SearchHeadCluster"], "status: off")).To(Equal(true), "Config map update not complete") - - //######### SWITCH FROM MANUAL TO PERIODIC POLLING ############ - // Get instance of current Cluster Manager CR with latest config - cm = &enterpriseApi.ClusterManager{} - err = deployment.GetInstance(ctx, deployment.GetName(), cm) - Expect(err).To(Succeed(), "Failed to edit Cluster Manager") - - // Set AppsRepoPollInterval for Cluster Manager to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Cluster Manager to 180 seconds") - cm.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, cm) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Cluster Manager") - - // Get instance of current Search Head Cluster CR with latest config - shc = &enterpriseApi.SearchHeadCluster{} - err = deployment.GetInstance(ctx, deployment.GetName()+"-shc", shc) - Expect(err).To(Succeed(), "Failed to edit Search Head Cluster") - - // Set AppsRepoPollInterval for Search Head Cluster to 180 seconds - testcaseEnvInst.Log.Info("Set AppsRepoPollInterval for Search Head Cluster to 180 seconds") - shc.Spec.AppFrameworkConfig.AppsRepoPollInterval = int64(180) - err = deployment.UpdateCR(ctx, shc) - Expect(err).To(Succeed(), "Failed to change AppsRepoPollInterval value for Search Head Cluster") - - // Change status to 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Cluster Manager") - config, err = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map") - - config.Data["ClusterManager"] = strings.Replace(config.Data["ClusterManager"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Cluster Manager") - - testcaseEnvInst.Log.Info("Change status to 'ON' in config map for Search Head Cluster") - config.Data["SearchHeadCluster"] = strings.Replace(config.Data["SearchHeadCluster"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map for Search Head Cluster") - - // Wait 5 seconds to be sure reconcile caused by CR update and config map update are done - testcaseEnvInst.Log.Info("Wait 5 seconds to be sure reconcile caused by CR update and config map update are done") - time.Sleep(5 * time.Second) - - // Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster - testcaseEnvInst.Log.Info("Verify status is 'ON' in config map for Cluster Manager and Search Head Cluster") - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["ClusterManager"], "status: on") && strings.Contains(config.Data["SearchHeadCluster"], "status: on")).To(Equal(true), "Config map update not complete") - - //############### UPGRADE APPS ################ - // Delete V1 apps on GCP - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCP", appVersion)) - testenv.DeleteFilesOnGCP(testGcsBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## UPGRADE VERIFICATIONS ############ - testcaseEnvInst.Log.Info("Verify apps are not updated before the end of AppsRepoPollInterval duration") - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Wait for the end of AppsRepoPollInterval duration - testcaseEnvInst.Log.Info("Wait for the end of AppsRepoPollInterval duration") - time.Sleep(2 * time.Minute) - - testcaseEnvInst.Log.Info("Verify apps are updated after the end of AppsRepoPollInterval duration") - appVersion = "V2" - cmAppSourceInfo.CrAppVersion = appVersion - cmAppSourceInfo.CrAppList = appListV2 - cmAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - shcAppSourceInfo.CrAppVersion = appVersion - shcAppSourceInfo.CrAppList = appListV2 - shcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * While app download is in progress, restart the operator - * While app download is completed, upload new versions of the apps - ######### VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - ######### UPGRADE VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify V1 apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download all apps from GCP - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeLocal, appSourceNameIdxc, s3TestDirIdxc, 120) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeLocal, appSourceNameShc, s3TestDirShc, 120) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is in progress on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCP for Indexer Cluster - appVersion = "V2" - appListV2 := []string{appListV2[0]} - appFileList = testenv.GetAppFileList(appListV2) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V2 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - appFileList = testenv.GetAppFileList(appListV2) - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - //############ UPGRADE VERIFICATIONS ############ - appVersion = "V2" - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("m4, integration, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Upload 15 apps of 100MB size each to GCP for Indexer Cluster and Search Head Cluster for cluster scope - * Create app sources for Cluster Master and Deployer with cluster scope - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - ######### INITIAL VERIFICATIONS ############# - * Verify Apps are Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied, installed on Search Heads and Indexers pods - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - s3TestDirIdxc := "m4appfw-idxc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search head Cluster", appVersion)) - s3TestDirShc := "m4appfw-shc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 30 - - // Create App framework Spec for C3 - appSourceNameIdxc := "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc := "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceVolumeNameIdxc := "appframework-test-volume-idxc-" + testenv.RandomDNSName(3) - appSourceVolumeNameShc := "appframework-test-volume-shc-" + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecIdxc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - appFrameworkSpecShc.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - - // Deploy Multisite Cluster and Search Head Cluster, with App Framework enabled on Cluster Manager and Deployer - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster with Search Head Cluster") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It("integration, m4, managerappframeworkm4gcptodo, appframework: can deploy a M4 SVA with App Framework enabled and delete apps from app directory when download is complete", func() { - - /* Test Steps - ################## SETUP ################## - * Upload big-size app to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework and wait for the pods to be ready - * When app download is complete, delete apps from app directory - ########## VERIFICATIONS ########## - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify bundle push is successful - * Verify apps are copied and installed on Monitoring Console and on Search Heads and Indexers pods - */ - - //################## SETUP ################## - // Download big size apps from GCP - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big-size app") - - // Upload big size app to GCP for Indexer Cluster - appVersion := "V1" - testcaseEnvInst.Log.Info("Upload big size app to GCP for Indexer Cluster") - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Indexer Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload big size app to GCP for Search Head Cluster - testcaseEnvInst.Log.Info("Upload big size app to GCP for Search Head Cluster") - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big size to GCP test directory for Search Head Cluster") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - shReplicas := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify App Download is completed on Cluster Manager - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), cm.Kind, deployment.GetName(), enterpriseApi.ScopeCluster, appSourceNameIdxc, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## VERIFICATIONS ########## - var idxcPodNames, shcPodNames []string - idxcPodNames = testenv.GeneratePodNameSlice(testenv.MultiSiteIndexerPod, deployment.GetName(), 1, true, siteCount) - shcPodNames = testenv.GeneratePodNameSlice(testenv.SearchHeadPod, deployment.GetName(), shReplicas, false, 1) - cmPod := []string{fmt.Sprintf(testenv.ClusterManagerPod, deployment.GetName())} - deployerPod := []string{fmt.Sprintf(testenv.DeployerPod, deployment.GetName())} - cmAppSourceInfo := testenv.AppSourceInfo{CrKind: cm.Kind, CrName: cm.Name, CrAppSourceName: appSourceNameIdxc, CrAppSourceVolumeName: appSourceVolumeNameIdxc, CrPod: cmPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: indexersPerSite, CrMultisite: true, CrClusterPods: idxcPodNames} - shcAppSourceInfo := testenv.AppSourceInfo{CrKind: shc.Kind, CrName: shc.Name, CrAppSourceName: appSourceNameShc, CrAppSourceVolumeName: appSourceVolumeNameShc, CrPod: deployerPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeCluster, CrAppList: appList, CrAppFileList: appFileList, CrReplicas: shReplicas, CrClusterPods: shcPodNames} - allAppSourceInfo := []testenv.AppSourceInfo{cmAppSourceInfo, shcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify no pods reset by checking the pod age - testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) - }) - }) - - Context("Multisite Indexer Cluster with Search Head Cluster (m4) with App Framework", func() { - It(" m4gcp, managerappframeworkm4gcp, appframeworkgcp, m4_mgr_gcp_sanity: can deploy a M4 SVA with App Framework enabled, install apps and check IsDeploymentInProgress for CM and SHC CR's", func() { - - /* Test Steps - ################## SETUP ################## - * Upload V1 apps to GCP for Indexer Cluster and Search Head Cluster - * Prepare and deploy M4 CRD with app framework - * Verify IsDeploymentInProgress is set - * Wait for the pods to be ready - */ - - //################## SETUP ################## - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - - // Upload V1 apps to GCP for Indexer Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Indexer Cluster", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGcsBucket, s3TestDirIdxc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Indexer Cluster %s", appVersion, testGcsBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload V1 apps to GCP for Search Head Cluster - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCP for Search Head Cluster", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGcsBucket, s3TestDirShc, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCP test directory for Search Head Cluster", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for M4 - appSourceNameIdxc = "appframework-idxc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appSourceNameShc = "appframework-shc-" + enterpriseApi.ScopeCluster + testenv.RandomDNSName(3) - appFrameworkSpecIdxc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameIdxc, enterpriseApi.ScopeCluster, appSourceNameIdxc, s3TestDirIdxc, 60) - appFrameworkSpecShc := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameShc, enterpriseApi.ScopeCluster, appSourceNameShc, s3TestDirShc, 60) - - // Deploy M4 CRD - testcaseEnvInst.Log.Info("Deploy Multisite Indexer Cluster with Search Head Cluster") - siteCount := 3 - indexersPerSite := 1 - cm, _, shc, err := deployment.DeployMultisiteClusterWithSearchHeadAndAppFramework(ctx, deployment.GetName(), indexersPerSite, siteCount, appFrameworkSpecIdxc, appFrameworkSpecShc, true, "", "") - - Expect(err).To(Succeed(), "Unable to deploy Multisite Indexer Cluster and Search Head Cluster with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Cluster Master CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for Cluster Manager") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, cm.Name, cm.Kind) - - // Ensure that the Cluster Manager goes to Ready phase - testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - - // Verify IsDeploymentInProgress Flag is set to true for SHC CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag for SHC") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Indexer Cluster configured as Multisite - testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - - // Verify RF SF is met - testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) - }) - }) -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_suite_test.go b/test/appframework_gcp/s1/appframework_gcs_suite_test.go deleted file mode 100644 index af2fab4c2..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_suite_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package gcps1appfw - -import ( - "os" - "path/filepath" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/splunk/splunk-operator/test/testenv" -) - -const ( - // PollInterval specifies the polling interval - PollInterval = 5 * time.Second - - // ConsistentPollInterval is the interval to use to consistently check a state is stable - ConsistentPollInterval = 200 * time.Millisecond - ConsistentDuration = 2000 * time.Millisecond -) - -var ( - testenvInstance *testenv.TestEnv - testSuiteName = "s1appfw-" + testenv.RandomDNSName(3) - appListV1 []string - appListV2 []string - testDataGcsBucket = os.Getenv("TEST_BUCKET") - testGCSBucket = os.Getenv("TEST_INDEXES_S3_BUCKET") - gcsAppDirV1 = testenv.AppLocationV1 - gcsAppDirV2 = testenv.AppLocationV2 - gcsPVTestApps = testenv.PVTestAppsLocation - currDir, _ = os.Getwd() - downloadDirV1 = filepath.Join(currDir, "s1appfwV1-"+testenv.RandomDNSName(4)) - downloadDirV2 = filepath.Join(currDir, "s1appfwV2-"+testenv.RandomDNSName(4)) - downloadDirPVTestApps = filepath.Join(currDir, "s1appfwPVTestApps-"+testenv.RandomDNSName(4)) -) - -// TestBasic is the main entry point -func TestBasic(t *testing.T) { - - RegisterFailHandler(Fail) - - RunSpecs(t, "Running "+testSuiteName) -} - -var _ = BeforeSuite(func() { - var err error - testenvInstance, err = testenv.NewDefaultTestEnv(testSuiteName) - Expect(err).ToNot(HaveOccurred()) - - if testenv.ClusterProvider == "gcp" { - // Create a list of apps to upload to GCP - appListV1 = testenv.BasicApps - appFileList := testenv.GetAppFileList(appListV1) - - // Download V1 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download V1 app files") - - // Create a list of apps to upload to GCP after poll period - appListV2 = append(appListV1, testenv.NewAppsAddedBetweenPolls...) - appFileList = testenv.GetAppFileList(appListV2) - - // Download V2 Apps from GCP - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download V2 app files") - } else { - testenvInstance.Log.Info("Skipping Before Suite Setup", "Cluster Provider", testenv.ClusterProvider) - } - -}) - -var _ = AfterSuite(func() { - if testenvInstance != nil { - Expect(testenvInstance.Teardown()).ToNot(HaveOccurred()) - } - - // Delete locally downloaded app files - err := os.RemoveAll(downloadDirV1) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V1 app files") - err = os.RemoveAll(downloadDirV2) - Expect(err).To(Succeed(), "Unable to delete locally downloaded V2 app files") -}) diff --git a/test/appframework_gcp/s1/appframework_gcs_test.go b/test/appframework_gcp/s1/appframework_gcs_test.go deleted file mode 100644 index 6bd3ad11c..000000000 --- a/test/appframework_gcp/s1/appframework_gcs_test.go +++ /dev/null @@ -1,2030 +0,0 @@ -// Copyright (c) 2018-2022 Splunk Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License.s -package gcps1appfw - -import ( - "context" - "fmt" - "path/filepath" - "strings" - "time" - - enterpriseApi "github.com/splunk/splunk-operator/api/v4" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - testenv "github.com/splunk/splunk-operator/test/testenv" - corev1 "k8s.io/api/core/v1" -) - -var _ = Describe("s1appfw test", func() { - - var testcaseEnvInst *testenv.TestCaseEnv - var deployment *testenv.Deployment - var gcsTestDir string - var uploadedApps []string - var appSourceName string - var appSourceVolumeName string - var filePresentOnOperator bool - - ctx := context.TODO() - - BeforeEach(func() { - var err error - name := fmt.Sprintf("%s-%s", testenvInstance.GetName(), testenv.RandomDNSName(3)) - testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) - Expect(err).To(Succeed(), "Unable to create testcaseenv") - deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) - Expect(err).To(Succeed(), "Unable to create deployment") - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - appSourceVolumeName = "appframework-test-volume-" + testenv.RandomDNSName(3) - }) - - AfterEach(func() { - // When a test spec failed, skip the teardown so we can troubleshoot. - if CurrentGinkgoTestDescription().Failed { - testcaseEnvInst.SkipTeardown = true - } - if deployment != nil { - deployment.Teardown() - } - // Delete files uploaded to GCS - if !testcaseEnvInst.SkipTeardown { - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - } - if testcaseEnvInst != nil { - Expect(testcaseEnvInst.Teardown()).ToNot(HaveOccurred()) - } - - if filePresentOnOperator { - //Delete files from app-directory - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(testenv.AppDownloadVolume, "test_file.img") - testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - } - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, appframeworks1gcp, appframeworkgcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then upgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V1 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to gcs App Source - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to gcs for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to gcs for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV2 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, appframeworks1gcp, appframeworkgcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps then downgrade them", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V2 apps to gcs for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload V2 apps to gcs for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ INITIAL VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############# DOWNGRADE APPS ################ - * Upload V1 apps on gcs - * Wait for Monitoring Console and Standalone pods to be ready - ########## DOWNGRADE VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V2 apps to gcs - appVersion := "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList := testenv.GetAppFileList(appListV2) - gcsTestDir = "s1appfw-" + testenv.RandomDNSName(4) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV2, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############# DOWNGRADE APPS ################ - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // get revision number of the resource - resourceVersion := testenv.GetResourceVersion(ctx, deployment, testcaseEnvInst, mc) - - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion = "V1" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV1) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // wait for custom resource resource version to change - testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## DOWNGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV1 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - mcAppSourceInfo.CrAppVersion = appVersion - mcAppSourceInfo.CrAppList = appListV1 - mcAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV1) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, install apps on new pod, scale down", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create 2 app sources for Monitoring Console and Standalone - * Prepare and deploy Monitoring Console CRD with app framework and wait for the pod to be ready - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING UP VERIFICATION ######### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############## SCALING DOWN ################# - * Scale down Standalone - * Wait for Monitoring Console and Standalone to be ready - ########### SCALING DOWN VERIFICATION ####### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone and Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - // testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - mcPod := []string{fmt.Sprintf(testenv.MonitoringConsolePod, deployment.GetName())} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - mcAppSourceInfo := testenv.AppSourceInfo{CrKind: mc.Kind, CrName: mc.Name, CrAppSourceName: appSourceNameMC, CrAppSourceVolumeName: appSourceNameMC, CrPod: mcPod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, mcAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING UP VERIFICATION ######### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - - //############## SCALING DOWN ################# - // Scale down Standalone instance - testcaseEnvInst.Log.Info("Scale down Standalone") - scaledReplicaCount = 1 - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone after scaling down") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale down Standalone") - - // Ensure Standalone is scaling down - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingDown) - - // Wait for Standalone to be in READY status - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseReady) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - //########### SCALING DOWN VERIFICATION ####### - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install apps, scale up, upgrade apps", func() { - - /* Test Steps - ################## SETUP #################### - * Upload apps on gcs - * Create app source for Standalone - * Prepare and deploy Standalone CRD with app framework and wait for the pod to be ready - ########## INITIAL VERIFICATION ############# - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############### SCALING UP ################## - * Scale up Standalone - * Wait for Standalone to be ready - ############### UPGRADE APPS ################ - * Upload V2 apps to gcs App Source - ###### SCALING UP/UPGRADE VERIFICATIONS ##### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Upload V1 apps to gcs for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone", appVersion)) - - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to gcs for Standalone - gcsTestDir := "s1appfw-" + testenv.RandomDNSName(4) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //########## INITIAL VERIFICATION ############# - scaledReplicaCount := 2 - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList, CrReplicas: scaledReplicaCount} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### SCALING UP ################## - // Scale up Standalone instance - testcaseEnvInst.Log.Info("Scale up Standalone") - - standalone = &enterpriseApi.Standalone{} - err = deployment.GetInstance(ctx, deployment.GetName(), standalone) - Expect(err).To(Succeed(), "Failed to get instance of Standalone") - - standalone.Spec.Replicas = int32(scaledReplicaCount) - - err = deployment.UpdateCR(ctx, standalone) - Expect(err).To(Succeed(), "Failed to scale up Standalone") - - // Ensure Standalone is scaling up - testenv.VerifyStandalonePhase(ctx, deployment, testcaseEnvInst, deployment.GetName(), enterpriseApi.PhaseScalingUp) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // ############## UPGRADE APPS ################# - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to gcs for Standalone and Monitoring Console - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to gcs for Standalone and Monitoring Console", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ SCALING UP/UPGRADE VERIFICATIONS ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - standaloneAppSourceInfo.CrPod = []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0), fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 1)} - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - // ES App Installation not supported at the time. Will be added back at a later time. - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, integrationgcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone and have ES app installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload ES app to gcs - * Create App Source for Standalone - * Prepare and deploy Standalone and wait for the pod to be ready - ################## VERIFICATION ############# - * Verify ES app is installed on Standalone - */ - - //################## SETUP #################### - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download ES app from gcs") - esApp := []string{"SplunkEnterpriseSecuritySuite"} - appFileList := testenv.GetAppFileList(esApp) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload ES app to gcs - testcaseEnvInst.Log.Info("Upload ES app on gcs") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload ES app to gcs test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopePremiumApps, appSourceName, gcsTestDir, 60) - appFrameworkSpec.AppSources[0].PremiumAppsProps = enterpriseApi.PremiumAppsProps{ - Type: enterpriseApi.PremiumAppsTypeEs, - EsDefaults: enterpriseApi.EsDefaults{ - SslEnablement: enterpriseApi.SslEnablementIgnore, - }, - } - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone with App framework") - - // Ensure Standalone goes to Ready phase - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - appVersion := "V1" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: esApp, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############## UPGRADE APPS ################# - - // Delete apps on gcs - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on gcs", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Download ES App from gcs - testcaseEnvInst.Log.Info("Download updated ES app from gcs") - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV2, downloadDirV2, appFileList) - Expect(err).To(Succeed(), "Unable to download ES app") - - // Upload V2 apps to gcs for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s Es app to gcs for Standalone and Monitoring Console", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s Es app to gcs test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = esApp - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(esApp) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled and install around 350MB of apps at once", func() { - - /* Test Steps - ################## SETUP #################### - * Create app source for Standalone - * Add more apps than usual on gcs for this test - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a bigger list of apps to be installed than the default one - appList := append(appListV1, testenv.RestartNeededApps...) - appFileList := testenv.GetAppFileList(appList) - appVersion := "V1" - - // Download apps from gcs - testcaseEnvInst.Log.Info("Download bigger amount of apps from gcs for this test") - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info("Upload bigger amount of apps to GCS for this test") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("s1gcp, smokegcp, appframeworks1gcp, appframeworkgcp: can deploy a standalone instance with App Framework enabled for manual poll", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Create app source for Standalone - * Prepare and deploy Standalone with app framework(MANUAL POLL) and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - * Verify Apps are not updated - ############ ENABLE MANUAL POLL ############ - * Verify Manual Poll disabled after the check - ############ V2 APP VERIFICATION FOR STANDALONE AND MONITORING CONSOLE ########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 0) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Upload V1 apps to GCS - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 0) - - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create Standalone Deployment with App Framework - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - //############### UPGRADE APPS ################ - - //Delete apps on GCS for new Apps - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - //Upload new Versioned Apps to GCS - appVersion = "V2" - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ############ VERIFICATION APPS ARE NOT UPDATED BEFORE ENABLING MANUAL POLL ############ - appVersion = "V1" - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ ENABLE MANUAL POLL ############ - appVersion = "V2" - testcaseEnvInst.Log.Info("Get config map for triggering manual update") - config, err := testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(err).To(Succeed(), "Unable to get config map for manual poll") - testcaseEnvInst.Log.Info("Config map data for", "Standalone", config.Data["Standalone"]) - - testcaseEnvInst.Log.Info("Modify config map to trigger manual update") - config.Data["Standalone"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - config.Data["MonitoringConsole"] = strings.Replace(config.Data["Standalone"], "off", "on", 1) - err = deployment.UpdateCR(ctx, config) - Expect(err).To(Succeed(), "Unable to update config map") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //Verify config map set back to off after poll trigger - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify config map set back to off after poll trigger for %s app", appVersion)) - config, _ = testenv.GetAppframeworkManualUpdateConfigMap(ctx, deployment, testcaseEnvInst.GetName()) - Expect(strings.Contains(config.Data["Standalone"], "status: off") && strings.Contains(config.Data["MonitoringConsole"], "status: off")).To(Equal(true), "Config map update not complete") - - //############### VERIFICATION FOR UPGRADE ################ - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can deploy Several standalone CRs in the same namespace with App Framework enabled", func() { - - /* Test Steps - ################## SETUP #################### - * Add more apps than usual on GCS for this test - * Split the App list into 2 segments with a common apps and uncommon apps for each Standalone - * Create app source for 2 Standalones - * Prepare and deploy Standalones with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - - // Creating a list of apps to be installed on both standalone - appList1 := append(appListV1, testenv.RestartNeededApps[len(testenv.RestartNeededApps)/2:]...) - appList2 := append(appListV1, testenv.RestartNeededApps[:len(testenv.RestartNeededApps)/2]...) - appVersion := "V1" - - // Download apps from GCS - testcaseEnvInst.Log.Info("Download the extra apps from GCS for this test") - appFileList := testenv.GetAppFileList(testenv.RestartNeededApps) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps files") - - // Upload apps to GCS for first Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 1st Standalone") - appFileListStandalone1 := testenv.GetAppFileList(appList1) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileListStandalone1, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Upload apps to GCS for second Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for 2nd Standalone") - gcsTestDirStandalone2 := "s1appfw-2-" + testenv.RandomDNSName(4) - appFileListStandalone2 := testenv.GetAppFileList(appList2) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirStandalone2, appFileListStandalone2, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework Spec - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Create App framework Spec - appSourceNameStandalone2 := "appframework-2-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appSourceVolumeNameStandalone2 := "appframework-test-volume-2-" + testenv.RandomDNSName(3) - appFrameworkSpecStandalone2 := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameStandalone2, enterpriseApi.ScopeLocal, appSourceNameStandalone2, gcsTestDirStandalone2, 60) - specStandalone2 := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecStandalone2, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy 1st Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy 1st Standalone instance") - testcaseEnvInst.Log.Info("Deploy 2nd Standalone") - standalone2Name := deployment.GetName() + testenv.RandomDNSName(3) - standalone2, err := deployment.DeployStandaloneWithGivenSpec(ctx, standalone2Name, specStandalone2) - Expect(err).To(Succeed(), "Unable to deploy 2nd Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone2, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList1, CrAppFileList: appFileListStandalone1} - standalone2Pod := []string{fmt.Sprintf(testenv.StandalonePod, standalone2Name, 0)} - standalone2AppSourceInfo := testenv.AppSourceInfo{CrKind: standalone2.Kind, CrName: standalone2Name, CrAppSourceName: appSourceNameStandalone2, CrPod: standalone2Pod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList2, CrAppFileList: appFileListStandalone2} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo, standalone2AppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can add new apps to app source while install is in progress and have all apps installed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework and wait for the pod to be ready - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList = testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - appList = testenv.BigSingleApp - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyComplete) - - // Upload more apps to GCS for Standalone - appList = testenv.ExtraApps - appFileList = testenv.GetAppFileList(appList) - testcaseEnvInst.Log.Info("Upload more apps to GCS for Standalone") - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload more apps to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for polling interval to pass - testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Verify all apps are installed on Standalone - appList = append(testenv.BigSingleApp, testenv.ExtraApps...) - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on Standalone", appList)) - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{standalonePodName}, appList, true, "enabled", false, false) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app install is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app install is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - if err != nil { - for i := 1; i < 10; i++ { - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - if err == nil { - continue - } else { - time.Sleep(1 * time.Second) - } - } - } - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - - // Verify App installation is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgInstallPending) - - //Delete configMap Object - err = testenv.DeleteConfigMap(testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to delete ConfigMao", "ConfigMap name", ConfigMapName) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Verify livenessProbe and readinessProbe config object and scripts############ - testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") - _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) - Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) - scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} - allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) - testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and reset operator pod while app download is in progress", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is in progress, restart the operator - ############## VERIFICATIONS ################ - * Verify App download is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download all test apps from GCS - appVersion := "V1" - appList := append(testenv.BigSingleApp, testenv.ExtraApps...) - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgDownloadComplete, enterpriseApi.AppPkgDownloadPending) - - // Delete Operator pod while Install in progress - testenv.DeleteOperatorPod(testcaseEnvInst) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, install an app then disable it and remove it from app source", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - * Disable the app - * Delete the app from GCS - * Check for repo state in App Deployment Info - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // Verify repo state on App to be disabled to be 1 (i.e app present on GCS bucket) - appName := appListV1[0] - appFileName := testenv.GetAppFileList([]string{appName}) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 1, appFileName[0]) - - // Disable the app - testenv.DisableAppsToGCP(downloadDirV1, appFileName, gcsTestDir) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileName) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Wait for App state to update after config file change - standalonePodName := fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0) - testenv.WaitforAppInstallState(ctx, deployment, testcaseEnvInst, []string{standalonePodName}, testcaseEnvInst.GetName(), appName, "disabled", false) - - // Delete the file from GCS - gcsFilepath := filepath.Join(gcsTestDir, appFileName[0]) - err = testenv.DeleteFileOnGCP(testGCSBucket, gcsFilepath) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to delete %s app on GCS test directory", appFileName[0])) - - // Verify repo state is set to 2 (i.e app deleted from GCS bucket) - testenv.VerifyAppRepoState(ctx, deployment, testcaseEnvInst, standalone.Name, standalone.Kind, appSourceName, 2, appFileName[0]) - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance with App Framework enabled, attempt to update using incorrect GCS credentials", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############ V1 APP VERIFICATION FOR STANDALONE########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - // ############ Modify secret key ########### - * Create App framework volume with random credentials and apply to Spec - * Check for changes in App phase to determine if next poll has been triggered - ############ UPGRADE V2 APPS ########### - * Upload V2 apps to GCS App Source - * Check no apps are updated as auth key is incorrect - ############ Modify secret key to correct one########### - * Apply spec with correct credentails - * Wait for the pod to be ready - ############ V2 APP VERIFICATION########### - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify no pod resets triggered due to app install - * Verify App enabled and version by running splunk cmd - */ - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - secretref := standalone.Spec.AppFrameworkConfig.VolList[0].SecretRef - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - secretStruct, err := testenv.GetSecretStruct(ctx, deployment, testcaseEnvInst.GetName(), secretref) - Expect(err).To(Succeed(), "Unable to obtain secret object") - secretData := secretStruct.Data - modifiedSecretData := map[string][]byte{"gcs_access_key": []byte(testenv.RandomDNSName(5)), "gcs_secret_key": []byte(testenv.RandomDNSName(5))} - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ INITIAL VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appListV1, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key ########### - // Create App framework volume with invalid credentials and apply to Spec - testcaseEnvInst.Log.Info("Update Standalone spec with invalid credentials") - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, modifiedSecretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // ############## UPGRADE APPS ################# - // Delete apps on - testcaseEnvInst.Log.Info(fmt.Sprintf("Delete %s apps on GCS", appVersion)) - testenv.DeleteFilesOnGCP(testGCSBucket, uploadedApps) - uploadedApps = nil - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList(appListV2) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Check no apps are updated as auth key is incorrect - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - // ############ Modify secret key to correct one########### - // Apply spec with correct credentials - err = testenv.ModifySecretObject(ctx, deployment, testcaseEnvInst.GetName(), secretref, secretData) - Expect(err).To(Succeed(), "Unable to update secret Object") - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge = testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - standaloneAppSourceInfo.CrAppVersion = appVersion - standaloneAppSourceInfo.CrAppList = appListV2 - standaloneAppSourceInfo.CrAppFileList = testenv.GetAppFileList(appListV2) - allAppSourceInfo = []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and update apps after app download is completed", func() { - - /* Test Steps - ################## SETUP #################### - * Upload app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * While app download is completed, upload new versions of the apps - ############## VERIFICATIONS ################ - * Verify App download is in completed on Standalone - * Upload updated app to GCS as pervious app download is complete - * Verify app is installed on Standalone - ############## UPGRADE VERIFICATIONS ################ - * Wait for next poll to trigger on Standalone - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download test app from GCS - appVersion := "V1" - appListV1 := []string{appListV1[0]} - appFileList := testenv.GetAppFileList(appListV1) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download apps") - - // Upload apps to GCS for Standalone - testcaseEnvInst.Log.Info("Upload apps to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 120) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App download is in progress on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgInstallComplete, enterpriseApi.AppPkgPodCopyPending) - - // Upload V2 apps to GCS for Standalone - appVersion = "V2" - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s app to GCS for Standalone", appVersion)) - appFileList = testenv.GetAppFileList([]string{appListV2[0]}) - - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV2) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s app to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - //######### VERIFICATIONS ############# - appVersion = "V1" - testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)}, appListV1, false, "enabled", false, false) - - // Check for changes in App phase to determine if next poll has been triggered - testenv.WaitforPhaseChange(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############ UPGRADE VERIFICATION ########### - appVersion = "V2" - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: []string{appListV2[0]}, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: can deploy a Standalone instance and install a bigger volume of apps than the operator PV disk space", func() { - - /* Test Steps - ################## SETUP #################### - * Create a file on operator to utilize over 1G of space - * Upload file to gcs for standalone - * Create app source for Standalone with parallelDownload=15 - * Prepare and deploy Standalone with app framework and wait for the pod to be ready - ############### VERIFICATION ################ - * Verify Apps Downloaded in App Deployment Info - * Verify Apps Copied in App Deployment Info - * Verify App Package is deleted from Operator Pod - * Verify Apps Installed in App Deployment Info - * Verify App Package is deleted from Splunk Pod - * Verify App Directory in under splunk path - * Verify App enabled and version by running splunk cmd - */ - - //################## SETUP #################### - // Create a large file on Operator pod - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - err := testenv.CreateDummyFileOnOperator(ctx, deployment, opPod, testenv.AppDownloadVolume, "1G", "test_file.img") - Expect(err).To(Succeed(), "Unable to create file on operator") - filePresentOnOperator = true - - // Download apps for test - appVersion := "V1" - appList := testenv.PVTestApps - appFileList := testenv.GetAppFileList(appList) - err = testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsPVTestApps, downloadDirPVTestApps, appFileList) - Expect(err).To(Succeed(), "Unable to download app files") - - // Upload apps to GCS - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirPVTestApps) - Expect(err).To(Succeed(), "Unable to upload apps to GCS test directory") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 15 - - // Create App framework Spec - appSourceName := "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - //############### VERIFICATION ################ - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("integrationgcp, s1gcp, appframeworks1gcp, appframeworkgcp: Deploy a Standalone instance with App Framework enabled and delete apps from app directory when app download is complete", func() { - - /* Test Steps - ################## SETUP #################### - * Upload big-size app to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone - * When app download is complete, delete apps from app directory - ############## VERIFICATIONS ################ - * Verify App installation is in progress on Standalone - * Upload more apps from GCS during bigger app install - * Wait for polling interval to pass - * Verify all apps are installed on Standalone - */ - - // ################## SETUP FOR STANDALONE #################### - // Download big size apps from GCS - appVersion := "V1" - appList := testenv.BigSingleApp - appFileList := testenv.GetAppFileList(appList) - err := testenv.DownloadFilesFromGCP(testDataGcsBucket, gcsAppDirV1, downloadDirV1, appFileList) - Expect(err).To(Succeed(), "Unable to download big app") - - // Upload big-size app to GCS for Standalone - testcaseEnvInst.Log.Info("Upload big-size app to GCS for Standalone") - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), "Unable to upload big-size app to GCS test directory for Standalone") - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify App Download is completed on Standalone - testenv.VerifyAppState(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind, appSourceName, appFileList, enterpriseApi.AppPkgPodCopyComplete, enterpriseApi.AppPkgPodCopyPending) - - //Delete apps from app-directory when app download is complete - opPod := testenv.GetOperatorPodName(testcaseEnvInst) - podDownloadPath := filepath.Join(splcommon.AppDownloadVolume, "downloadedApps", testenvInstance.GetName(), standalone.Kind, deployment.GetName(), enterpriseApi.ScopeLocal, appSourceName, testenv.AppInfo[appList[0]]["filename"]) - err = testenv.DeleteFilesOnOperatorPod(ctx, deployment, opPod, []string{podDownloadPath}) - Expect(err).To(Succeed(), "Unable to delete file on pod") - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Get Pod age to check for pod resets later - splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) - - // ############ VERIFICATION ########### - standalonePod := []string{fmt.Sprintf(testenv.StandalonePod, deployment.GetName(), 0)} - standaloneAppSourceInfo := testenv.AppSourceInfo{CrKind: standalone.Kind, CrName: standalone.Name, CrAppSourceName: appSourceName, CrPod: standalonePod, CrAppVersion: appVersion, CrAppScope: enterpriseApi.ScopeLocal, CrAppList: appList, CrAppFileList: appFileList} - allAppSourceInfo := []testenv.AppSourceInfo{standaloneAppSourceInfo} - testenv.AppFrameWorkVerifications(ctx, deployment, testcaseEnvInst, allAppSourceInfo, splunkPodAge, "") - }) - }) - - Context("Standalone deployment (S1) with App Framework", func() { - It("smokegcp, s1gcp, appframeworks1gcp, appframeworkgcp, s1_gcp_sanity: can deploy a Standalone instance with App Framework enabled, install apps and check isDeploymentInProgress is set for Standaloen and MC CR's", func() { - - /* Test Steps - ################## SETUP #################### - * Upload V1 apps to GCS for Monitoring Console - * Create app source for Monitoring Console - * Prepare and deploy Monitoring Console with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - * Upload V1 apps to GCS for Standalone - * Create app source for Standalone - * Prepare and deploy Standalone with app framework - * Check isDeploymentInProgress is set for Monitoring Console CR - * Wait for the pod to be ready - */ - - // ################## SETUP FOR MONITORING CONSOLE #################### - - // Upload V1 apps to GCS for Monitoring Console - appVersion := "V1" - appFileList := testenv.GetAppFileList(appListV1) - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Monitoring Console", appVersion)) - - gcsTestDirMC := "s1appfw-mc-" + testenv.RandomDNSName(4) - uploadedFiles, err := testenv.UploadFilesToGCP(testGCSBucket, gcsTestDirMC, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Monitoring Console %s", appVersion, testGCSBucket)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Create App framework spec for Monitoring Console - appSourceNameMC := "appframework-" + enterpriseApi.ScopeLocal + "mc-" + testenv.RandomDNSName(3) - appSourceVolumeNameMC := "appframework-test-volume-mc-" + testenv.RandomDNSName(3) - appFrameworkSpecMC := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeNameMC, enterpriseApi.ScopeLocal, appSourceNameMC, gcsTestDirMC, 60) - mcSpec := enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "IfNotPresent", - }, - Volumes: []corev1.Volume{}, - }, - AppFrameworkConfig: appFrameworkSpecMC, - } - - // Deploy Monitoring Console - testcaseEnvInst.Log.Info("Deploy Monitoring Console") - mcName := deployment.GetName() - mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) - Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") - - // Verify IsDeploymentInProgress Flag is set to true for Monitroing Console CR - testcaseEnvInst.Log.Info("Checking isDeploymentInProgressFlag") - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, mcName, mc.Kind) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - // ################## SETUP FOR STANDALONE #################### - // Upload V1 apps to GCS for Standalone - testcaseEnvInst.Log.Info(fmt.Sprintf("Upload %s apps to GCS for Standalone", appVersion)) - uploadedFiles, err = testenv.UploadFilesToGCP(testGCSBucket, gcsTestDir, appFileList, downloadDirV1) - Expect(err).To(Succeed(), fmt.Sprintf("Unable to upload %s apps to GCS test directory for Standalone", appVersion)) - uploadedApps = append(uploadedApps, uploadedFiles...) - - // Maximum apps to be downloaded in parallel - maxConcurrentAppDownloads := 5 - - // Create App framework spec for Standalone - appSourceName = "appframework-" + enterpriseApi.ScopeLocal + testenv.RandomDNSName(3) - appFrameworkSpec := testenv.GenerateAppFrameworkSpec(ctx, testcaseEnvInst, appSourceVolumeName, enterpriseApi.ScopeLocal, appSourceName, gcsTestDir, 60) - appFrameworkSpec.MaxConcurrentAppDownloads = uint64(maxConcurrentAppDownloads) - spec := enterpriseApi.StandaloneSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: mcName, - }, - }, - AppFrameworkConfig: appFrameworkSpec, - } - - // Deploy Standalone - testcaseEnvInst.Log.Info("Deploy Standalone") - standalone, err := deployment.DeployStandaloneWithGivenSpec(ctx, deployment.GetName(), spec) - Expect(err).To(Succeed(), "Unable to deploy Standalone instance with App framework") - - // Verify IsDeploymentInProgress Flag is set to true for Standalone CR - testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, deployment.GetName(), standalone.Kind) - - // Wait for Standalone to be in READY status - testenv.StandaloneReady(ctx, deployment, deployment.GetName(), standalone, testcaseEnvInst) - - // Verify Monitoring Console is Ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - - }) - }) - -})