From 513a894ee75021632dd6788725bd060933c60144 Mon Sep 17 00:00:00 2001 From: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> Date: Wed, 17 Jan 2024 10:41:48 -0800 Subject: [PATCH] Level-2: Single state machine for Level-2 support (#1216) * Added SHC functions * Check error in change annotation * Added Single Site IDX functions * Added functional test case * Removed Change annotation; Added TODO * Added documentation * Added multisite func * Added branch to workflow * Commiting * Added specific test * Changed image * Added cm ref * Removed cm ref * Only CM and LM * Added image output * Added mc change image * Added shc change image * Fixed shc name * Added idxc * Check this * Test with only CM, SHC, IDX * Test with only CM, IDX * Test with only LM, CM, SHC, IDX * Test only with CM, MC, SHC, IDX * Addd cm ref to CM,MC,SHC,IDX * All the instances * Test with LM,CM,MC * Check without cm ref * Cm Ref + LM,CM,MC,SHC * CM ref + LM,CM,MC,IDX * Testing all with no idx update code * Fixed commit * All + only single site * With everything * Fixed mgr client * Final * one stop for all the upgrade scenarios Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * added upgradepath to clustermanager Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * added upgradepath to all CR Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * Made changes in upgrade checks * somemore changes to fix test case for upgrade scenario Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * ignore tel app install in unit test Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * intermittent, changes Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * fixed searchhead cluster,mc, lm, cm Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * fixed test case Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * working test code for upgrade Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * unit test cases fixed Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * added comments to the new code Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * fixed some test cases Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * fixed some test cases Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * formatting changes Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> * addressed review comments Signed-off-by: vivekr-splunk * changing go to 1.21 Signed-off-by: vivekr-splunk * changing go to 1.21 Signed-off-by: vivekr-splunk * changing go to 1.21 Signed-off-by: vivekr-splunk * adding this branch for int test pipeline Signed-off-by: vivekr-splunk * test case fix - adding extra timeout * test case fix - adding extra timeout * changed splunk version to 9.1.2 * changed order in the test case for level-2 support * changing timeout to so test can pass Signed-off-by: vivekr-splunk * changed order first search and then index Signed-off-by: vivekr-splunk * adding back kind name in controller * adding more timeout Signed-off-by: vivekr-splunk * increasing to 10 min Signed-off-by: vivekr-splunk * increasing overall test run to 6 hours Signed-off-by: vivekr-splunk * doc changes Signed-off-by: vivekr-splunk * just run m4 tests Signed-off-by: vivekr-splunk * just run c3 test Signed-off-by: vivekr-splunk * enabled all the test Signed-off-by: vivekr-splunk * fixed go libraries * increasing time to 7h for test * adding helm test * removed unused functions * adding comment --------- Signed-off-by: vivekr-splunk <94569031+vivekr-splunk@users.noreply.github.com> Signed-off-by: vivekr-splunk Co-authored-by: Tanya Garg --- .env | 4 +- .github/workflows/helm-test-workflow.yml | 2 + .github/workflows/int-test-workflow.yml | 2 + Makefile | 6 +- docs/SplunkOperatorUpgrade.md | 23 +- go.mod | 18 +- go.sum | 50 +- pkg/splunk/controller/statefulset.go | 5 +- pkg/splunk/enterprise/clustermanager.go | 74 +- pkg/splunk/enterprise/clustermanager_test.go | 86 ++- pkg/splunk/enterprise/clustermaster.go | 1 + pkg/splunk/enterprise/clustermaster_test.go | 10 +- pkg/splunk/enterprise/events.go | 7 + pkg/splunk/enterprise/indexercluster.go | 75 +- pkg/splunk/enterprise/indexercluster_test.go | 42 +- pkg/splunk/enterprise/licensemanager.go | 1 + pkg/splunk/enterprise/licensemanager_test.go | 6 +- pkg/splunk/enterprise/monitoringconsole.go | 63 +- .../enterprise/monitoringconsole_test.go | 111 +-- pkg/splunk/enterprise/searchheadcluster.go | 28 +- .../enterprise/searchheadcluster_test.go | 39 +- pkg/splunk/enterprise/standalone.go | 1 + pkg/splunk/enterprise/standalone_test.go | 6 +- pkg/splunk/enterprise/upgrade.go | 315 ++++++++ pkg/splunk/enterprise/upgrade_test.go | 726 ++++++++++++++++++ .../c3/appframework_aws_test.go | 163 ++-- .../c3/manager_appframework_test.go | 301 ++++++-- .../m4/appframework_aws_test.go | 6 +- .../m4/manager_appframework_test.go | 12 +- .../c3/appframework_azure_test.go | 162 ++-- .../c3/manager_appframework_azure_test.go | 162 ++-- .../custom_resource_crud_c3_test.go | 6 +- .../manager_custom_resource_crud_c3_test.go | 6 +- test/delete_cr/deletecr_test.go | 6 +- test/licensemanager/manager_lm_c3_test.go | 8 +- test/licensemaster/lm_c3_test.go | 8 +- .../manager_monitoring_console_test.go | 16 +- .../monitoring_console_test.go | 16 +- test/run-tests.sh | 18 +- test/secret/manager_secret_c3_test.go | 16 +- test/secret/secret_c3_test.go | 16 +- test/smoke/smoke_test.go | 8 +- test/testenv/testenv.go | 4 +- 43 files changed, 1935 insertions(+), 700 deletions(-) create mode 100644 pkg/splunk/enterprise/upgrade.go create mode 100644 pkg/splunk/enterprise/upgrade_test.go diff --git a/.env b/.env index 9d59fde8e..e69b6f579 100644 --- a/.env +++ b/.env @@ -1,9 +1,9 @@ OPERATOR_SDK_VERSION=v1.28.1 REVIEWERS=smohan-splunk,sgontla,gaurav-splunk,vivekr-splunk,kumarajeet -GO_VERSION=1.19.2 +GO_VERSION=1.21.5 AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip KUBECTL_VERSION=v1.28.0 AZ_CLI_VERSION=2.30.0 EKSCTL_VERSION=v0.143.0 EKS_CLUSTER_K8_VERSION=1.26 -SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.1 +SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.1.2 diff --git a/.github/workflows/helm-test-workflow.yml b/.github/workflows/helm-test-workflow.yml index 7a36b8fae..b735787c1 100644 --- a/.github/workflows/helm-test-workflow.yml +++ b/.github/workflows/helm-test-workflow.yml @@ -4,6 +4,8 @@ on: branches: - develop - main + - feature** + - testing jobs: build-operator-image: runs-on: ubuntu-latest diff --git a/.github/workflows/int-test-workflow.yml b/.github/workflows/int-test-workflow.yml index 17b34ff19..04432c084 100644 --- a/.github/workflows/int-test-workflow.yml +++ b/.github/workflows/int-test-workflow.yml @@ -5,9 +5,11 @@ on: - develop - main - feature** + - testing jobs: build-operator-image: runs-on: ubuntu-latest + timeout-minutes: 360 env: SPLUNK_ENTERPRISE_IMAGE: ${{ secrets.SPLUNK_ENTERPRISE_IMAGE }} SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-operator diff --git a/Makefile b/Makefile index dd59513ae..359648d7d 100644 --- a/Makefile +++ b/Makefile @@ -3,17 +3,17 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 2.2.1 +VERSION ?= 2.5.0 # SPLUNK_ENTERPRISE_IMAGE defines the splunk docker tag that is used as default image. SPLUNK_ENTERPRISE_IMAGE ?= "docker.io/splunk/splunk:edge" # WATCH_NAMESPACE defines if its clusterwide operator or namespace specific -# by default we leave it as clusterwide if it has to be namespace specific, +# by default we leave it as clusterwide if it has to be namespace specific, # add namespace to this WATCH_NAMESPACE ?= "" -# NAMESPACE defines default namespace where operator will be installed +# NAMESPACE defines default namespace where operator will be installed NAMESPACE ?= "splunk-operator" # CHANNELS define the bundle channels used in the bundle. diff --git a/docs/SplunkOperatorUpgrade.md b/docs/SplunkOperatorUpgrade.md index db2b18f73..76e7ca4c1 100644 --- a/docs/SplunkOperatorUpgrade.md +++ b/docs/SplunkOperatorUpgrade.md @@ -1,6 +1,6 @@ # How to upgrade Splunk Operator and Splunk Enterprise Deployments -To upgrade the Splunk Operator for Kubernetes, you will overwrite the prior Operator release with the latest version. Once the lastest version of `splunk-operator-namespace.yaml` ([see below](#upgrading-splunk-operator-and-splunk-operator-deployment)) is applied the CRD's are updated and Operator deployment is updated with newer version of Splunk Operator image. Any new spec defined by the operator will be applied to the pods managed by Splunk Operator for Kubernetes. +To upgrade the Splunk Operator for Kubernetes, you will overwrite the prior Operator release with the latest version. Once the lastest version of `splunk-operator-namespace.yaml` ([see below](#upgrading-splunk-operator-and-splunk-operator-deployment)) is applied the CRD's are updated and Operator deployment is updated with newer version of Splunk Operator image. Any new spec defined by the operator will be applied to the pods managed by Splunk Operator for Kubernetes. ​ A Splunk Operator for Kubernetes upgrade might include support for a later version of the Splunk Enterprise Docker image. In that scenario, after the Splunk Operator completes its upgrade, the pods managed by Splunk Operator for Kubernetes will be restarted using the latest Splunk Enterprise Docker image. ​ @@ -10,7 +10,7 @@ A Splunk Operator for Kubernetes upgrade might include support for a later versi ​ * Before you upgrade, review the Splunk Operator [change log](https://github.com/splunk/splunk-operator/releases) page for information on changes made in the latest release. The Splunk Enterprise Docker image compatibility is noted in each release version. ​ -* If the Splunk Enterprise Docker image changes, review the Splunk Enterprise [Upgrade Readme](https://docs.splunk.com/Documentation/Splunk/latest/Installation/AboutupgradingREADTHISFIRST) page before upgrading. +* If the Splunk Enterprise Docker image changes, review the Splunk Enterprise [Upgrade Readme](https://docs.splunk.com/Documentation/Splunk/latest/Installation/AboutupgradingREADTHISFIRST) page before upgrading. ​ * For general information about Splunk Enterprise compatibility and the upgrade process, see [How to upgrade Splunk Enterprise](https://docs.splunk.com/Documentation/Splunk/latest/Installation/HowtoupgradeSplunk). ​ @@ -44,9 +44,12 @@ NAME READY STATUS RESTARTS splunk-operator-controller-manager-75f5d4d85b-8pshn 1/1 Running 0 5s ``` ​ -If a Splunk Operator release changes the custom resource (CRD) API version, the administrator is responsible for updating their Custom Resource specification to reference the latest CRD API version. -​ -If a Splunk Operator release includes an updated Splunk Enterprise Docker image, the operator upgrade will also initiate pod restart using the latest Splunk Enterprise Docker image. +If a Splunk Operator release changes the custom resource (CRD) API version, the administrator is responsible for updating their Custom Resource specification to reference the latest CRD API version. + +### Upgrading Splunk Enterprise Docker Image with the Operator Upgrade + +Splunk Operator follows the upgrade path steps mentioned in [Splunk documentation](https://docs.splunk.com/Documentation/Splunk/9.1.2/Installation/HowtoupgradeSplunk). If a Splunk Operator release includes an updated Splunk Enterprise Docker image, the operator upgrade will also initiate pod restart using the latest Splunk Enterprise Docker image. To follow the best practices described under the [General Process to Upgrade the Splunk Enterprise], a recommeded upgrade path is followed while initiating pod restarts of different Splunk Instances. At each step, if a particular CR instance exists, a certain flow is imposed to ensure that each instance is updated in the correct order. After an instance is upgraded, the Operator verifies if the upgrade was successful and all the components are working as expected. If any unexpected behaviour is detected, the process is terminated. + ## Steps to Upgrade from 1.0.5 or older version to latest @@ -128,7 +131,7 @@ imagePullPolicy: IfNotPresent ```bash kubectl get pod -o yaml | grep -i image image: docker.io/splunk/splunk-operator: -imagePullPolicy: IfNotPresent +imagePullPolicy: IfNotPresent ``` ​ To verify that a new Splunk Enterprise Docker image was applied to a pod, you can check the version of the image. Example: @@ -143,8 +146,10 @@ imagePullPolicy: IfNotPresent This is an example of the process followed by the Splunk Operator if the operator version is upgraded and a later Splunk Enterprise Docker image is available: ​ 1. A new Splunk Operator pod will be created, and the existing operator pod will be terminated. -2. Any existing License Manager, Search Head, Deployer, ClusterManager, Standalone pods will be terminated to be redeployed with the upgraded spec. -3. After a ClusterManager pod is restarted, the Indexer Cluster pods which are connected to it are terminated and redeployed. -4. After all pods in the Indexer cluster and Search head cluster are redeployed, the Monitoring Console pod is terminated and redeployed. +3. Any existing License Manager, Standalone, Monitoring console, Cluster manager, Search Head, ClusterManager, Indexer pods will be terminated to be redeployed with the upgraded spec. +4. Splunk Operator follows the upgrade path steps mentioned in Splunk documentation. The termination and redeployment of the pods happen in a particular order based on a recommended upgrade path. +5. After a ClusterManager pod is restarted, the Indexer Cluster pods which are connected to it are terminated and redeployed. +6. After all pods in the Indexer cluster and Search head cluster are redeployed, the Monitoring Console pod is terminated and redeployed. +7. Each pod upgrade is verified to ensure the process was successful and everything is working as expected. * Note: If there are multiple pods per Custom Resource, the pods are terminated and re-deployed in a descending order with the highest numbered pod going first diff --git a/go.mod b/go.mod index 656ba090e..1bc357ff5 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module github.com/splunk/splunk-operator -go 1.19 +go 1.21 require ( github.com/aws/aws-sdk-go v1.42.16 - github.com/go-logr/logr v1.2.4 - github.com/google/go-cmp v0.5.9 + github.com/go-logr/logr v1.3.0 + github.com/google/go-cmp v0.6.0 github.com/minio/minio-go/v7 v7.0.16 - github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.27.10 + github.com/onsi/ginkgo/v2 v2.14.0 + github.com/onsi/gomega v1.30.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/stretchr/testify v1.8.1 @@ -70,13 +70,13 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index f5ca52069..63f4f77ea 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.42.16 h1:jOUmYYpC77NZYQVHTOTFT4lwFBT1u3s8ETKciU4l6gQ= github.com/aws/aws-sdk-go v1.42.16/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -68,6 +69,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -94,8 +96,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -156,8 +158,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -251,12 +253,12 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= -github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -332,6 +334,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= @@ -370,7 +373,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -431,10 +435,10 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -446,10 +450,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -500,10 +504,10 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/splunk/controller/statefulset.go b/pkg/splunk/controller/statefulset.go index 15b31361b..c43938e17 100644 --- a/pkg/splunk/controller/statefulset.go +++ b/pkg/splunk/controller/statefulset.go @@ -351,7 +351,10 @@ func DeleteReferencesToAutomatedMCIfExists(ctx context.Context, client splcommon // isCurrentCROwner returns true if current CR is the ONLY owner of the automated MC func isCurrentCROwner(cr splcommon.MetaObject, currentOwners []metav1.OwnerReference) bool { - return reflect.DeepEqual(currentOwners[0].UID, cr.GetUID()) + // adding extra verification as unit test cases fails since fakeclient do not set UID + return reflect.DeepEqual(currentOwners[0].UID, cr.GetUID()) && + (currentOwners[0].Kind == cr.GetObjectKind().GroupVersionKind().Kind) && + (currentOwners[0].Name == cr.GetName()) } // IsStatefulSetScalingUpOrDown checks if we are currently scaling up or down diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index fad4c23b8..c5d5eea0d 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -50,7 +50,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyClusterManager") eventPublisher, _ := newK8EventPublisher(client, cr) - + cr.Kind = "ClusterManager" if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) } @@ -124,7 +124,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, // check if deletion has been requested if cr.ObjectMeta.DeletionTimestamp != nil { if cr.Spec.MonitoringConsoleRef.Name != "" { - extraEnv, _ := VerifyCMisMultisite(ctx, cr, namespaceScopedSecret) + extraEnv, _ := VerifyCMisMultisiteCall(ctx, cr, namespaceScopedSecret) _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, extraEnv, false) if err != nil { return result, err @@ -174,14 +174,14 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, } //make changes to respective mc configmap when changing/removing mcRef from spec - extraEnv, err := VerifyCMisMultisite(ctx, cr, namespaceScopedSecret) + extraEnv, err := VerifyCMisMultisiteCall(ctx, cr, namespaceScopedSecret) err = validateMonitoringConsoleRef(ctx, client, statefulSet, extraEnv) if err != nil { return result, err } // check if the ClusterManager is ready for version upgrade, if required - continueReconcile, err := isClusterManagerReadyForUpgrade(ctx, client, cr) + continueReconcile, err := UpgradePathValidation(ctx, client, cr, cr.Spec.CommonSplunkSpec, nil) if err != nil || !continueReconcile { return result, err } @@ -431,8 +431,8 @@ func getClusterManagerList(ctx context.Context, c splcommon.ControllerClient, cr return numOfObjects, nil } -// VerifyCMisMultisite checks if its a multisite -func VerifyCMisMultisite(ctx context.Context, cr *enterpriseApi.ClusterManager, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { +// VerifyCMisMultisite checks if its a multisite used also in mock +var VerifyCMisMultisiteCall = func(ctx context.Context, cr *enterpriseApi.ClusterManager, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { var err error reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("Verify if Multisite Indexer Cluster").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) @@ -450,68 +450,6 @@ func VerifyCMisMultisite(ctx context.Context, cr *enterpriseApi.ClusterManager, return extraEnv, err } -// isClusterManagerReadyForUpgrade checks if ClusterManager can be upgraded if a version upgrade is in-progress -// No-operation otherwise; returns bool, err accordingly -func isClusterManagerReadyForUpgrade(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isClusterManagerReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) - eventPublisher, _ := newK8EventPublisher(c, cr) - - // check if a LicenseManager is attached to the instance - licenseManagerRef := cr.Spec.LicenseManagerRef - if licenseManagerRef.Name == "" { - return true, nil - } - - namespacedName := types.NamespacedName{ - Namespace: cr.GetNamespace(), - Name: GetSplunkStatefulsetName(SplunkClusterManager, cr.GetName()), - } - - // check if the stateful set is created at this instance - statefulSet := &appsv1.StatefulSet{} - err := c.Get(ctx, namespacedName, statefulSet) - if err != nil && k8serrors.IsNotFound(err) { - return true, nil - } - - namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: licenseManagerRef.Name} - licenseManager := &enterpriseApi.LicenseManager{} - - // get the license manager referred in cluster manager - err = c.Get(ctx, namespacedName, licenseManager) - if err != nil { - if k8serrors.IsNotFound(err) { - return true, nil - } - eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not find the License Manager. Reason %v", err)) - scopedLog.Error(err, "Unable to get licenseManager") - return false, err - } - - lmImage, err := getCurrentImage(ctx, c, cr, SplunkLicenseManager) - if err != nil { - eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the License Manager Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get licenseManager current image") - return false, err - } - - cmImage, err := getCurrentImage(ctx, c, cr, SplunkClusterManager) - if err != nil { - eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager current image") - return false, err - } - - // check if an image upgrade is happening and whether LM has finished updating yet, return false to stop - // further reconcile operations on CM until LM is ready - if (cr.Spec.Image != cmImage) && (licenseManager.Status.Phase != enterpriseApi.PhaseReady || lmImage != cr.Spec.Image) { - return false, nil - } - - return true, nil -} - // changeClusterManagerAnnotations updates the splunk/image-tag field of the ClusterManager annotations to trigger the reconcile loop // on update, and returns error if something is wrong func changeClusterManagerAnnotations(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.LicenseManager) error { diff --git a/pkg/splunk/enterprise/clustermanager_test.go b/pkg/splunk/enterprise/clustermanager_test.go index fef2ac8bc..ec30cf103 100644 --- a/pkg/splunk/enterprise/clustermanager_test.go +++ b/pkg/splunk/enterprise/clustermanager_test.go @@ -138,6 +138,7 @@ func TestApplyClusterManager(t *testing.T) { } c := spltest.NewMockClient() _ = errors.New(splcommon.Rerr) + current.Kind = "ClusterManager" _, err := ApplyClusterManager(ctx, c, ¤t) if err == nil { t.Errorf("Expected error") @@ -204,6 +205,7 @@ func TestApplyClusterManager(t *testing.T) { }, } + current.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, ¤t) if err == nil { t.Errorf("Expected error") @@ -220,6 +222,7 @@ func TestApplyClusterManager(t *testing.T) { current.Spec.SmartStore.VolList[0].SecretRef = "s3-secret" current.Status.SmartStore.VolList[0].SecretRef = "s3-secret" current.Status.ResourceRevMap["s3-secret"] = "v2" + current.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, ¤t) if err == nil { t.Errorf("Expected error") @@ -234,6 +237,7 @@ func TestApplyClusterManager(t *testing.T) { c.Create(ctx, &cmap) current.Spec.SmartStore.VolList[0].SecretRef = "" current.Spec.SmartStore.Defaults.IndexAndGlobalCommonSpec.VolName = "msos_s2s3_vol" + current.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, ¤t) if err != nil { t.Errorf("Don't expected error here") @@ -290,6 +294,7 @@ func TestApplyClusterManager(t *testing.T) { }, }, } + current.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, ¤t) if err == nil { t.Errorf("Expected error") @@ -307,6 +312,7 @@ func TestApplyClusterManager(t *testing.T) { } rerr := errors.New(splcommon.Rerr) c.InduceErrorKind[splcommon.MockClientInduceErrorGet] = rerr + current.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, ¤t) if err == nil { t.Errorf("Expected error") @@ -583,6 +589,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { } // Without S3 keys, ApplyClusterManager should fail + current.Kind = "ClusterManager" _, err := ApplyClusterManager(ctx, client, ¤t) if err == nil { t.Errorf("ApplyClusterManager should fail without S3 secrets configured") @@ -612,6 +619,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *spltest.MockClient, cr interface{}) error { + current.Kind = "ClusterManager" _, err := ApplyClusterManager(context.Background(), c, cr.(*enterpriseApi.ClusterManager)) return err } @@ -639,6 +647,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { spltest.ReconcileTesterWithoutRedundantCheck(t, "TestApplyClusterManagerWithSmartstore-0", ¤t, revised, createCalls, updateCalls, reconcile, true, secret, &smartstoreConfigMap, ss, pod) current.Status.BundlePushTracker.NeedToPushManagerApps = true + current.Kind = "ClusterManager" if _, err = ApplyClusterManager(context.Background(), client, ¤t); err != nil { t.Errorf("ApplyClusterManager() should not have returned error") } @@ -866,13 +875,14 @@ func TestAppFrameworkApplyClusterManagerShouldNotFail(t *testing.T) { t.Errorf(err.Error()) } + cm.Kind = "ClusterManager" _, err = ApplyClusterManager(context.Background(), client, &cm) if err != nil { t.Errorf("ApplyClusterManager should not have returned error here.") } } -func TestApplyCLusterManagerDeletion(t *testing.T) { +func TestApplyClusterManagerDeletion(t *testing.T) { ctx := context.TODO() cm := enterpriseApi.ClusterManager{ ObjectMeta: metav1.ObjectMeta{ @@ -960,7 +970,7 @@ func TestApplyCLusterManagerDeletion(t *testing.T) { if err != nil { t.Errorf("Unable to create download directory for apps :%s", splcommon.AppDownloadVolume) } - + cm.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, &cm) if err != nil { t.Errorf("ApplyClusterManager should not have returned error here.") @@ -1421,6 +1431,14 @@ func TestIsClusterManagerReadyForUpgrade(t *testing.T) { if err != nil { t.Errorf("applyLicenseManager should not have returned error; err=%v", err) } + namespacedName := types.NamespacedName{ + Name: "test", + Namespace: "test", + } + err = client.Get(ctx, namespacedName, &lm) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } lm.Status.Phase = enterpriseApi.PhaseReady err = client.Status().Update(ctx, &lm) if err != nil { @@ -1448,18 +1466,26 @@ func TestIsClusterManagerReadyForUpgrade(t *testing.T) { }, } + cm.Kind = "ClusterManager" err = client.Create(ctx, &cm) _, err = ApplyClusterManager(ctx, client, &cm) if err != nil { t.Errorf("applyClusterManager should not have returned error; err=%v", err) } - cm.Spec.Image = "splunk2" + // create pods for license manager + lm.Status.TelAppInstalled = true lm.Spec.Image = "splunk2" + createPods(t, ctx, client, "license-manager", fmt.Sprintf("splunk-%s-license-manager-0", lm.Name), lm.Namespace, lm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-license-manager", lm.Name), lm.Namespace) + // now the statefulset image in spec is updated to splunk2 + _, err = ApplyLicenseManager(ctx, client, &lm) + + // now the statefulset and license manager both should be in ready state _, err = ApplyLicenseManager(ctx, client, &lm) clusterManager := &enterpriseApi.ClusterManager{} - namespacedName := types.NamespacedName{ + namespacedName = types.NamespacedName{ Name: cm.Name, Namespace: cm.Namespace, } @@ -1467,8 +1493,13 @@ func TestIsClusterManagerReadyForUpgrade(t *testing.T) { if err != nil { t.Errorf("changeClusterManagerAnnotations should not have returned error=%v", err) } + clusterManager.Spec.Image = "splunk2" + err = client.Update(ctx, clusterManager) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } - check, err := isClusterManagerReadyForUpgrade(ctx, client, clusterManager) + check, err := UpgradePathValidation(ctx, client, clusterManager, clusterManager.Spec.CommonSplunkSpec, nil) if err != nil { t.Errorf("Unexpected upgradeScenario error %v", err) @@ -1491,6 +1522,7 @@ func TestChangeClusterManagerAnnotations(t *testing.T) { Spec: enterpriseApi.LicenseManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ + Image: "splunk/splunk:latest", ImagePullPolicy: "Always", }, Volumes: []corev1.Volume{}, @@ -1506,6 +1538,7 @@ func TestChangeClusterManagerAnnotations(t *testing.T) { Spec: enterpriseApi.ClusterManagerSpec{ CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ Spec: enterpriseApi.Spec{ + Image: "splunk/splunk:latest", ImagePullPolicy: "Always", }, Volumes: []corev1.Volume{}, @@ -1527,12 +1560,44 @@ func TestChangeClusterManagerAnnotations(t *testing.T) { if err != nil { t.Errorf("applyLicenseManager should not have returned error; err=%v", err) } + + namespacedName := types.NamespacedName{ + Name: lm.Name, + Namespace: lm.Namespace, + } + err = client.Get(ctx, namespacedName, lm) + if err != nil { + t.Errorf("changeLicenseManagerAnnotations should not have returned error=%v", err) + } + + // create pods for license manager + createPods(t, ctx, client, "license-manager", fmt.Sprintf("splunk-%s-license-manager-0", lm.Name), lm.Namespace, lm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-license-manager", lm.Name), lm.Namespace) + lm.Status.TelAppInstalled = true + // create license manager statefulset + _, err = ApplyLicenseManager(ctx, client, lm) + if err != nil { + t.Errorf("ApplyLicenseManager should not have returned error; err=%v", err) + } + + err = client.Get(ctx, namespacedName, lm) + if err != nil { + t.Errorf("changeLicenseManagerAnnotations should not have returned error=%v", err) + } + lm.Status.Phase = enterpriseApi.PhaseReady err = client.Status().Update(ctx, lm) if err != nil { t.Errorf("Unexpected update pod %v", err) debug.PrintStack() } + + VerifyCMisMultisiteCall = func(ctx context.Context, cr *enterpriseApi.ClusterManager, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { + extraEnv := getClusterManagerExtraEnv(cr, &cr.Spec.CommonSplunkSpec) + return extraEnv, err + } + + cm.Kind = "ClusterManager" client.Create(ctx, cm) _, err = ApplyClusterManager(ctx, client, cm) if err != nil { @@ -1544,7 +1609,7 @@ func TestChangeClusterManagerAnnotations(t *testing.T) { t.Errorf("changeClusterManagerAnnotations should not have returned error=%v", err) } clusterManager := &enterpriseApi.ClusterManager{} - namespacedName := types.NamespacedName{ + namespacedName = types.NamespacedName{ Name: cm.Name, Namespace: cm.Namespace, } @@ -1672,6 +1737,7 @@ func TestClusterManagerWitReadyState(t *testing.T) { // simulate create stateful set c.Create(ctx, statefulset) + clustermanager.Kind = "ClusterManager" // simulate create clustermanager instance before reconcilation c.Create(ctx, clustermanager) @@ -1685,6 +1751,12 @@ func TestClusterManagerWitReadyState(t *testing.T) { Namespace: clustermanager.Namespace, } + // cluster manager + err = c.Get(ctx, namespacedName, clustermanager) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + // simulate Ready state clustermanager.Status.Phase = enterpriseApi.PhaseReady clustermanager.Spec.ServiceTemplate.Annotations = map[string]string{ @@ -1711,6 +1783,7 @@ func TestClusterManagerWitReadyState(t *testing.T) { } // call reconciliation + clustermanager.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, clustermanager) if err != nil { t.Errorf("Unexpected error while running reconciliation for cluster manager with app framework %v", err) @@ -1829,6 +1902,7 @@ func TestClusterManagerWitReadyState(t *testing.T) { } // call reconciliation + clustermanager.Kind = "ClusterManager" _, err = ApplyClusterManager(ctx, c, clustermanager) if err != nil { t.Errorf("Unexpected error while running reconciliation for cluster manager with app framework %v", err) diff --git a/pkg/splunk/enterprise/clustermaster.go b/pkg/splunk/enterprise/clustermaster.go index 8782e49b3..ef746dfd1 100644 --- a/pkg/splunk/enterprise/clustermaster.go +++ b/pkg/splunk/enterprise/clustermaster.go @@ -48,6 +48,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyClusterMaster") eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "ClusterMaster" if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) diff --git a/pkg/splunk/enterprise/clustermaster_test.go b/pkg/splunk/enterprise/clustermaster_test.go index 155570478..79efec942 100644 --- a/pkg/splunk/enterprise/clustermaster_test.go +++ b/pkg/splunk/enterprise/clustermaster_test.go @@ -296,6 +296,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) { } // Without S3 keys, ApplyClusterManager should fail + current.Kind = "ClusterMaster" _, err := ApplyClusterMaster(ctx, client, ¤t) if err == nil { t.Errorf("ApplyClusterMaster should fail without S3 secrets configured") @@ -351,6 +352,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) { spltest.ReconcileTesterWithoutRedundantCheck(t, "TestApplyClusterMasterWithSmartstore-0", ¤t, revised, createCalls, updateCalls, reconcile, true, secret, &smartstoreConfigMap, ss, pod) + current.Kind = "ClusterMaster" current.Status.BundlePushTracker.NeedToPushMasterApps = true if _, err = ApplyClusterMaster(context.Background(), client, ¤t); err != nil { t.Errorf("ApplyClusterMaster() should not have returned error") @@ -572,6 +574,7 @@ func TestAppFrameworkApplyClusterMasterShouldNotFail(t *testing.T) { t.Errorf(err.Error()) } + cm.Kind = "ClusterMaster" _, err = ApplyClusterMaster(context.Background(), client, &cm) if err != nil { t.Errorf("ApplyClusterMaster should not have returned error here.") @@ -667,6 +670,7 @@ func TestApplyCLusterMasterDeletion(t *testing.T) { t.Errorf("Unable to create download directory for apps :%s", splcommon.AppDownloadVolume) } + cm.Kind = "ClusterMaster" _, err = ApplyClusterMaster(ctx, c, &cm) if err != nil { t.Errorf("ApplyClusterMaster should not have returned error here.") @@ -1220,7 +1224,11 @@ func TestClusterMasterWitReadyState(t *testing.T) { Name: clustermaster.Name, Namespace: clustermaster.Namespace, } - + err = c.Get(ctx, namespacedName, clustermaster) + if err != nil { + t.Errorf("Unexpected get cluster master %v", err) + debug.PrintStack() + } // simulate Ready state clustermaster.Status.Phase = enterpriseApi.PhaseReady clustermaster.Spec.ServiceTemplate.Annotations = map[string]string{ diff --git a/pkg/splunk/enterprise/events.go b/pkg/splunk/enterprise/events.go index f05917f18..ebdc13d62 100644 --- a/pkg/splunk/enterprise/events.go +++ b/pkg/splunk/enterprise/events.go @@ -55,12 +55,19 @@ func (k *K8EventPublisher) publishEvent(ctx context.Context, eventType, reason, // based on the custom resource instance type find name, type and create new event switch v := k.instance.(type) { case *enterpriseApi.Standalone: + event = v.NewEvent(eventType, reason, message) case *enterpriseApiV3.LicenseMaster: + event = v.NewEvent(eventType, reason, message) case *enterpriseApi.LicenseManager: + event = v.NewEvent(eventType, reason, message) case *enterpriseApi.IndexerCluster: + event = v.NewEvent(eventType, reason, message) case *enterpriseApi.ClusterManager: + event = v.NewEvent(eventType, reason, message) case *enterpriseApiV3.ClusterMaster: + event = v.NewEvent(eventType, reason, message) case *enterpriseApi.MonitoringConsole: + event = v.NewEvent(eventType, reason, message) case *enterpriseApi.SearchHeadCluster: event = v.NewEvent(eventType, reason, message) default: diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index c52528713..740cb3fac 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "regexp" + "sort" "strconv" "strings" "time" @@ -55,6 +56,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyIndexerClusterManager").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "IndexerCluster" // validate and updates defaults for CR err := validateIndexerClusterSpec(ctx, client, cr) @@ -93,7 +95,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller Name: cr.Spec.ClusterManagerRef.Name, } managerIdxCluster := &enterpriseApi.ClusterManager{} - err = client.Get(context.TODO(), namespacedName, managerIdxCluster) + err = client.Get(ctx, namespacedName, managerIdxCluster) if err == nil { // when user creates both cluster manager and index cluster yaml file at the same time // cluser manager status is not yet set so it will be blank @@ -194,6 +196,13 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller } } + // check if the IndexerCluster is ready for version upgrade + cr.Kind = "IndexerCluster" + continueReconcile, err := UpgradePathValidation(ctx, client, cr, cr.Spec.CommonSplunkSpec, &mgr) + if err != nil || !continueReconcile { + return result, err + } + // check if version upgrade is set if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) @@ -205,7 +214,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mitmatch for indexer clustre and indexer container, delete statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mismatch for indexer cluster and indexer container, delete statefulset failed. Error=%s", err.Error())) eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("%s-%s, %s-%s", "indexer-image", cr.Spec.Image, "container-image", statefulSet.Spec.Template.Spec.Containers[0].Image)) return result, err } @@ -299,6 +308,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyIndexerCluster") eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "IndexerCluster" // validate and updates defaults for CR err := validateIndexerClusterSpec(ctx, client, cr) @@ -337,7 +347,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, Name: cr.Spec.ClusterMasterRef.Name, } managerIdxCluster := &enterpriseApiV3.ClusterMaster{} - err = client.Get(context.TODO(), namespacedName, managerIdxCluster) + err = client.Get(ctx, namespacedName, managerIdxCluster) if err == nil { // when user creates both cluster manager and index cluster yaml file at the same time // cluser master status is not yet set so it will be blank @@ -438,6 +448,13 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, } } + // check if the IndexerCluster is ready for version upgrade + cr.Kind = "IndexerCluster" + continueReconcile, err := UpgradePathValidation(ctx, client, cr, cr.Spec.CommonSplunkSpec, &mgr) + if err != nil || !continueReconcile { + return result, err + } + // check if version upgrade is set if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) @@ -939,6 +956,16 @@ func (mgr *indexerClusterPodManager) verifyRFPeers(ctx context.Context, c splcom return nil } +var GetClusterManagerInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager) (*splclient.ClusterManagerInfo, error) { + c := mgr.getClusterManagerClient(ctx) + return c.GetClusterManagerInfo() +} + +var GetClusterManagerPeersCall = func(ctx context.Context, mgr *indexerClusterPodManager) (map[string]splclient.ClusterManagerPeerInfo, error) { + c := mgr.getClusterManagerClient(ctx) + return c.GetClusterManagerPeers() +} + // updateStatus for indexerClusterPodManager uses the REST API to update the status for an IndexerCluster custom resource func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulSet *appsv1.StatefulSet) error { mgr.cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas @@ -952,8 +979,7 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS } // get indexer cluster info from cluster manager if it's ready - c := mgr.getClusterManagerClient(ctx) - clusterInfo, err := c.GetClusterManagerInfo() + clusterInfo, err := GetClusterManagerInfoCall(ctx, mgr) if err != nil { return err } @@ -963,7 +989,7 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS mgr.cr.Status.MaintenanceMode = clusterInfo.MaintenanceMode // get peer information from cluster manager - peers, err := c.GetClusterManagerPeers() + peers, err := GetClusterManagerPeersCall(ctx, mgr) if err != nil { return err } @@ -1067,6 +1093,43 @@ func RetrieveCMSpec(ctx context.Context, client splcommon.ControllerClient, cr * return "", nil } +func getIndexerClusterSortedSiteList(ctx context.Context, c splcommon.ControllerClient, ref corev1.ObjectReference, indexerList enterpriseApi.IndexerClusterList) (enterpriseApi.IndexerClusterList, error) { + + namespaceList := enterpriseApi.IndexerClusterList{} + for _, v := range indexerList.Items { + if v.Spec.ClusterManagerRef == ref { + namespaceList.Items = append(namespaceList.Items, v) + } + } + + sort.SliceStable(namespaceList.Items, func(i, j int) bool { + return getSiteName(ctx, c, &namespaceList.Items[i]) < getSiteName(ctx, c, &namespaceList.Items[j]) + }) + + return namespaceList, nil +} + +func getSiteName(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.IndexerCluster) string { + defaults := cr.Spec.Defaults + // site name starts with site: + pattern := `site:\s+(\w+)` + + // Compile the regular expression pattern + re := regexp.MustCompile(pattern) + + // Find the first match in the input string + match := re.FindStringSubmatch(defaults) + + var extractedValue string + if len(match) > 1 { + // Extracted value is stored in the second element of the match array + extractedValue := match[1] + return extractedValue + } + + return extractedValue +} + // Tells if there is an image migration from 8.x.x to 9.x.x func imageUpdatedTo9(previousImage string, currentImage string) bool { // If there is no colon, version can't be detected diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index 1eeda101b..c97450a97 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -735,7 +735,8 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.StatefulSet-test-splunk-stack1"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, - {MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, + //{MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, + {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-0"}, } @@ -750,7 +751,7 @@ func TestIndexerClusterPodManager(t *testing.T) { listmockCall := []spltest.MockFuncCall{ {ListOpts: listOpts}} - wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} + wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} // test 1 ready pod mockHandlers := []spltest.MockHTTPHandler{ @@ -802,6 +803,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, @@ -815,6 +817,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-0"}, } mockHandlers = []spltest.MockHTTPHandler{mockHandlers[0], mockHandlers[1]} @@ -841,7 +844,7 @@ func TestIndexerClusterPodManager(t *testing.T) { statefulSet.Status.Replicas = 2 statefulSet.Status.ReadyReplicas = 2 statefulSet.Status.UpdatedReplicas = 2 - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4]}, "Create": {funcCalls[1]}} method = "indexerClusterPodManager.Update(Pod Not Found)" indexerClusterPodManagerUpdateTester(t, method, mockHandlers, 1, enterpriseApi.PhaseScalingDown, statefulSet, wantCalls, nil, statefulSet, pod) @@ -864,6 +867,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"}, } @@ -1576,7 +1580,11 @@ func TestIndexerClusterWithReadyState(t *testing.T) { Name: clustermanager.Name, Namespace: clustermanager.Namespace, } - + err := c.Get(ctx, namespacedName, clustermanager) + if err != nil { + t.Errorf("Unexpected get cluster manager %v", err) + debug.PrintStack() + } clustermanager.Status.Phase = enterpriseApi.PhaseReady clustermanager.Spec.ServiceTemplate.Annotations = map[string]string{ "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", @@ -1589,7 +1597,7 @@ func TestIndexerClusterWithReadyState(t *testing.T) { "app.kubernetes.io/name": "cluster-manager", "app.kubernetes.io/part-of": "splunk-test-cluster-manager", } - err := c.Status().Update(ctx, clustermanager) + err = c.Status().Update(ctx, clustermanager) if err != nil { t.Errorf("Unexpected error while running reconciliation for cluster manager with app framework %v", err) debug.PrintStack() @@ -1755,6 +1763,24 @@ func TestIndexerClusterWithReadyState(t *testing.T) { // simulate create clustermanager instance before reconcilation c.Create(ctx, indexercluster) + GetClusterInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager, mockCall bool) (*splclient.ClusterInfo, error) { + cinfo := &splclient.ClusterInfo{ + MultiSite: "false", + } + return cinfo, nil + } + GetClusterManagerPeersCall = func(ctx context.Context, mgr *indexerClusterPodManager) (map[string]splclient.ClusterManagerPeerInfo, error) { + response := map[string]splclient.ClusterManagerPeerInfo{ + "splunk-test-indexer-0": { + ID: "site-1", + Status: "Up", + ActiveBundleID: "1", + BucketCount: 10, + Searchable: true, + }, + } + return response, err + } _, err = ApplyIndexerClusterManager(ctx, c, indexercluster) if err != nil { t.Errorf("Unexpected error while running reconciliation for indexer cluster %v", err) @@ -1765,7 +1791,11 @@ func TestIndexerClusterWithReadyState(t *testing.T) { Name: indexercluster.Name, Namespace: indexercluster.Namespace, } - + err = c.Get(ctx, namespacedName, indexercluster) + if err != nil { + t.Errorf("Unexpected get indexer cluster %v", err) + debug.PrintStack() + } // simulate Ready state indexercluster.Status.Phase = enterpriseApi.PhaseReady indexercluster.Spec.ServiceTemplate.Annotations = map[string]string{ diff --git a/pkg/splunk/enterprise/licensemanager.go b/pkg/splunk/enterprise/licensemanager.go index ad572de10..aa03a1b9c 100644 --- a/pkg/splunk/enterprise/licensemanager.go +++ b/pkg/splunk/enterprise/licensemanager.go @@ -46,6 +46,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyLicenseManager") eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "LicenseManager" // validate and updates defaults for CR err := validateLicenseManagerSpec(ctx, client, cr) diff --git a/pkg/splunk/enterprise/licensemanager_test.go b/pkg/splunk/enterprise/licensemanager_test.go index 7cb3c46b2..f5d0885fd 100644 --- a/pkg/splunk/enterprise/licensemanager_test.go +++ b/pkg/splunk/enterprise/licensemanager_test.go @@ -1075,7 +1075,11 @@ func TestLicenseManagerWithReadyState(t *testing.T) { Name: licensemanager.Name, Namespace: licensemanager.Namespace, } - + err = c.Get(ctx, namespacedName, licensemanager) + if err != nil { + t.Errorf("Unexpected get license manager %v", err) + debug.PrintStack() + } // simulate Ready state licensemanager.Status.Phase = enterpriseApi.PhaseReady licensemanager.Spec.ServiceTemplate.Annotations = map[string]string{ diff --git a/pkg/splunk/enterprise/monitoringconsole.go b/pkg/splunk/enterprise/monitoringconsole.go index 50c16891a..7bedf70a0 100644 --- a/pkg/splunk/enterprise/monitoringconsole.go +++ b/pkg/splunk/enterprise/monitoringconsole.go @@ -50,6 +50,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplyMonitoringConsole") eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "MonitoringConsole" if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) @@ -138,7 +139,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie } // check if the Monitoring Console is ready for version upgrade, if required - continueReconcile, err := isMonitoringConsoleReadyForUpgrade(ctx, client, cr) + continueReconcile, err := UpgradePathValidation(ctx, client, cr, cr.Spec.CommonSplunkSpec, nil) if err != nil || !continueReconcile { return result, err } @@ -155,6 +156,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie if cr.Status.Phase == enterpriseApi.PhaseReady { finalResult := handleAppFrameworkActivity(ctx, client, cr, &cr.Status.AppContext, &cr.Spec.AppFrameworkConfig) result = *finalResult + } // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. // Implies that Requeue is true, there is no need to set Requeue to true at the same time as RequeueAfter. @@ -363,65 +365,6 @@ func DeleteURLsConfigMap(revised *corev1.ConfigMap, crName string, newURLs []cor } } -// isMonitoringConsoleReadyForUpgrade checks if MonitoringConsole can be upgraded if a version upgrade is in-progress -// No-operation otherwise; returns bool, err accordingly -func isMonitoringConsoleReadyForUpgrade(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.MonitoringConsole) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isMonitoringConsoleReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) - eventPublisher, _ := newK8EventPublisher(c, cr) - - // check if a LicenseManager is attached to the instance - clusterManagerRef := cr.Spec.ClusterManagerRef - if clusterManagerRef.Name == "" { - return true, nil - } - - namespacedName := types.NamespacedName{ - Namespace: cr.GetNamespace(), - Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetName()), - } - - // check if the stateful set is created at this instance - statefulSet := &appsv1.StatefulSet{} - err := c.Get(ctx, namespacedName, statefulSet) - if err != nil && k8serrors.IsNotFound(err) { - return true, nil - } - - namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: clusterManagerRef.Name} - clusterManager := &enterpriseApi.ClusterManager{} - - // get the cluster manager referred in monitoring console - err = c.Get(ctx, namespacedName, clusterManager) - if err != nil { - eventPublisher.Warning(ctx, "isMonitoringConsoleReadyForUpgrade", fmt.Sprintf("Could not find the Cluster Manager. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager") - return true, err - } - - cmImage, err := getCurrentImage(ctx, c, cr, SplunkClusterManager) - if err != nil { - eventPublisher.Warning(ctx, "isMonitoringConsoleReadyForUpgrade", fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager current image") - return false, err - } - - mcImage, err := getCurrentImage(ctx, c, cr, SplunkMonitoringConsole) - if err != nil { - eventPublisher.Warning(ctx, "isMonitoringConsolerReadyForUpgrade", fmt.Sprintf("Could not get the Monitoring Console Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get monitoring console current image") - return false, err - } - - // check if an image upgrade is happening and whether CM has finished updating yet, return false to stop - // further reconcile operations on MC until CM is ready - if (cr.Spec.Image != mcImage) && (clusterManager.Status.Phase != enterpriseApi.PhaseReady || cmImage != cr.Spec.Image) { - return false, nil - } - - return true, nil -} - // changeMonitoringConsoleAnnotations updates the splunk/image-tag field of the MonitoringConsole annotations to trigger the reconcile loop // on update, and returns error if something is wrong. func changeMonitoringConsoleAnnotations(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) error { diff --git a/pkg/splunk/enterprise/monitoringconsole_test.go b/pkg/splunk/enterprise/monitoringconsole_test.go index b30c93c6f..84186ef31 100644 --- a/pkg/splunk/enterprise/monitoringconsole_test.go +++ b/pkg/splunk/enterprise/monitoringconsole_test.go @@ -72,6 +72,7 @@ func TestApplyMonitoringConsole(t *testing.T) { {MetaName: "*v1.ConfigMap-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, {MetaName: "*v4.MonitoringConsole-test-stack1"}, {MetaName: "*v4.MonitoringConsole-test-stack1"}, } @@ -81,15 +82,19 @@ func TestApplyMonitoringConsole(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Service-test-splunk-stack1-monitoring-console-headless"}, {MetaName: "*v1.Service-test-splunk-stack1-monitoring-console-service"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-stack1-monitoring-console-secret-v1"}, + {MetaName: "*v1.ConfigMap-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.ConfigMap-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-monitoring-console"}, + {MetaName: "*v4.MonitoringConsole-test-stack1"}, {MetaName: "*v4.MonitoringConsole-test-stack1"}, } @@ -875,7 +880,11 @@ func TestMonitoringConsoleWithReadyState(t *testing.T) { Name: monitoringconsole.Name, Namespace: monitoringconsole.Namespace, } - + err = c.Get(ctx, namespacedName, monitoringconsole) + if err != nil { + t.Errorf("Unexpected get monitoring console. Error = %v", err) + debug.PrintStack() + } // simulate Ready state monitoringconsole.Status.Phase = enterpriseApi.PhaseReady monitoringconsole.Spec.ServiceTemplate.Annotations = map[string]string{ @@ -1101,96 +1110,6 @@ func TestGetMonitoringConsoleList(t *testing.T) { } } -func TestIsMonitoringConsoleReadyForUpgrade(t *testing.T) { - ctx := context.TODO() - - builder := fake.NewClientBuilder() - client := builder.Build() - utilruntime.Must(enterpriseApi.AddToScheme(clientgoscheme.Scheme)) - - // Create Cluster Manager - cm := enterpriseApi.ClusterManager{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test", - }, - Spec: enterpriseApi.ClusterManagerSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: "splunk/splunk:latest", - }, - Volumes: []corev1.Volume{}, - MonitoringConsoleRef: corev1.ObjectReference{ - Name: "test", - }, - }, - }, - } - - err := client.Create(ctx, &cm) - _, err = ApplyClusterManager(ctx, client, &cm) - if err != nil { - t.Errorf("applyClusterManager should not have returned error; err=%v", err) - } - cm.Status.Phase = enterpriseApi.PhaseReady - err = client.Status().Update(ctx, &cm) - if err != nil { - t.Errorf("Unexpected status update %v", err) - debug.PrintStack() - } - - // Create Monitoring Console - mc := enterpriseApi.MonitoringConsole{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test", - Namespace: "test", - }, - Spec: enterpriseApi.MonitoringConsoleSpec{ - CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ - Spec: enterpriseApi.Spec{ - ImagePullPolicy: "Always", - Image: "splunk/splunk:latest", - }, - Volumes: []corev1.Volume{}, - ClusterManagerRef: corev1.ObjectReference{ - Name: "test", - }, - }, - }, - } - - err = client.Create(ctx, &mc) - _, err = ApplyMonitoringConsole(ctx, client, &mc) - if err != nil { - t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err) - } - - mc.Spec.Image = "splunk2" - cm.Spec.Image = "splunk2" - _, err = ApplyClusterManager(ctx, client, &cm) - - monitoringConsole := &enterpriseApi.MonitoringConsole{} - namespacedName := types.NamespacedName{ - Name: cm.Name, - Namespace: cm.Namespace, - } - err = client.Get(ctx, namespacedName, monitoringConsole) - if err != nil { - t.Errorf("isMonitoringConsoleReadyForUpgrade should not have returned error=%v", err) - } - - check, err := isMonitoringConsoleReadyForUpgrade(ctx, client, monitoringConsole) - - if err != nil { - t.Errorf("Unexpected upgradeScenario error %v", err) - } - - if !check { - t.Errorf("isMonitoringConsoleReadyForUpgrade: MC should be ready for upgrade") - } -} - func TestChangeMonitoringConsoleAnnotations(t *testing.T) { ctx := context.TODO() @@ -1239,6 +1158,14 @@ func TestChangeMonitoringConsoleAnnotations(t *testing.T) { if err != nil { t.Errorf("applyClusterManager should not have returned error; err=%v", err) } + namespacedName := types.NamespacedName{ + Name: cm.Name, + Namespace: cm.Namespace, + } + err = client.Get(ctx, namespacedName, cm) + if err != nil { + t.Errorf("changeMonitoringConsoleAnnotations should not have returned error=%v", err) + } cm.Status.Phase = enterpriseApi.PhaseReady err = client.Status().Update(ctx, cm) if err != nil { @@ -1256,7 +1183,7 @@ func TestChangeMonitoringConsoleAnnotations(t *testing.T) { t.Errorf("changeMonitoringConsoleAnnotations should not have returned error=%v", err) } monitoringConsole := &enterpriseApi.MonitoringConsole{} - namespacedName := types.NamespacedName{ + namespacedName = types.NamespacedName{ Name: cm.Name, Namespace: cm.Namespace, } diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index 8419bb90d..7006ed0c9 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -49,6 +49,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie reqLogger := log.FromContext(ctx) scopedLog := reqLogger.WithName("ApplySearchHeadCluster") eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "SearchHeadCluster" // validate and updates defaults for CR err := validateSearchHeadClusterSpec(ctx, client, cr) @@ -160,6 +161,11 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie return result, err } + continueReconcile, err := UpgradePathValidation(ctx, client, cr, cr.Spec.CommonSplunkSpec, nil) + if err != nil || !continueReconcile { + return result, err + } + deployerManager := splctrl.DefaultStatefulSetPodManager{} phase, err := deployerManager.Update(ctx, client, statefulSet, 1) if err != nil { @@ -179,7 +185,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie return result, err } - mgr := newSerachHeadClusterPodManager(client, scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient) + mgr := newSearchHeadClusterPodManager(client, scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient) phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { return result, err @@ -247,7 +253,7 @@ type searchHeadClusterPodManager struct { } // newSerachHeadClusterPodManager function to create pod manager this is added to write unit test case -var newSerachHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { +var newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { return searchHeadClusterPodManager{ log: log, cr: cr, @@ -553,6 +559,18 @@ func (mgr *searchHeadClusterPodManager) getClient(ctx context.Context, n int32) return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) } +// GetSearchHeadClusterMemberInfo used in mocking this function +var GetSearchHeadClusterMemberInfo = func(ctx context.Context, mgr *searchHeadClusterPodManager, n int32) (*splclient.SearchHeadClusterMemberInfo, error) { + c := mgr.getClient(ctx, n) + return c.GetSearchHeadClusterMemberInfo() +} + +// GetSearchHeadCaptainInfo used in mocking this function +var GetSearchHeadCaptainInfo = func(ctx context.Context, mgr *searchHeadClusterPodManager, n int32) (*splclient.SearchHeadCaptainInfo, error) { + c := mgr.getClient(ctx, n) + return c.GetSearchHeadCaptainInfo() +} + // updateStatus for searchHeadClusterPodManager uses the REST API to update the status for a SearcHead custom resource func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statefulSet *appsv1.StatefulSet) error { // populate members status using REST API to get search head cluster member info @@ -564,10 +582,10 @@ func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statef } gotCaptainInfo := false for n := int32(0); n < statefulSet.Status.Replicas; n++ { - c := mgr.getClient(ctx, n) + //c := mgr.getClient(ctx, n) memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) memberStatus := enterpriseApi.SearchHeadClusterMemberStatus{Name: memberName} - memberInfo, err := c.GetSearchHeadClusterMemberInfo() + memberInfo, err := GetSearchHeadClusterMemberInfo(ctx, mgr, n) if err == nil { memberStatus.Status = memberInfo.Status memberStatus.Adhoc = memberInfo.Adhoc @@ -580,7 +598,7 @@ func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statef if err == nil && !gotCaptainInfo { // try querying captain api; note that this should work on any node - captainInfo, err := c.GetSearchHeadCaptainInfo() + captainInfo, err := GetSearchHeadCaptainInfo(ctx, mgr, n) if err == nil { mgr.cr.Status.Captain = captainInfo.Label mgr.cr.Status.CaptainReady = captainInfo.ServiceReady diff --git a/pkg/splunk/enterprise/searchheadcluster_test.go b/pkg/splunk/enterprise/searchheadcluster_test.go index 1c6f9fd23..ef1a5c7b9 100644 --- a/pkg/splunk/enterprise/searchheadcluster_test.go +++ b/pkg/splunk/enterprise/searchheadcluster_test.go @@ -68,38 +68,51 @@ func TestApplySearchHeadCluster(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, + {MetaName: "*v1.Service-test-splunk-stack1-search-head-headless"}, {MetaName: "*v1.Service-test-splunk-stack1-search-head-service"}, + {MetaName: "*v1.Service-test-splunk-stack1-deployer-service"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-deployer"}, + {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, + {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-stack1-deployer-secret-v1"}, + + {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-deployer"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, + {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-stack1-search-head-secret-v1"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, + {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v4.SearchHeadCluster-test-stack1"}, {MetaName: "*v4.SearchHeadCluster-test-stack1"}, } + createFuncCalls := []spltest.MockFuncCall{ {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Service-test-splunk-stack1-search-head-headless"}, {MetaName: "*v1.Service-test-splunk-stack1-search-head-service"}, + {MetaName: "*v1.Service-test-splunk-stack1-deployer-service"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-deployer"}, + {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-stack1-deployer-secret-v1"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-deployer"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-deployer"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-search-head"}, + {MetaName: "*v1.ConfigMap-test-splunk-test-probe-configmap"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-stack1-search-head-secret-v1"}, @@ -122,8 +135,8 @@ func TestApplySearchHeadCluster(t *testing.T) { listmockCall := []spltest.MockFuncCall{ {ListOpts: listOpts}} - createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[0], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[8], funcCalls[10], funcCalls[11], funcCalls[15], funcCalls[16]}, "Update": {funcCalls[0]}, "List": {listmockCall[0], listmockCall[0]}} - updateCalls := map[string][]spltest.MockFuncCall{"Get": createFuncCalls, "Update": {createFuncCalls[5], createFuncCalls[11]}, "List": {listmockCall[0], listmockCall[0]}} + createCalls := map[string][]spltest.MockFuncCall{"Get": funcCalls, "Create": {funcCalls[0], funcCalls[3], funcCalls[4], funcCalls[5], funcCalls[8], funcCalls[10], funcCalls[12], funcCalls[16], funcCalls[17]}, "Update": {funcCalls[0]}, "List": {listmockCall[0], listmockCall[0]}} + updateCalls := map[string][]spltest.MockFuncCall{"Get": createFuncCalls, "Update": {createFuncCalls[5], createFuncCalls[9]}, "List": {listmockCall[0], listmockCall[0]}} statefulSet := enterpriseApi.SearchHeadCluster{ TypeMeta: metav1.TypeMeta{ Kind: "SearchHeadCluster", @@ -294,7 +307,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { }, } method = "searchHeadClusterPodManager.Update(All pods ready)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseReady, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => transition to detention @@ -307,7 +320,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { }) pod.ObjectMeta.Labels["controller-revision-hash"] = "v0" method = "searchHeadClusterPodManager.Update(Quarantine Pod)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => wait for searches to drain @@ -315,13 +328,13 @@ func TestSearchHeadClusterPodManager(t *testing.T) { mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"status":"Up"`, `"status":"ManualDetention"`, 1) mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":0`, `"active_historical_search_count":1`, 1) method = "searchHeadClusterPodManager.Update(Draining Searches)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => delete pod mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":1`, `"active_historical_search_count":0`, 1) method = "searchHeadClusterPodManager.Update(Delete Pod)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod update finished => release from detention @@ -334,7 +347,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { Body: ``, }) method = "searchHeadClusterPodManager.Update(Release Quarantine)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test scale down => remove member @@ -362,6 +375,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"}, + {MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"}, @@ -1436,7 +1450,7 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) { } // mock new search pod manager - newSerachHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { + newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { return searchHeadClusterPodManager{ log: log, cr: cr, @@ -1542,7 +1556,11 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) { Name: searchheadcluster.Name, Namespace: searchheadcluster.Namespace, } - + err = c.Get(ctx, namespacedName, searchheadcluster) + if err != nil { + t.Errorf("Unexpected get search head cluster. Error=%v", err) + debug.PrintStack() + } // simulate Ready state searchheadcluster.Status.Phase = enterpriseApi.PhaseReady searchheadcluster.Spec.ServiceTemplate.Annotations = map[string]string{ @@ -1869,7 +1887,6 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) { // call reconciliation _, err = ApplySearchHeadCluster(ctx, c, searchheadcluster) if err != nil { - t.Errorf("Unexpected error while running reconciliation for search head cluster with app framework %v", err) - debug.PrintStack() + t.Errorf("Unexpected error while running reconciliation for search head cluster with app framework. Error=%v", err) } } diff --git a/pkg/splunk/enterprise/standalone.go b/pkg/splunk/enterprise/standalone.go index b8ec3d976..5eed285aa 100644 --- a/pkg/splunk/enterprise/standalone.go +++ b/pkg/splunk/enterprise/standalone.go @@ -49,6 +49,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr cr.Status.ResourceRevMap = make(map[string]string) } eventPublisher, _ := newK8EventPublisher(client, cr) + cr.Kind = "Standalone" // validate and updates defaults for CR err := validateStandaloneSpec(ctx, client, cr) diff --git a/pkg/splunk/enterprise/standalone_test.go b/pkg/splunk/enterprise/standalone_test.go index be8903b9f..50491306e 100644 --- a/pkg/splunk/enterprise/standalone_test.go +++ b/pkg/splunk/enterprise/standalone_test.go @@ -1281,7 +1281,11 @@ func TestStandaloneWithReadyState(t *testing.T) { Name: standalone.Name, Namespace: standalone.Namespace, } - + err = c.Get(ctx, namespacedName, &standalone) + if err != nil { + t.Errorf("Unexpected get standalone. Error=%v", err) + debug.PrintStack() + } // simulate Ready state standalone.Status.Phase = enterpriseApi.PhaseReady standalone.Spec.ServiceTemplate.Annotations = map[string]string{ diff --git a/pkg/splunk/enterprise/upgrade.go b/pkg/splunk/enterprise/upgrade.go new file mode 100644 index 000000000..68759f2ad --- /dev/null +++ b/pkg/splunk/enterprise/upgrade.go @@ -0,0 +1,315 @@ +package enterprise + +import ( + "context" + "fmt" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splclient "github.com/splunk/splunk-operator/pkg/splunk/client" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + appsv1 "k8s.io/api/apps/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + rclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// helps in mock function +var GetClusterInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager, mockCall bool) (*splclient.ClusterInfo, error) { + cm := mgr.getClusterManagerClient(ctx) + return cm.GetClusterInfo(false) +} + +// UpgradePathValidation is used in validating if upgrade can be done to given custom resource +// +// the method follows the sequence +// 1. Standalone or License Manager +// 2. Cluster Manager - if LM ref is defined, wait for License manager to complete +// 3. Monitoring Console - if CM ref is defined, wait for Cluster Manager to complete +// 4. Search Head Cluster - if MC ref , CM ref , LM ref is defined, wait for them to complete in order, +// if any one of them not defined, ignore them and wait for the one added in ref +// 5. Indexer Cluster - same as above also wait for search head cluster to complete before starting upgrade +// if its multisite then do 1 site at a time +// function returns bool and error , true - go ahead with upgrade +// false - exit the reconciliation loop with error +func UpgradePathValidation(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, spec enterpriseApi.CommonSplunkSpec, mgr *indexerClusterPodManager) (bool, error) { + reqLogger := log.FromContext(ctx) + scopedLog := reqLogger.WithName("isClusterManagerReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + eventPublisher, _ := newK8EventPublisher(c, cr) + kind := cr.GroupVersionKind().Kind + scopedLog.Info("kind is set to ", "kind", kind) + // start from standalone first + goto Standalone + + // if custom resource type is standalone or license manager go ahead and upgrade +Standalone: + if cr.GroupVersionKind().Kind == "Standalone" { + return true, nil + } else { + goto LicenseManager + } +LicenseManager: + if cr.GroupVersionKind().Kind == "LicenseManager" { + return true, nil + } else { + licenseManagerRef := spec.LicenseManagerRef + // if custom resource type not license manager or standalone then + // check if there is license manager reference + // if no reference go to cluster manager + if licenseManagerRef.Name == "" { + goto ClusterManager + } + + namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: licenseManagerRef.Name} + licenseManager := &enterpriseApi.LicenseManager{} + // get the license manager referred in CR + err := c.Get(ctx, namespacedName, licenseManager) + if err != nil { + if k8serrors.IsNotFound(err) { + goto ClusterManager + } + return false, err + } + + // get current image of license manager + lmImage, err := getCurrentImage(ctx, c, licenseManager, SplunkLicenseManager) + if err != nil { + eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the License Manager Image. Reason %v", err)) + scopedLog.Error(err, "Unable to get licenseManager current image") + return false, err + } + // if license manager status is ready and CR spec and current license manager image are not same + // then return with error + if licenseManager.Status.Phase != enterpriseApi.PhaseReady || lmImage != spec.Image { + return false, fmt.Errorf("license manager is not ready or license manager current image is different than CR image") + } + goto ClusterManager + } +ClusterManager: + if cr.GroupVersionKind().Kind == "ClusterManager" { + + licenseManagerRef := spec.LicenseManagerRef + if licenseManagerRef.Name == "" { + return true, nil + } + namespacedName := types.NamespacedName{ + Namespace: cr.GetNamespace(), + Name: GetSplunkStatefulsetName(SplunkClusterManager, cr.GetName()), + } + + // check if the stateful set is created at this instance + statefulSet := &appsv1.StatefulSet{} + err := c.Get(ctx, namespacedName, statefulSet) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, nil + } + return true, nil + } else { + // check if a cluster manager reference is added to custom resource + clusterManagerRef := spec.ClusterManagerRef + if clusterManagerRef.Name == "" { + // if ref is not defined go to monitoring console step + goto MonitoringConsole + } + + namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: clusterManagerRef.Name} + clusterManager := &enterpriseApi.ClusterManager{} + + // get the cluster manager referred in custom resource + err := c.Get(ctx, namespacedName, clusterManager) + if err != nil { + eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Cluster Manager. Reason %v", err)) + scopedLog.Error(err, "Unable to get clusterManager") + goto MonitoringConsole + } + + /// get the cluster manager image referred in custom resource + cmImage, err := getCurrentImage(ctx, c, clusterManager, SplunkClusterManager) + if err != nil { + eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) + scopedLog.Error(err, "Unable to get clusterManager current image") + return false, err + } + + // check if an image upgrade is happening and whether CM has finished updating yet, return false to stop + // further reconcile operations on custom resource until CM is ready + if clusterManager.Status.Phase != enterpriseApi.PhaseReady || cmImage != spec.Image { + return false, nil + } + goto MonitoringConsole + } +MonitoringConsole: + if cr.GroupVersionKind().Kind == "MonitoringConsole" { + + namespacedName := types.NamespacedName{ + Namespace: cr.GetNamespace(), + Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetName()), + } + + // check if the stateful set is created at this instance + statefulSet := &appsv1.StatefulSet{} + err := c.Get(ctx, namespacedName, statefulSet) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, nil + } + return true, nil + } else { + + // check if a MonitoringConsole is attached to the instance + monitoringConsoleRef := spec.MonitoringConsoleRef + if monitoringConsoleRef.Name == "" { + goto SearchHeadCluster + } + + namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: monitoringConsoleRef.Name} + monitoringConsole := &enterpriseApi.MonitoringConsole{} + + // get the monitoring console referred in custom resource + err := c.Get(ctx, namespacedName, monitoringConsole) + if err != nil { + if k8serrors.IsNotFound(err) { + goto SearchHeadCluster + } + eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Monitoring Console. Reason %v", err)) + scopedLog.Error(err, "Unable to get Monitoring Console") + return false, err + } + + mcImage, err := getCurrentImage(ctx, c, monitoringConsole, SplunkMonitoringConsole) + if err != nil { + eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not get the Monitoring Console Image. Reason %v", err)) + scopedLog.Error(err, "Unable to get Monitoring Console current image") + return false, err + } + + // check if an image upgrade is happening and whether MC has finished updating yet, return false to stop + // further reconcile operations on custom resource until MC is ready + if monitoringConsole.Status.Phase != enterpriseApi.PhaseReady || mcImage != spec.Image { + return false, nil + } + + goto SearchHeadCluster + } +SearchHeadCluster: + if cr.GroupVersionKind().Kind == "SearchHeadCluster" { + + namespacedName := types.NamespacedName{ + Namespace: cr.GetNamespace(), + Name: GetSplunkStatefulsetName(SplunkSearchHead, cr.GetName()), + } + + // check if the stateful set is created at this instance + statefulSet := &appsv1.StatefulSet{} + err := c.Get(ctx, namespacedName, statefulSet) + if err != nil { + if k8serrors.IsNotFound(err) { + return true, nil + } + return false, nil + } + return true, nil + } else { + + // get the clusterManagerRef attached to the instance + clusterManagerRef := spec.ClusterManagerRef + + // check if a search head cluster exists with the same ClusterManager instance attached + searchHeadClusterInstance := enterpriseApi.SearchHeadCluster{} + opts := []rclient.ListOption{ + rclient.InNamespace(cr.GetNamespace()), + } + searchHeadList, err := getSearchHeadClusterList(ctx, c, cr, opts) + if err != nil { + if err.Error() == "NotFound" { + goto IndexerCluster + } + return false, err + } + if len(searchHeadList.Items) == 0 { + goto IndexerCluster + } + + // check if instance has the ClusterManagerRef defined + for _, shc := range searchHeadList.Items { + if shc.Spec.ClusterManagerRef.Name == clusterManagerRef.Name { + searchHeadClusterInstance = shc + break + } + } + if len(searchHeadClusterInstance.GetName()) == 0 { + goto IndexerCluster + } + + shcImage, err := getCurrentImage(ctx, c, &searchHeadClusterInstance, SplunkSearchHead) + if err != nil { + eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not get the Search Head Cluster Image. Reason %v", err)) + scopedLog.Error(err, "Unable to get SearchHeadCluster current image") + return false, err + } + + // check if an image upgrade is happening and whether SHC has finished updating yet, return false to stop + // further reconcile operations on IDX until SHC is ready + if searchHeadClusterInstance.Status.Phase != enterpriseApi.PhaseReady || shcImage != spec.Image { + return false, nil + } + goto IndexerCluster + } +IndexerCluster: + if cr.GroupVersionKind().Kind == "IndexerCluster" { + + // if manager client is not defined, then assign current client + if mgr.c == nil { + mgr.c = c + } + + // check cluster info call using splunk rest api + clusterInfo, err := GetClusterInfoCall(ctx, mgr, false) + if err != nil { + return false, fmt.Errorf("could not get cluster info from cluster manager") + } + // check if cluster is multisite + if clusterInfo.MultiSite == "true" { + opts := []rclient.ListOption{ + rclient.InNamespace(cr.GetNamespace()), + } + indexerList, err := getIndexerClusterList(ctx, c, cr, opts) + if err != nil { + return false, err + } + // get sorted current indexer site list + sortedList, err := getIndexerClusterSortedSiteList(ctx, c, spec.ClusterManagerRef, indexerList) + + preIdx := enterpriseApi.IndexerCluster{} + + for i, v := range sortedList.Items { + if &v == cr { + if i > 0 { + preIdx = sortedList.Items[i-1] + } + break + + } + } + if len(preIdx.Name) != 0 { + // check if previous indexer have completed before starting next one + image, _ := getCurrentImage(ctx, c, &preIdx, SplunkIndexer) + if preIdx.Status.Phase != enterpriseApi.PhaseReady || image != spec.Image { + return false, nil + } + } + + } + return true, nil + } else { + goto EndLabel + } +EndLabel: + return true, nil + +} diff --git a/pkg/splunk/enterprise/upgrade_test.go b/pkg/splunk/enterprise/upgrade_test.go new file mode 100644 index 000000000..f965e1782 --- /dev/null +++ b/pkg/splunk/enterprise/upgrade_test.go @@ -0,0 +1,726 @@ +package enterprise + +import ( + "context" + "fmt" + "runtime/debug" + "testing" + + enterpriseApi "github.com/splunk/splunk-operator/api/v4" + splclient "github.com/splunk/splunk-operator/pkg/splunk/client" + "github.com/splunk/splunk-operator/pkg/splunk/common" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestUpgradePathValidation(t *testing.T) { + + builder := fake.NewClientBuilder() + client := builder.Build() + utilruntime.Must(enterpriseApi.AddToScheme(clientgoscheme.Scheme)) + + ctx := context.TODO() + stdln := enterpriseApi.Standalone{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.StandaloneSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + }, + }, + } + + err := client.Create(ctx, &stdln) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + _, err = ApplyStandalone(ctx, client, &stdln) + if err != nil { + t.Errorf("ApplyStandalone should not have returned error; err=%v", err) + } + + // cluster manager + + lm := enterpriseApi.LicenseManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.LicenseManagerSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + }, + }, + } + + cm := enterpriseApi.ClusterManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.ClusterManagerSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + LicenseManagerRef: corev1.ObjectReference{ + Name: "test", + }, + }, + }, + } + + mc := enterpriseApi.MonitoringConsole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + LicenseManagerRef: corev1.ObjectReference{ + Name: "test", + }, + ClusterManagerRef: corev1.ObjectReference{ + Name: "test", + }, + }, + }, + } + + idx := enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.IndexerClusterSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + LicenseManagerRef: corev1.ObjectReference{ + Name: "test", + }, + ClusterManagerRef: corev1.ObjectReference{ + Name: "test", + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: "test", + }, + }, + }, + } + + shc := enterpriseApi.SearchHeadCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: enterpriseApi.SearchHeadClusterSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "Always", + Image: "splunk/splunk:old", + }, + Volumes: []corev1.Volume{}, + LicenseManagerRef: corev1.ObjectReference{ + Name: "test", + }, + ClusterManagerRef: corev1.ObjectReference{ + Name: "test", + }, + MonitoringConsoleRef: corev1.ObjectReference{ + Name: "test", + }, + }, + }, + } + + err = client.Create(ctx, &lm) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + err = client.Create(ctx, &cm) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + err = client.Create(ctx, &mc) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + err = client.Create(ctx, &idx) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + err = client.Create(ctx, &shc) + if err != nil { + t.Errorf("create should not have returned error; err=%v", err) + } + + _, err = ApplySearchHeadCluster(ctx, client, &shc) + // license manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplySearchHeadCluster should not have returned error; err=%v", err) + } + + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + // license manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplyIndexerClusterManagershould not have returned error; err=%v", err) + } + + _, err = ApplyMonitoringConsole(ctx, client, &mc) + // license manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err) + } + + _, err = ApplyClusterManager(ctx, client, &cm) + // license manager statefulset is not created + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("applyClusterManager should not have returned error; err=%v", err) + } + + // create license manager statefulset + _, err = ApplyLicenseManager(ctx, client, &lm) + if err != nil { + t.Errorf("ApplyLicenseManager should not have returned error; err=%v", err) + } + + // create pods for license manager + createPods(t, ctx, client, "license-manager", fmt.Sprintf("splunk-%s-license-manager-0", lm.Name), lm.Namespace, lm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-license-manager", lm.Name), lm.Namespace) + lm.Status.TelAppInstalled = true + // create license manager statefulset + _, err = ApplyLicenseManager(ctx, client, &lm) + if err != nil { + t.Errorf("ApplyLicenseManager should not have returned error; err=%v", err) + } + + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + // cluster manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplySearchHeadCluster should not have returned error; err=%v", err) + } + + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + // cluster manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplyIndexerClusterManagershould not have returned error; err=%v", err) + } + + _, err = ApplyMonitoringConsole(ctx, client, &mc) + // cluster manager statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err) + } + + namespacedName := types.NamespacedName{ + Name: "test", + Namespace: "test", + } + err = client.Get(ctx, namespacedName, &lm) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + if lm.Status.Phase != enterpriseApi.PhaseReady { + t.Errorf("lm is not in ready state") + } + + _, err = ApplyClusterManager(ctx, client, &cm) + // lm statefulset should have been created by now, this should pass + if err != nil { + t.Errorf("applyClusterManager should not have returned error; err=%v", err) + } + + // create pods for cluster manager + createPods(t, ctx, client, "cluster-manager", fmt.Sprintf("splunk-%s-cluster-manager-0", cm.Name), cm.Namespace, cm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-cluster-manager", cm.Name), cm.Namespace) + cm.Status.TelAppInstalled = true + // cluster manager is found and creat + _, err = ApplyClusterManager(ctx, client, &cm) + // lm statefulset should have been created by now, this should pass + if err != nil { + t.Errorf("applyClusterManager should not have returned error; err=%v", err) + } + + err = client.Get(ctx, namespacedName, &cm) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + if cm.Status.Phase != enterpriseApi.PhaseReady { + t.Errorf("cm is not in ready state") + } + + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + // monitoring console statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplySearchHeadCluster should not have returned error; err=%v", err) + } + + // mock the verify RF peer funciton + VerifyRFPeers = func(ctx context.Context, mgr indexerClusterPodManager, client splcommon.ControllerClient) error { + return nil + } + + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + // monitoring console statefulset is not created so if its NotFound error we are good + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplyIndexerClusterManager should not have returned error; err=%v", err) + } + + // mointoring console statefulset is created here + _, err = ApplyMonitoringConsole(ctx, client, &mc) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err) + } + // create pods for cluster manager + createPods(t, ctx, client, "monitoring-console", fmt.Sprintf("splunk-%s-monitoring-console-0", lm.Name), lm.Namespace, lm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-monitoring-console", lm.Name), lm.Namespace) + // mointoring console statefulset is created here + _, err = ApplyMonitoringConsole(ctx, client, &mc) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("applyMonitoringConsole should not have returned error; err=%v", err) + } + + err = client.Get(ctx, namespacedName, &mc) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + if mc.Status.Phase != enterpriseApi.PhaseReady { + t.Errorf("mc is not in ready state") + } + + // Monitoring console is ready now, now this should crete statefulset but statefulset is not in ready phase + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + if err != nil { + t.Errorf("ApplySearchHeadCluster should not have returned error; err=%v", err) + } + + // create pods for cluster manager + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-0", shc.Name), shc.Namespace, shc.Spec.Image) + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-1", shc.Name), shc.Namespace, shc.Spec.Image) + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-2", shc.Name), shc.Namespace, shc.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 3, fmt.Sprintf("splunk-%s-search-head", shc.Name), shc.Namespace) + createPods(t, ctx, client, "deployer", fmt.Sprintf("splunk-%s-deployer-0", shc.Name), shc.Namespace, shc.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-deployer", shc.Name), shc.Namespace) + + // used in mocking this function + GetSearchHeadClusterMemberInfo = func(ctx context.Context, mgr *searchHeadClusterPodManager, n int32) (*splclient.SearchHeadClusterMemberInfo, error) { + shcm := &splclient.SearchHeadClusterMemberInfo{ + Status: "Up", + } + return shcm, nil + } + + // used in mocking this function + GetSearchHeadCaptainInfo = func(ctx context.Context, mgr *searchHeadClusterPodManager, n int32) (*splclient.SearchHeadCaptainInfo, error) { + shci := &splclient.SearchHeadCaptainInfo{ + ServiceReady: true, + Initialized: true, + } + return shci, nil + } + // Now SearchheadCluster should move to READY state + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + if err != nil { + t.Errorf("ApplySearchHeadCluster should not have returned error; err=%v", err) + } + + err = client.Get(ctx, namespacedName, &shc) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + if shc.Status.Phase != enterpriseApi.PhaseReady { + t.Errorf("shc is not in ready state") + } + + // mock the verify RF peer funciton + VerifyRFPeers = func(ctx context.Context, mgr indexerClusterPodManager, client splcommon.ControllerClient) error { + return nil + } + + // mock the call + GetClusterInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager, mockCall bool) (*splclient.ClusterInfo, error) { + cinfo := &splclient.ClusterInfo{ + MultiSite: "false", + } + return cinfo, nil + } + GetClusterManagerPeersCall = func(ctx context.Context, mgr *indexerClusterPodManager) (map[string]splclient.ClusterManagerPeerInfo, error) { + response := map[string]splclient.ClusterManagerPeerInfo{ + "splunk-test-indexer-0": { + ID: "site-1", + Status: "Up", + ActiveBundleID: "1", + BucketCount: 10, + Searchable: true, + }, + } + return response, err + } + GetClusterManagerInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager) (*splclient.ClusterManagerInfo, error) { + response := &splclient.ClusterManagerInfo{ + Initialized: true, + IndexingReady: true, + ServiceReady: true, + } + return response, err + } + + // search head cluster is ready, this should create statefulset but they are not ready + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplyIndexerClusterManager should not have returned error; err=%v", err) + } + + // create pods for indexer cluster + createPods(t, ctx, client, "indexer", fmt.Sprintf("splunk-%s-indexer-0", idx.Name), idx.Namespace, idx.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-indexer", idx.Name), idx.Namespace) + + // search head cluster is not ready, so wait for search head cluster + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("ApplyIndexerClusterManager should not have returned error; err=%v", err) + } + + err = client.Get(ctx, namespacedName, &idx) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + if idx.Status.Phase != enterpriseApi.PhaseReady { + t.Errorf("shc is not in ready state") + } + + VerifyCMisMultisiteCall = func(ctx context.Context, cr *enterpriseApi.ClusterManager, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { + extraEnv := getClusterManagerExtraEnv(cr, &cr.Spec.CommonSplunkSpec) + return extraEnv, err + } + + // ------- Step2 starts here ----- + // Update + // standalone + err = client.Get(ctx, namespacedName, &stdln) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + stdln.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &stdln) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + _, err = ApplyStandalone(ctx, client, &stdln) + if err != nil { + t.Errorf("ApplyStandalone should not have returned error; err=%v", err) + } + + // cluster manager + err = client.Get(ctx, namespacedName, &cm) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + + cm.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &cm) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + + // license manager + err = client.Get(ctx, namespacedName, &lm) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + lm.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &lm) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + + // monitoring console + err = client.Get(ctx, namespacedName, &mc) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + mc.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &mc) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + + // indexer cluster console + err = client.Get(ctx, namespacedName, &idx) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + idx.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &idx) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + + // searchhead cluster console + err = client.Get(ctx, namespacedName, &shc) + if err != nil { + t.Errorf("get should not have returned error; err=%v", err) + } + shc.Spec.Image = "splunk/splunk:latest" + err = client.Update(ctx, &shc) + if err != nil { + t.Errorf("update should not have returned error; err=%v", err) + } + + lm.Status.TelAppInstalled = true + _, err = ApplyLicenseManager(ctx, client, &lm) + if err != nil { + t.Errorf("ApplyLicenseManager after update should not have returned error; err=%v", err) + } + + lm.Status.TelAppInstalled = true + _, err = ApplyLicenseManager(ctx, client, &lm) + if err != nil { + t.Errorf("ApplyLicenseManager after update should not have returned error; err=%v", err) + } + + cm.Status.TelAppInstalled = true + _, err = ApplyClusterManager(ctx, client, &cm) + if err != nil { + t.Errorf("applyClusterManager after update should not have returned error; err=%v", err) + } + _, err = ApplyMonitoringConsole(ctx, client, &mc) + if err != nil { + t.Errorf("applyMonitoringConsole after update should not have returned error; err=%v", err) + } + + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + if err != nil { + t.Errorf("applySearchHeadCluster after update should not have returned error; err=%v", err) + } + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + if err != nil { + t.Errorf("ApplyIndexerClusterManager after update should not have returned error; err=%v", err) + } + newImage := "splunk/splunk:latest" + // create pods for license manager + createPods(t, ctx, client, "license-manager", fmt.Sprintf("splunk-%s-license-manager-0", lm.Name), lm.Namespace, newImage) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-license-manager", lm.Name), lm.Namespace) + lm.Status.TelAppInstalled = true + + // create pods for cluster manager + createPods(t, ctx, client, "cluster-manager", fmt.Sprintf("splunk-%s-cluster-manager-0", cm.Name), cm.Namespace, cm.Spec.Image) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-cluster-manager", cm.Name), cm.Namespace) + cm.Status.TelAppInstalled = true + + // create pods for indexer cluster + createPods(t, ctx, client, "indexer", fmt.Sprintf("splunk-%s-indexer-0", idx.Name), idx.Namespace, newImage) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-indexer", idx.Name), idx.Namespace) + + // create pods for cluster manager + createPods(t, ctx, client, "monitoring-console", fmt.Sprintf("splunk-%s-monitoring-console-0", lm.Name), lm.Namespace, newImage) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-monitoring-console", lm.Name), lm.Namespace) + + // create pods for cluster manager + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-0", shc.Name), shc.Namespace, newImage) + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-1", shc.Name), shc.Namespace, newImage) + createPods(t, ctx, client, "search-head", fmt.Sprintf("splunk-%s-search-head-2", shc.Name), shc.Namespace, newImage) + updateStatefulSetsInTest(t, ctx, client, 3, fmt.Sprintf("splunk-%s-search-head", shc.Name), shc.Namespace) + createPods(t, ctx, client, "deployer", fmt.Sprintf("splunk-%s-deployer-0", shc.Name), shc.Namespace, newImage) + updateStatefulSetsInTest(t, ctx, client, 1, fmt.Sprintf("splunk-%s-deployer", shc.Name), shc.Namespace) + shc.Status.TelAppInstalled = true + + lm.Status.TelAppInstalled = true + _, err = ApplyLicenseManager(ctx, client, &lm) + if err != nil { + t.Errorf("ApplyLicenseManager after update should not have returned error; err=%v", err) + } + + cm.Status.TelAppInstalled = true + _, err = ApplyClusterManager(ctx, client, &cm) + if err != nil { + t.Errorf("applyClusterManager after update should not have returned error; err=%v", err) + } + + cm.Status.TelAppInstalled = true + _, err = ApplyClusterManager(ctx, client, &cm) + if err != nil { + t.Errorf("applyClusterManager after update should not have returned error; err=%v", err) + } + + _, err = ApplyMonitoringConsole(ctx, client, &mc) + if err != nil { + t.Errorf("applyMonitoringConsole after update should not have returned error; err=%v", err) + } + + _, err = ApplyMonitoringConsole(ctx, client, &mc) + if err != nil { + t.Errorf("applyMonitoringConsole after update should not have returned error; err=%v", err) + } + + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + if err != nil { + t.Errorf("applySearchHeadCluster after update should not have returned error; err=%v", err) + } + + shc.Status.TelAppInstalled = true + _, err = ApplySearchHeadCluster(ctx, client, &shc) + if err != nil { + t.Errorf("applySearchHeadCluster after update should not have returned error; err=%v", err) + } + + _, err = ApplyIndexerClusterManager(ctx, client, &idx) + if err != nil { + t.Errorf("ApplyIndexerClusterManager after update should not have returned error; err=%v", err) + } + +} + +func createPods(t *testing.T, ctx context.Context, client common.ControllerClient, crtype, name, namespace, image string) { + stpod := &corev1.Pod{} + namespacesName := types.NamespacedName{ + Name: name, + Namespace: namespace, + } + err := client.Get(ctx, namespacesName, stpod) + if err != nil && k8serrors.IsNotFound(err) { + // create pod + stpod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "splunk-operator", + "app.kubernetes.io/component": crtype, + "app.kubernetes.io/name": crtype, + "app.kubernetes.io/part-of": fmt.Sprintf("splunk-test-%s", crtype), + "app.kubernetes.io/instance": fmt.Sprintf("splunk-test-%s", crtype), + }, + Annotations: map[string]string{ + "traffic.sidecar.istio.io/excludeOutboundPorts": "8089,8191,9997", + "traffic.sidecar.istio.io/includeInboundPorts": "8000", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "splunk", + Image: image, + Env: []corev1.EnvVar{ + { + Name: "test", + Value: "test", + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "http-splunkweb", + HostPort: 0, + ContainerPort: 8000, + Protocol: "TCP", + HostIP: "", + }, + { + Name: "https-splunkd", + HostPort: 0, + ContainerPort: 8089, + Protocol: "TCP", + HostIP: "", + }, + }, + }, + }, + }, + } + // simulate create stateful set + err := client.Create(ctx, stpod) + if err != nil { + t.Errorf("Unexpected create pod failed %v", err) + debug.PrintStack() + } + } else if err != nil { + t.Errorf("Unexpected erro while get pod %v", err) + debug.PrintStack() + } + if stpod.Spec.Containers[0].Image != image { + stpod.Spec.Containers[0].Image = image + err := client.Update(ctx, stpod) + if err != nil { + t.Errorf("Unexpected create pod failed %v", err) + debug.PrintStack() + } + } + + // update statefulset + stpod.Status.Phase = corev1.PodRunning + stpod.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + Image: image, + Name: "splunk", + Ready: true, + }, + } + err = client.Status().Update(ctx, stpod) + if err != nil { + t.Errorf("Unexpected update pod %v", err) + debug.PrintStack() + } +} + +func updateStatefulSetsInTest(t *testing.T, ctx context.Context, client common.ControllerClient, replicas int32, name, namespace string) { + stNamespacedName := types.NamespacedName{ + Name: name, + Namespace: namespace, + } + statefulset := &appsv1.StatefulSet{} + err := client.Get(ctx, stNamespacedName, statefulset) + if err != nil { + t.Errorf("Unexpected get cluster manager %v", err) + debug.PrintStack() + } + // update statefulset + statefulset.Status.ReadyReplicas = replicas + statefulset.Status.Replicas = replicas + statefulset.Status.CurrentReplicas = replicas + statefulset.Status.AvailableReplicas = replicas + err = client.Status().Update(ctx, statefulset) + if err != nil { + t.Errorf("Unexpected update statefulset %v", err) + debug.PrintStack() + } +} diff --git a/test/appframework_aws/c3/appframework_aws_test.go b/test/appframework_aws/c3/appframework_aws_test.go index 86e89a172..70d309bf9 100644 --- a/test/appframework_aws/c3/appframework_aws_test.go +++ b/test/appframework_aws/c3/appframework_aws_test.go @@ -57,6 +57,7 @@ var _ = Describe("c3appfw test", func() { name := fmt.Sprintf("%s-%s", "master"+testenvInstance.GetName(), testenv.RandomDNSName(3)) testcaseEnvInst, err = testenv.NewDefaultTestCaseEnv(testenvInstance.GetKubeClient(), name) Expect(err).To(Succeed(), "Unable to create testcaseenv") + testenv.SpecifiedTestTimeout = 4000 deployment, err = testcaseEnvInst.NewDeployment(testenv.RandomDNSName(3)) Expect(err).To(Succeed(), "Unable to create deployment") }) @@ -189,12 +190,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -270,12 +271,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -409,12 +410,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -478,12 +479,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -588,12 +589,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -854,12 +855,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -903,12 +904,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1043,12 +1044,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1107,12 +1108,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1250,12 +1251,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1315,12 +1316,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1413,12 +1414,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1539,12 +1540,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1601,12 +1602,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1751,12 +1752,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1800,12 +1801,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1976,12 +1977,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2052,12 +2053,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2331,6 +2332,9 @@ var _ = Describe("c3appfw test", func() { // Wait for polling interval to pass testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName(), cm.Kind, appSourceNameIdxc, appFileList) + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) @@ -2341,9 +2345,6 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info(fmt.Sprintf("Verify all apps %v are installed on indexers", appList)) testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), idxcPodNames, appList, true, "enabled", false, true) - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - // Wait for polling interval to pass testenv.WaitForAppInstall(ctx, deployment, testcaseEnvInst, deployment.GetName()+"-shc", shc.Kind, appSourceNameShc, appFileList) @@ -2420,12 +2421,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2514,12 +2515,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2599,12 +2600,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2748,12 +2749,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2842,12 +2843,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2935,12 +2936,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3019,12 +3020,12 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) }) @@ -3197,12 +3198,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index 96fd1efd0..38dc33dae 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -187,12 +187,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -268,12 +268,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -304,6 +304,147 @@ var _ = Describe("c3appfw test", func() { }) }) + Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework and Image Upgrade", func() { + It("smoke, c3, managerappframeworkc3t, appframework: can deploy a C3 SVA with App Framework enabled, install apps then upgrade the image and apps", func() { + + //################## SETUP #################### + + // Download License File + downloadDir := "licenseFolder" + switch testenv.ClusterProvider { + case "eks": + licenseFilePath, err := testenv.DownloadLicenseFromS3Bucket() + Expect(err).To(Succeed(), "Unable to download license file from S3") + // Create License Config Map + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + case "azure": + licenseFilePath, err := testenv.DownloadLicenseFromAzure(ctx, downloadDir) + Expect(err).To(Succeed(), "Unable to download license file from Azure") + // Create License Config Map + testcaseEnvInst.CreateLicenseConfigMap(licenseFilePath) + default: + fmt.Printf("Unable to download license file") + testcaseEnvInst.Log.Info(fmt.Sprintf("Unable to download license file with Cluster Provider set as %v", testenv.ClusterProvider)) + } + + // Upload V1 apps to S3 for Monitoring Console + oldImage := "splunk/splunk:9.0.3-a2" + newImage := "splunk/splunk:9.0.5" + + lm, err := deployment.DeployLicenseManager(ctx, deployment.GetName()) + cm, err := deployment.DeployClusterManager(ctx, deployment.GetName(), lm.GetName(), "", "") + + mcSpec := enterpriseApi.MonitoringConsoleSpec{ + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Spec: enterpriseApi.Spec{ + ImagePullPolicy: "IfNotPresent", + }, + Volumes: []corev1.Volume{}, + ClusterManagerRef: corev1.ObjectReference{ + Name: cm.GetName(), + }, + }, + } + mcName := deployment.GetName() + mc, err := deployment.DeployMonitoringConsoleWithGivenSpec(ctx, testcaseEnvInst.GetName(), mcName, mcSpec) + Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") + + shcName := fmt.Sprintf("%s-shc", deployment.GetName()) + idxName := fmt.Sprintf("%s-idxc", deployment.GetName()) + shc, err := deployment.DeploySearchHeadCluster(ctx, shcName, cm.GetName(), lm.GetName(), "", mcName) + idxc, err := deployment.DeployIndexerCluster(ctx, idxName, lm.GetName(), 3, cm.GetName(), "") + + // Wait for License Manager to be in READY phase + testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + + // Ensure Cluster Manager goes to Ready phase + testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + // Ensure Monitoring Console goes to Ready phase + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + + // // Verify Monitoring Console is ready and stays in ready state + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // // Verify no SH in disconnected status is present on CM + testenv.VerifyNoDisconnectedSHPresentOnCM(ctx, deployment, testcaseEnvInst) + + // Get Pod age to check for pod resets later + splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) + + // ############ Verify livenessProbe and readinessProbe config object and scripts############ + testcaseEnvInst.Log.Info("Get config map for livenessProbe and readinessProbe") + ConfigMapName := enterprise.GetProbeConfigMapName(testcaseEnvInst.GetName()) + _, err = testenv.GetConfigMap(ctx, deployment, testcaseEnvInst.GetName(), ConfigMapName) + Expect(err).To(Succeed(), "Unable to get config map for livenessProbe and readinessProbe", "ConfigMap name", ConfigMapName) + scriptsNames := []string{enterprise.GetLivenessScriptName(), enterprise.GetReadinessScriptName(), enterprise.GetStartupScriptName()} + allPods := testenv.DumpGetPods(testcaseEnvInst.GetName()) + testenv.VerifyFilesInDirectoryOnPod(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), allPods, scriptsNames, enterprise.GetProbeMountDirectory(), false, true) + + // Verify no pods reset by checking the pod age + testenv.VerifyNoPodReset(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), splunkPodAge, nil) + + //############### UPGRADE IMAGE ################ + + // Update LM Image + + testcaseEnvInst.Log.Info("Upgrading the License Manager Image", "Current Image", oldImage, "New Image", newImage) + lm.Spec.Image = newImage + err = deployment.UpdateCR(ctx, lm) + Expect(err).To(Succeed(), "Failed upgrade License Manager image") + + // Update CM image + + testcaseEnvInst.Log.Info("Upgrading the Cluster Manager Image", "Current Image", oldImage, "New Image", newImage) + cm.Spec.Image = newImage + err = deployment.UpdateCR(ctx, cm) + Expect(err).To(Succeed(), "Failed upgrade Cluster Manager image") + + // Update MC image + + testcaseEnvInst.Log.Info("Upgrading the Monitoring Console Image", "Current Image", oldImage, "New Image", newImage) + mc.Spec.Image = newImage + err = deployment.UpdateCR(ctx, mc) + Expect(err).To(Succeed(), "Failed upgrade Monitoring Console image") + + // Update SHC image + + testcaseEnvInst.Log.Info("Upgrading the Search Head Cluster Image", "Current Image", oldImage, "New Image", newImage) + shc.Spec.Image = newImage + err = deployment.UpdateCR(ctx, shc) + Expect(err).To(Succeed(), "Failed upgrade Search Head Cluster image") + + // // Update IDXC image + + testcaseEnvInst.Log.Info("Upgrading the Indexer Cluster Image", "Current Image", oldImage, "New Image", newImage) + idxc.Spec.Image = newImage + err = deployment.UpdateCR(ctx, idxc) + Expect(err).To(Succeed(), "Failed upgrade Indexer Cluster image") + + // Ensure Cluster Manager goes to Ready phase + testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + + // Wait for License Manager to be in READY phase + testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) + + // // Verify Monitoring Console is ready and stays in ready state + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) + + // // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + + // // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + + }) + }) + Context("Single Site Indexer Cluster with Search Head Cluster (C3) with App Framework", func() { It("smoke, c3, managerappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled, install apps then downgrade them", func() { @@ -407,12 +548,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -476,12 +617,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -586,12 +727,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -852,12 +993,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -901,12 +1042,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1041,12 +1182,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1105,12 +1246,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1248,12 +1389,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1313,12 +1454,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1411,12 +1552,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1537,12 +1678,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1599,12 +1740,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1749,12 +1890,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1798,12 +1939,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1974,12 +2115,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2050,12 +2191,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2217,7 +2358,7 @@ var _ = Describe("c3appfw test", func() { testenv.VerifyAppInstalled(ctx, deployment, testcaseEnvInst, testcaseEnvInst.GetName(), deployerPod, appList, true, "enabled", false, false) }) }) - + // Vivek need testing Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3, add new apps to app source while install is in progress and have all apps installed cluster-wide", func() { @@ -2352,7 +2493,7 @@ var _ = Describe("c3appfw test", func() { }) }) - + // Vivek need testing Context("Single Site Indexer Cluster with Search Head Cluster (C3) and App Framework", func() { It("integration, c3, managerappframeworkc3, appframework: can deploy a C3 SVA with App Framework enabled and reset operator pod while app install is in progress", func() { @@ -2418,12 +2559,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2512,12 +2653,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2597,12 +2738,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2746,12 +2887,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2840,12 +2981,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2933,12 +3074,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3017,12 +3158,12 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) }) @@ -3195,12 +3336,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/m4/appframework_aws_test.go b/test/appframework_aws/m4/appframework_aws_test.go index 1549f92e1..c7c6c6e8d 100644 --- a/test/appframework_aws/m4/appframework_aws_test.go +++ b/test/appframework_aws/m4/appframework_aws_test.go @@ -2422,15 +2422,15 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) // Ensure Indexer Cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) diff --git a/test/appframework_aws/m4/manager_appframework_test.go b/test/appframework_aws/m4/manager_appframework_test.go index 8f44be55b..5310000cb 100644 --- a/test/appframework_aws/m4/manager_appframework_test.go +++ b/test/appframework_aws/m4/manager_appframework_test.go @@ -2421,15 +2421,15 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) + // Ensure Search Head Cluster go to Ready phase + testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure the Indexers of all sites go to Ready phase testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) // Ensure Indexer Cluster configured as Multisite testenv.IndexerClusterMultisiteStatus(ctx, deployment, testcaseEnvInst, siteCount) - // Ensure Search Head Cluster go to Ready phase - testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) - // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2517,12 +2517,12 @@ var _ = Describe("m4appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure the Indexers of all sites go to Ready phase - testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure the Indexers of all sites go to Ready phase + testenv.IndexersReady(ctx, deployment, testcaseEnvInst, siteCount) + // Get Pod age to check for pod resets later splunkPodAge := testenv.GetPodsStartTime(testcaseEnvInst.GetName()) diff --git a/test/appframework_az/c3/appframework_azure_test.go b/test/appframework_az/c3/appframework_azure_test.go index c8eb33545..97bcbea77 100644 --- a/test/appframework_az/c3/appframework_azure_test.go +++ b/test/appframework_az/c3/appframework_azure_test.go @@ -187,12 +187,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -261,12 +261,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -400,12 +400,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -470,12 +470,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -580,12 +580,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -828,12 +828,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -878,12 +878,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1013,12 +1013,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1063,12 +1063,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1197,12 +1197,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1262,12 +1262,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1408,12 +1408,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1473,12 +1473,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1572,12 +1572,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1698,12 +1698,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1761,12 +1761,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1911,12 +1911,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1961,12 +1961,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2137,12 +2137,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2214,12 +2214,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2585,12 +2585,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2680,12 +2680,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2766,12 +2766,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2918,12 +2918,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3008,12 +3008,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3102,12 +3102,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3186,12 +3186,12 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) }) diff --git a/test/appframework_az/c3/manager_appframework_azure_test.go b/test/appframework_az/c3/manager_appframework_azure_test.go index 5197ed894..503b3cce3 100644 --- a/test/appframework_az/c3/manager_appframework_azure_test.go +++ b/test/appframework_az/c3/manager_appframework_azure_test.go @@ -185,12 +185,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -259,12 +259,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -398,12 +398,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -468,12 +468,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -578,12 +578,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -826,12 +826,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -876,12 +876,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1011,12 +1011,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1061,12 +1061,12 @@ var _ = Describe("c3appfw test", func() { // // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1196,12 +1196,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1261,12 +1261,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1407,12 +1407,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1473,12 +1473,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1572,12 +1572,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1698,12 +1698,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1761,12 +1761,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1911,12 +1911,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -1961,12 +1961,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2137,12 +2137,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2214,12 +2214,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2585,12 +2585,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2680,12 +2680,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2766,12 +2766,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -2918,12 +2918,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3008,12 +3008,12 @@ var _ = Describe("c3appfw test", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3102,12 +3102,12 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) @@ -3186,12 +3186,12 @@ var _ = Describe("c3appfw test", func() { testcaseEnvInst.Log.Info("Checking isDeploymentInProgress Flag") testenv.VerifyIsDeploymentInProgressFlagIsSet(ctx, deployment, testcaseEnvInst, shc.Name, shc.Kind) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) }) diff --git a/test/custom_resource_crud/custom_resource_crud_c3_test.go b/test/custom_resource_crud/custom_resource_crud_c3_test.go index d020c8a72..b847e517d 100644 --- a/test/custom_resource_crud/custom_resource_crud_c3_test.go +++ b/test/custom_resource_crud/custom_resource_crud_c3_test.go @@ -74,12 +74,12 @@ var _ = Describe("Crcrud test for SVA C3", func() { // Ensure that the Cluster Master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") diff --git a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go index b30b4ce02..a86a58322 100644 --- a/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go +++ b/test/custom_resource_crud/manager_custom_resource_crud_c3_test.go @@ -75,12 +75,12 @@ var _ = Describe("Crcrud test for SVA C3", func() { // Ensure that the Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, "") Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") diff --git a/test/delete_cr/deletecr_test.go b/test/delete_cr/deletecr_test.go index f0e94c81a..300d3cd1f 100644 --- a/test/delete_cr/deletecr_test.go +++ b/test/delete_cr/deletecr_test.go @@ -94,12 +94,12 @@ var _ = Describe("DeleteCR test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + idxc := &enterpriseApi.IndexerCluster{} idxcName := deployment.GetName() + "-idxc" err = deployment.GetInstance(ctx, idxcName, idxc) diff --git a/test/licensemanager/manager_lm_c3_test.go b/test/licensemanager/manager_lm_c3_test.go index 83b954169..af80f1fd2 100644 --- a/test/licensemanager/manager_lm_c3_test.go +++ b/test/licensemanager/manager_lm_c3_test.go @@ -85,12 +85,12 @@ var _ = Describe("Licensemanager test", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") diff --git a/test/licensemaster/lm_c3_test.go b/test/licensemaster/lm_c3_test.go index f93d917ed..986bb2d1f 100644 --- a/test/licensemaster/lm_c3_test.go +++ b/test/licensemaster/lm_c3_test.go @@ -86,12 +86,12 @@ var _ = Describe("licensemaster test", func() { // Ensure that the cluster-master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, mcRef, deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console") diff --git a/test/monitoring_console/manager_monitoring_console_test.go b/test/monitoring_console/manager_monitoring_console_test.go index d38dc3ebb..42af80f06 100644 --- a/test/monitoring_console/manager_monitoring_console_test.go +++ b/test/monitoring_console/manager_monitoring_console_test.go @@ -451,12 +451,12 @@ var _ = Describe("Monitoring Console test", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) @@ -626,12 +626,12 @@ var _ = Describe("Monitoring Console test", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) diff --git a/test/monitoring_console/monitoring_console_test.go b/test/monitoring_console/monitoring_console_test.go index b6f80e0ea..99375cb59 100644 --- a/test/monitoring_console/monitoring_console_test.go +++ b/test/monitoring_console/monitoring_console_test.go @@ -94,12 +94,12 @@ var _ = Describe("Monitoring Console test", func() { // Ensure that the cluster-master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) @@ -269,12 +269,12 @@ var _ = Describe("Monitoring Console test", func() { // Ensure that the cluster-master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) diff --git a/test/run-tests.sh b/test/run-tests.sh index 27e01ac29..6eabde8bc 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -22,7 +22,7 @@ if [ -n "${PRIVATE_REGISTRY}" ]; then PRIVATE_SPLUNK_ENTERPRISE_IMAGE=${PRIVATE_REGISTRY}/${SPLUNK_ENTERPRISE_IMAGE} echo "docker images -q ${SPLUNK_OPERATOR_IMAGE}" # Don't pull splunk operator if exists locally since we maybe building it locally - if [ -z $(docker images -q ${SPLUNK_OPERATOR_IMAGE}) ]; then + if [ -z $(docker images -q ${SPLUNK_OPERATOR_IMAGE}) ]; then docker pull ${SPLUNK_OPERATOR_IMAGE} if [ $? -ne 0 ]; then echo "Unable to pull ${SPLUNK_OPERATOR_IMAGE}. Exiting..." @@ -55,7 +55,7 @@ if [ -n "${PRIVATE_REGISTRY}" ]; then docker images fi -if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then +if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then echo "Installing Splunk Operator using Helm charts" helm uninstall splunk-operator -n splunk-operator if [ "${CLUSTER_WIDE}" != "true" ]; then @@ -63,10 +63,10 @@ if [ "${DEPLOYMENT_TYPE}" == "helm" ]; then else helm install splunk-operator --create-namespace --namespace splunk-operator --set splunkOperator.image.repository=${PRIVATE_SPLUNK_OPERATOR_IMAGE} --set image.repository=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} helm-chart/splunk-operator fi -elif [ "${CLUSTER_WIDE}" != "true" ]; then +elif [ "${CLUSTER_WIDE}" != "true" ]; then # Install the CRDs echo "Installing enterprise CRDs..." - make kustomize + make kustomize make uninstall bin/kustomize build config/crd | kubectl create -f - else @@ -79,7 +79,7 @@ if [ $? -ne 0 ]; then exit 1 fi -if [ "${CLUSTER_WIDE}" == "true" ]; then +if [ "${CLUSTER_WIDE}" == "true" ]; then echo "wait for operator pod to be ready..." # sleep before checking for deployment, in slow clusters deployment call may not even started # in those cases, kubectl will fail with error: no matching resources found @@ -98,14 +98,14 @@ if [ -z "$rc" ]; then go get github.com/onsi/gomega/... go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@latest -fi +fi echo "Running test using number of nodes: ${NUM_NODES}" echo "Running test using these images: ${PRIVATE_SPLUNK_OPERATOR_IMAGE} and ${PRIVATE_SPLUNK_ENTERPRISE_IMAGE}..." -# Check if test focus is set +# Check if test focus is set if [[ -z "${TEST_FOCUS}" ]]; then TEST_TO_RUN="${TEST_REGEX}" echo "Test focus not set running smoke test by default :: ${TEST_TO_RUN}" @@ -191,5 +191,5 @@ fi echo "Skipping following test :: ${TEST_TO_SKIP}" # Running only smoke test cases by default or value passed through TEST_FOCUS env variable. To run different test packages add/remove path from focus argument or TEST_FOCUS variable -echo "ginkgo --junit-report=inttest.xml -vv --keep-going --trace -r --timeout=3h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE}" -ginkgo --junit-report=inttest-junit.xml --output-dir=`pwd` -vv --keep-going --trace -r --timeout=3h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE} \ No newline at end of file +echo "ginkgo --junit-report=inttest.xml -vv --keep-going --trace -r --timeout=7h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE}" +ginkgo --junit-report=inttest-junit.xml --output-dir=`pwd` -vv --keep-going --trace -r --timeout=7h -nodes=${CLUSTER_NODES} --focus="${TEST_TO_RUN}" --skip="${TEST_TO_SKIP}" --output-interceptor-mode=none --cover ${topdir}/test/ -- -commit-hash=${COMMIT_HASH} -operator-image=${PRIVATE_SPLUNK_OPERATOR_IMAGE} -splunk-image=${PRIVATE_SPLUNK_ENTERPRISE_IMAGE} -cluster-wide=${CLUSTER_WIDE} \ No newline at end of file diff --git a/test/secret/manager_secret_c3_test.go b/test/secret/manager_secret_c3_test.go index 0122f0ede..1c4740472 100644 --- a/test/secret/manager_secret_c3_test.go +++ b/test/secret/manager_secret_c3_test.go @@ -91,12 +91,12 @@ var _ = Describe("Secret Test for SVA C3", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") @@ -134,12 +134,12 @@ var _ = Describe("Secret Test for SVA C3", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) diff --git a/test/secret/secret_c3_test.go b/test/secret/secret_c3_test.go index d10d9a3d1..2375deb5a 100644 --- a/test/secret/secret_c3_test.go +++ b/test/secret/secret_c3_test.go @@ -91,12 +91,12 @@ var _ = Describe("Secret Test for SVA C3", func() { // Ensure that the cluster-master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Deploy Monitoring Console CRD mc, err := deployment.DeployMonitoringConsole(ctx, deployment.GetName(), deployment.GetName()) Expect(err).To(Succeed(), "Unable to deploy Monitoring Console One instance") @@ -134,12 +134,12 @@ var _ = Describe("Secret Test for SVA C3", func() { // Ensure that the cluster-master goes to Ready phase testenv.ClusterMasterReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // wait for custom resource resource version to change testenv.VerifyCustomResourceVersionChanged(ctx, deployment, testcaseEnvInst, mc, resourceVersion) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index 61ef1195b..ed8d82626 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -74,12 +74,12 @@ var _ = Describe("Smoke test", func() { // Ensure that the cluster-manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure indexers go to Ready phase - testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) - - // Ensure search head cluster go to Ready phase + // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) + // Ensure Indexers go to Ready phase + testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // Verify RF SF is met testenv.VerifyRFSFMet(ctx, deployment, testcaseEnvInst) }) diff --git a/test/testenv/testenv.go b/test/testenv/testenv.go index 361f14614..db8a5b139 100644 --- a/test/testenv/testenv.go +++ b/test/testenv/testenv.go @@ -46,7 +46,7 @@ const ( defaultSplunkImage = "splunk/splunk:latest" // defaultTestTimeout is the max timeout in seconds before async test failed. - defaultTestTimeout = 3000 + defaultTestTimeout = 10000 // PollInterval specifies the polling interval PollInterval = 5 * time.Second @@ -58,7 +58,7 @@ const ( ConsistentDuration = 2000 * time.Millisecond // DefaultTimeout is the max timeout before we failed. - DefaultTimeout = 5 * time.Minute + DefaultTimeout = 10 * time.Minute // SearchHeadPod Template String for search head pod SearchHeadPod = "splunk-%s-shc-search-head-%d"