-
Notifications
You must be signed in to change notification settings - Fork 170
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Provider client specific alerting tests (#9078)
* add provider client specific alerting tests Signed-off-by: fbalak <[email protected]>
- Loading branch information
Showing
4 changed files
with
304 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| """ | ||
| A module for all StorageConsumer functionalities and abstractions. | ||
| """ | ||
| import logging | ||
|
|
||
| from ocs_ci.framework import config | ||
| from ocs_ci.ocs import constants, ocp | ||
| from ocs_ci.ocs.resources.ocs import OCS | ||
| from ocs_ci.utility.utils import exec_cmd | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class StorageConsumer: | ||
| """ | ||
| Base StorageConsumer class | ||
| """ | ||
|
|
||
| def __init__(self, consumer_name, consumer_context=None): | ||
| """ | ||
| Args: | ||
| consumer_name (string): name of the StorageConsumer resource | ||
| consumer_context (int): index of cluster context. This is needed for | ||
| consumer operations executed on consumer | ||
| (e.g. manipulation of heartbeat cronjob) | ||
| """ | ||
| self.consumer_context = consumer_context | ||
| self.name = consumer_name | ||
| self.ocp = ocp.OCP( | ||
| resource_name=self.name, | ||
| kind=constants.STORAGECONSUMER, | ||
| namespace=config.cluster_ctx.ENV_DATA["cluster_namespace"], | ||
| ) | ||
| if self.consumer_context: | ||
| self.provider_context = config.cluster_ctx.MULTICLUSTER[ | ||
| "multicluster_index" | ||
| ] | ||
| self.heartbeat_cronjob = self.get_heartbeat_cronjob() | ||
| else: | ||
| self.provider_context = None | ||
| self.heartbeat_cronjob = None | ||
|
|
||
| def get_ocs_version(self): | ||
| """ | ||
| Get ocs version from storageconsumer resource. | ||
| Returns: | ||
| string: consumer ocs version | ||
| """ | ||
| return ( | ||
| self.ocp.get(resource_name=self.name) | ||
| .get("status") | ||
| .get("client") | ||
| .get("operatorVersion") | ||
| ) | ||
|
|
||
| def set_ocs_version(self, version): | ||
| """ | ||
| Update ocs consumer version in storageconsumer resource. This change assumes | ||
| that the hearthbeat is stopped so that the version is not overwritten by it. | ||
| Args: | ||
| version (str): OCS version to be set | ||
| """ | ||
| cmd = [ | ||
| "oc", | ||
| "patch", | ||
| "StorageConsumer", | ||
| self.name, | ||
| "--type", | ||
| "json", | ||
| "-p=" | ||
| + "'" | ||
| + f'[{{"op": "replace", "path": "/status/client/operatorVersion", "value":"{version}"}}]' | ||
| + "'", | ||
| "--subresource", | ||
| "status", | ||
| ] | ||
| exec_cmd(" ".join(cmd)) | ||
|
|
||
| def stop_heartbeat(self): | ||
| """ | ||
| Suspend status reporter cron job. | ||
| """ | ||
| self._switch_consumer_cluster() | ||
| patch_param = '{"spec": {"suspend": true}}' | ||
| self.heartbeat_cronjob.ocp.patch( | ||
| resource_name=self.heartbeat_cronjob.name, params=patch_param | ||
| ) | ||
| self._switch_provider_cluster() | ||
|
|
||
| def resume_heartbeat(self): | ||
| """ | ||
| Resume status reporter cron job. | ||
| """ | ||
| self._switch_consumer_cluster() | ||
| patch_param = '{"spec": {"suspend": false}}' | ||
| self.heartbeat_cronjob.ocp.patch( | ||
| resource_name=self.heartbeat_cronjob.name, params=patch_param | ||
| ) | ||
| self._switch_provider_cluster() | ||
|
|
||
| def get_heartbeat_cronjob(self): | ||
| """ | ||
| Returns: | ||
| object: status reporter cronjob OCS object | ||
| """ | ||
| self._switch_consumer_cluster() | ||
| cronjobs_obj = ocp.OCP( | ||
| kind=constants.CRONJOB, | ||
| namespace=config.cluster_ctx.ENV_DATA["cluster_namespace"], | ||
| ) | ||
| cronjob = [ | ||
| OCS(**job) | ||
| for job in cronjobs_obj.get().get("items") | ||
| if job["metadata"]["name"].endswith("status-reporter") | ||
| ][0] | ||
| self._switch_provider_cluster() | ||
| return cronjob | ||
|
|
||
| def _switch_provider_cluster(self): | ||
| """ | ||
| Switch context to provider cluster. | ||
| """ | ||
| config.switch_ctx(self.provider_context) | ||
| log.info(f"Switched to provider cluster with index {self.provider_context}") | ||
|
|
||
| def _switch_consumer_cluster(self): | ||
| """ | ||
| Switch context to consumer cluster. | ||
| """ | ||
| config.switch_ctx(self.consumer_context) | ||
| log.info(f"Switched to consumer cluster with index {self.consumer_context}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
84 changes: 84 additions & 0 deletions
84
tests/functional/monitoring/prometheus/alerts/test_provider_client.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| import logging | ||
| import pytest | ||
|
|
||
| from ocs_ci.framework.pytest_customization.marks import blue_squad | ||
| from ocs_ci.framework.testlib import ( | ||
| tier4c, | ||
| runs_on_provider, | ||
| hci_provider_and_client_required, | ||
| ) | ||
| from ocs_ci.ocs import constants | ||
| from ocs_ci.utility import prometheus | ||
| from ocs_ci.ocs.ocp import OCP | ||
|
|
||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| @blue_squad | ||
| @tier4c | ||
| @runs_on_provider | ||
| @hci_provider_and_client_required | ||
| @pytest.mark.polarion_id("OCS-5392") | ||
| def test_change_client_ocs_version_and_stop_heartbeat( | ||
| measure_change_client_ocs_version_and_stop_heartbeat, threading_lock | ||
| ): | ||
| """ | ||
| Test that there are appropriate alerts raised when ocs version of client | ||
| is changed to a different version and those alerts are cleared when the | ||
| heartbeat is resumed. During the test is stopped heartbeat cronjob on | ||
| client in order to stop overwritting the version set for testing. When the | ||
| heartbeat is resumed thereshould be also resumed version reporting so the | ||
| version should contain previous version. | ||
| """ | ||
| api = prometheus.PrometheusAPI(threading_lock=threading_lock) | ||
|
|
||
| # get alerts from time when manager deployment was scaled down | ||
| alerts = measure_change_client_ocs_version_and_stop_heartbeat.get( | ||
| "prometheus_alerts" | ||
| ) | ||
| client_name = measure_change_client_ocs_version_and_stop_heartbeat.get( | ||
| "metadata" | ||
| ).get("client_name") | ||
| target_alerts = [ | ||
| { | ||
| "label": constants.ALERT_STORAGECLIENTHEARTBEATMISSED, | ||
| "msg": f"Storage Client ({client_name}) heartbeat missed for more than 120 (s). " | ||
| "Lossy network connectivity might exist", | ||
| }, | ||
| { | ||
| "label": constants.ALERT_STORAGECLIENTINCOMPATIBLEOPERATORVERSION, | ||
| "msg": f"Storage Client Operator ({client_name}) differs by more " | ||
| "than 1 minor version. Client configuration may be incompatible and unsupported", | ||
| }, | ||
| ] | ||
| states = ["firing"] | ||
|
|
||
| for target_alert in target_alerts: | ||
| prometheus.check_alert_list( | ||
| label=target_alert["label"], | ||
| msg=target_alert["msg"], | ||
| alerts=alerts, | ||
| states=states, | ||
| severity="error", | ||
| ) | ||
| prometheus.check_alert_list( | ||
| label=target_alert["label"], | ||
| msg=target_alert["msg"], | ||
| alerts=alerts, | ||
| states=states, | ||
| severity="warning", | ||
| ) | ||
| api.check_alert_cleared( | ||
| label=target_alert["label"], | ||
| measure_end_time=measure_change_client_ocs_version_and_stop_heartbeat.get( | ||
| "stop" | ||
| ), | ||
| time_min=300, | ||
| ) | ||
|
|
||
|
|
||
| def teardown_module(): | ||
| ocs_obj = OCP() | ||
| ocs_obj.login_as_sa() |