From 148c3dfc9a01e21919ca0afc0cd1df840ba6d83c Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 14:35:22 -0400 Subject: [PATCH 1/6] Changed output of function and bugfix Change: `get_machine_sync_details` has been changed back to returning a dict of values. The machine ID will be the dict key for the CE machines returned from a project. Bugfix: CE machines not in a `STARTED` state may have an empty `lastSeenDateTime` in its returned response from the API. This is better guarded against in the report result. --- cloudendure/cloudendure.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cloudendure/cloudendure.py b/cloudendure/cloudendure.py index e1bd315f1..04ddbadb9 100644 --- a/cloudendure/cloudendure.py +++ b/cloudendure/cloudendure.py @@ -447,34 +447,35 @@ def check_licenses(self) -> Dict[str, Any]: return response_dict - def get_machine_sync_details(self) -> List[Any]: + def get_machine_sync_details(self) -> Dict[Any]: """Checks CloudEndure Project inventory and returns register machine replication state. """ - response_list: List[Any] = [] print(f"INFO: Retreiving sync status for all machines in Project: ({self.project_name})") machines_response: Response = self.api.api_call(f"projects/{self.project_id}/machines") if not machines_response.ok: print(f"ERROR: API response did not return a 2XX status; Returned {machines_response.status_code} ...") return {} + response_dict: Dict[str, Any] = {} ce_project_inventory = json.loads(machines_response.text).get("items", []) for _query_value in ce_project_inventory: - machine_name: str = _query_value["sourceProperties"]["name"] - sync_details: Dict[str, Any] = { - "machine_name": machine_name, + machine_id: str = _query_value["id"] + response_dict[machine_id] = { + "machine_name": _query_value["sourceProperties"]["name"], "in_inventory": _query_value["isAgentInstalled"], "replication_status": _query_value["replicationStatus"], - "last_seen_utc": _query_value["replicationInfo"]["lastSeenDateTime"], + "last_seen_utc": "N/A", "total_storage_bytes": _query_value["replicationInfo"]["totalStorageBytes"], "replicated_storage_bytes": _query_value["replicationInfo"]["replicatedStorageBytes"], "rescanned_storage_bytes": 0, "backlogged_storage_bytes": _query_value["replicationInfo"]["backloggedStorageBytes"], } - if "rescannedStorageBytes" in _query_value["replicationInfo"]: - sync_details["recanned_storage_bytes"] = _query_value["replicationInfo"]["rescannedStorageBytes"] - response_list.append(sync_details) + if "lastSeenDateTime" in _query_value["replicationInfo"].keys(): + response_dict[machine_id]["last_seen_utc"] = _query_value["replicationInfo"]["lastSeenDateTime"] + if "rescannedStorageBytes" in _query_value["replicationInfo"].keys(): + response_dict[machine_id]["rescanned_storage_bytes"] = _query_value["replicationInfo"]["rescannedStorageBytes"] # Project is still printing to console as a convention; Emitting an # output to stdout for interactive usage - return response_list + return response_dict def inspect_ce_project(self, check_type: str) -> List[Any]: if not check_type: From f16419b7449ae5a8d4ac88bca86afa4645f82ac9 Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 14:41:51 -0400 Subject: [PATCH 2/6] version bump to 0.3.2 --- Makefile | 2 +- cloudendure/__version__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6bd99acac..fcc636c40 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SHA1 := $$(git log -1 --pretty=%h) CURRENT_BRANCH := $$(git symbolic-ref -q --short HEAD) LATEST_TAG := ${REPO_NAME}:latest GIT_TAG := ${REPO_NAME}:${SHA1} -VERSION := v0.3.1 +VERSION := v0.3.2 info: ## Show information about the current git state. @echo "Github Project: https://github.com/${REPO_NAME}\nCurrent Branch: ${CURRENT_BRANCH}\nSHA1: ${SHA1}\n" diff --git a/cloudendure/__version__.py b/cloudendure/__version__.py index 260c070a8..f9aa3e110 100644 --- a/cloudendure/__version__.py +++ b/cloudendure/__version__.py @@ -1 +1 @@ -__version__ = "0.3.1" +__version__ = "0.3.2" diff --git a/pyproject.toml b/pyproject.toml index 8fb7886b5..3886fe505 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cloudendure" -version = "0.3.1" +version = "0.3.2" description = "Python wrapper and CLI for CloudEndure" authors = ["Mark Beacom ", "Tom Warnock "] maintainers = ["Evan Lucchesi ", "Nick Selpa "] From f28fb26bc9bcb27b3fa7cfb20518723b2bc6f685 Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 14:47:20 -0400 Subject: [PATCH 3/6] Applied linter --- cloudendure/cloudendure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudendure/cloudendure.py b/cloudendure/cloudendure.py index 04ddbadb9..f8a8bab06 100644 --- a/cloudendure/cloudendure.py +++ b/cloudendure/cloudendure.py @@ -472,7 +472,9 @@ def get_machine_sync_details(self) -> Dict[Any]: if "lastSeenDateTime" in _query_value["replicationInfo"].keys(): response_dict[machine_id]["last_seen_utc"] = _query_value["replicationInfo"]["lastSeenDateTime"] if "rescannedStorageBytes" in _query_value["replicationInfo"].keys(): - response_dict[machine_id]["rescanned_storage_bytes"] = _query_value["replicationInfo"]["rescannedStorageBytes"] + response_dict[machine_id]["rescanned_storage_bytes"] = _query_value["replicationInfo"][ + "rescannedStorageBytes" + ] # Project is still printing to console as a convention; Emitting an # output to stdout for interactive usage return response_dict From f3979d9c5872aac299b424d28c0a668f0f34ac7a Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 15:24:57 -0400 Subject: [PATCH 4/6] Refactored dictionary build for report result --- cloudendure/cloudendure.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/cloudendure/cloudendure.py b/cloudendure/cloudendure.py index f8a8bab06..7ecf37dd8 100644 --- a/cloudendure/cloudendure.py +++ b/cloudendure/cloudendure.py @@ -459,22 +459,18 @@ def get_machine_sync_details(self) -> Dict[Any]: ce_project_inventory = json.loads(machines_response.text).get("items", []) for _query_value in ce_project_inventory: machine_id: str = _query_value["id"] + replication_info: Dict[str, Any] = _query_value.get("replicationInfo", {}) response_dict[machine_id] = { - "machine_name": _query_value["sourceProperties"]["name"], + "machine_name": _query_value.get("sourceProperties",{}).get("name",""), "in_inventory": _query_value["isAgentInstalled"], "replication_status": _query_value["replicationStatus"], - "last_seen_utc": "N/A", - "total_storage_bytes": _query_value["replicationInfo"]["totalStorageBytes"], - "replicated_storage_bytes": _query_value["replicationInfo"]["replicatedStorageBytes"], - "rescanned_storage_bytes": 0, - "backlogged_storage_bytes": _query_value["replicationInfo"]["backloggedStorageBytes"], + "total_storage_bytes": replication_info.get("totalStorageBytes", -1), + "replicated_storage_bytes": replication_info.get("replicatedStorageBytes", -1), + "rescanned_storage_bytes": replication_info.get("rescannedStorageBytes", 0), + "backlogged_storage_bytes": replication_info.get("backloggedStorageBytes", -1), } - if "lastSeenDateTime" in _query_value["replicationInfo"].keys(): - response_dict[machine_id]["last_seen_utc"] = _query_value["replicationInfo"]["lastSeenDateTime"] - if "rescannedStorageBytes" in _query_value["replicationInfo"].keys(): - response_dict[machine_id]["rescanned_storage_bytes"] = _query_value["replicationInfo"][ - "rescannedStorageBytes" - ] + if replication_info.get("lastSeenDateTime"): + response_dict[machine_id]["last_seen_utc"] = replication_info.get("lastSeenDateTime") # Project is still printing to console as a convention; Emitting an # output to stdout for interactive usage return response_dict @@ -509,12 +505,12 @@ def not_started(self, machine) -> bool: return True def not_current(self, machine, delta_seconds: int = 86400) -> bool: - now: datetime = datetime.now(timezone.utc) - machine_last_seen: datetime = datetime.fromisoformat(machine["last_seen_utc"]) - last_seen_delta: datetime = now - machine_last_seen - # If you're exceeding the size of int, you have bigger problems - if int(last_seen_delta.total_seconds()) >= delta_seconds: - return True + if machine.get("last_seen_utc"): + now: datetime = datetime.now(datetime.timezone.utc) + machine_last_seen: datetime = datetime.datetime.fromisoformat(machine["last_seen_utc"]) + last_seen_delta: datetime = now - machine_last_seen + if int(last_seen_delta.total_seconds()) >= delta_seconds: + return True else: return False From 73a7359551123dd8a39aaeff38822c576964a158 Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 15:47:57 -0400 Subject: [PATCH 5/6] Updated valid_check_types and changed inspect_ce_project to return a dict --- cloudendure/cloudendure.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cloudendure/cloudendure.py b/cloudendure/cloudendure.py index 7ecf37dd8..77c32c7e4 100644 --- a/cloudendure/cloudendure.py +++ b/cloudendure/cloudendure.py @@ -475,20 +475,21 @@ def get_machine_sync_details(self) -> Dict[Any]: # output to stdout for interactive usage return response_dict - def inspect_ce_project(self, check_type: str) -> List[Any]: - if not check_type: + def inspect_ce_project(self, check_type: str) -> Dict[str, Any]: + valid_check_types: List[str] = ['not_synced', 'not_started', 'not_current'] + if check_type not in valid_check_types: print( - f"ERROR: Unknown check_type of '{check_type}'; Please use 'not_synced', 'not_started', or 'not_current' ..." + f'ERROR: Unknown check_type of "{check_type}"; Please use a valid check_type: [ {", ".join(valid_check_types)} ] ...' ) return - result: List[Any] = [] - sync_report: List[Any] = self.get_machine_sync_details() + result: Dict[str, Any] = {} + sync_report: Dict[str, Any] = self.get_machine_sync_details() print(f"INFO: Using check '{check_type}' on Project: ({self.project_name})") inspector = getattr(self, check_type) - for item in sync_report: - mcheck = inspector(machine=item) + for item in sync_report.keys(): + mcheck = inspector(machine=sync_report[item]) if mcheck: - result.append(item) + result[item] = sync_report[item] print(f"INFO: Check '{check_type}' completed; {len(result)} machines matched in Project: ({self.project_name})") return result @@ -506,7 +507,7 @@ def not_started(self, machine) -> bool: def not_current(self, machine, delta_seconds: int = 86400) -> bool: if machine.get("last_seen_utc"): - now: datetime = datetime.now(datetime.timezone.utc) + now: datetime = datetime.datetime.now(datetime.timezone.utc) machine_last_seen: datetime = datetime.datetime.fromisoformat(machine["last_seen_utc"]) last_seen_delta: datetime = now - machine_last_seen if int(last_seen_delta.total_seconds()) >= delta_seconds: From fcff0b81e15abc0fb5d14fef5e5c847141934a04 Mon Sep 17 00:00:00 2001 From: Nick Selpa Date: Tue, 26 May 2020 15:48:37 -0400 Subject: [PATCH 6/6] Applied black linter --- cloudendure/cloudendure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudendure/cloudendure.py b/cloudendure/cloudendure.py index 77c32c7e4..fb43a743d 100644 --- a/cloudendure/cloudendure.py +++ b/cloudendure/cloudendure.py @@ -461,7 +461,7 @@ def get_machine_sync_details(self) -> Dict[Any]: machine_id: str = _query_value["id"] replication_info: Dict[str, Any] = _query_value.get("replicationInfo", {}) response_dict[machine_id] = { - "machine_name": _query_value.get("sourceProperties",{}).get("name",""), + "machine_name": _query_value.get("sourceProperties", {}).get("name", ""), "in_inventory": _query_value["isAgentInstalled"], "replication_status": _query_value["replicationStatus"], "total_storage_bytes": replication_info.get("totalStorageBytes", -1), @@ -476,7 +476,7 @@ def get_machine_sync_details(self) -> Dict[Any]: return response_dict def inspect_ce_project(self, check_type: str) -> Dict[str, Any]: - valid_check_types: List[str] = ['not_synced', 'not_started', 'not_current'] + valid_check_types: List[str] = ["not_synced", "not_started", "not_current"] if check_type not in valid_check_types: print( f'ERROR: Unknown check_type of "{check_type}"; Please use a valid check_type: [ {", ".join(valid_check_types)} ] ...'