[azure][feat] Add cosmos-db resources collection (#2167)

Co-authored-by: Matthias Veit <[email protected]>

fixcore/fixcore/report/report_config.py

@@ -75,14 +75,18 @@ def from_config(cfg: ConfigEntity) -> List[ReportCheck]:
     @staticmethod
     def from_json(js: Json) -> List[ReportCheck]:
         def report_check(pdr: str, svc: str, check: Json) -> ReportCheck:
-            cr = check.copy()
-            cr["provider"] = pdr
-            cr["service"] = svc
-            cr["id"] = f"{pdr}_{svc}_{check['name']}"
-            # handle legacy result_kind
-            if "result_kind" in cr and "result_kinds" not in cr:
-                cr["result_kinds"] = [cr.pop("result_kind")]
-            return from_js(cr, ReportCheck)
+            try:
+                cr = check.copy()
+                cr["provider"] = pdr
+                cr["service"] = svc
+                cr["id"] = f"{pdr}_{svc}_{check['name']}"
+                # handle legacy result_kind
+                if "result_kind" in cr and "result_kinds" not in cr:
+                    cr["result_kinds"] = [cr.pop("result_kind")]
+                return from_js(cr, ReportCheck)
+            except Exception:
+                log.error(f"Failed to load check {check}")
+                raise
 
         pdr = js["provider"]
         svc = js["service"]

fixcore/tests/fixcore/report/report_config_test.py

@@ -0,0 +1,17 @@
+from typing import List
+
+import pytest
+from fixcore.report import ReportCheck
+from fixcore.report.report_config import ReportCheckCollectionConfig
+from fixcore.types import Json
+from fixlib.json import to_json
+
+
+def test_report_config(inspection_checks: List[ReportCheck]) -> None:
+    valid_js: List[Json] = to_json(inspection_checks)  # type: ignore
+    for a in valid_js:
+        a["name"] = a.pop("id")
+    icj = {"provider": "a", "service": "b", "checks": valid_js}
+    ReportCheckCollectionConfig.from_json(icj)
+    with pytest.raises(Exception):
+        ReportCheckCollectionConfig.from_json({"provider": "a", "service": "b", "checks": [{}]})

plugins/azure/fix_plugin_azure/collector.py

@@ -43,6 +43,10 @@
 )
 from fix_plugin_azure.resource.security import resources as security_resources
 from fix_plugin_azure.resource.sql_server import resources as sql_resources
+from fix_plugin_azure.resource.cosmosdb import (
+    AzureCosmosDBLocation,
+    resources as cosmosdb_resources,
+)
 from fix_plugin_azure.resource.storage import AzureStorageAccountUsage, AzureStorageSku, resources as storage_resources
 from fix_plugin_azure.resource.web import resources as web_resources
 from fixlib.baseresources import Cloud, GraphRoot, BaseAccount, BaseRegion
@@ -66,6 +70,7 @@ def resource_with_params(clazz: Type[MicrosoftResource], param: str) -> bool:
     + aks_resources
     + authorization_resources
     + compute_resources
+    + cosmosdb_resources
     + keyvault_resources
     + monitor_resources
     + mysql_resources
@@ -219,14 +224,17 @@ def collect_with(self, builder: GraphBuilder, locations: Dict[str, BaseRegion])
     def remove_unused(self) -> None:
         remove_nodes = []
 
-        def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None) -> None:  # type: ignore
+        def rm_nodes(cls, ignore_kinds: Optional[Type[Any]] = None, check_pred: bool = True) -> None:  # type: ignore
             for node in self.graph.nodes:
                 if not isinstance(node, cls):
                     continue
-                pred = list(self.graph.predecessors(node))
+                if check_pred:
+                    nodes = list(self.graph.predecessors(node))
+                else:
+                    nodes = list(self.graph.successors(node))
                 if ignore_kinds is not None:
-                    pred = [p for p in pred if not isinstance(p, ignore_kinds)]
-                if not pred:
+                    nodes = [n for n in nodes if not isinstance(n, ignore_kinds)]
+                if not nodes:
                     remove_nodes.append(node)
             self._delete_nodes(remove_nodes)
             log.debug(f"Removing {len(remove_nodes)} unreferenced nodes of type {cls}")
@@ -249,6 +257,8 @@ def remove_usage_zero_value() -> None:
         rm_nodes(AzureStorageSku, AzureLocation)
         rm_nodes(AzureMysqlServerType, AzureSubscription)
         rm_nodes(AzurePostgresqlServerType, AzureSubscription)
+        rm_nodes(AzureCosmosDBLocation, AzureSubscription, check_pred=False)
+        rm_nodes(AzureLocation, check_pred=False)
         remove_usage_zero_value()
 
     def after_collect(self) -> None:

plugins/azure/test/collector_test.py

@@ -49,8 +49,8 @@ def test_collect(
         config, Cloud(id="azure"), azure_subscription, credentials, core_feedback
     )
     subscription_collector.collect()
-    assert len(subscription_collector.graph.nodes) == 463
-    assert len(subscription_collector.graph.edges) == 701
+    assert len(subscription_collector.graph.nodes) == 492
+    assert len(subscription_collector.graph.edges) == 754
 
     graph_collector = MicrosoftGraphOrganizationCollector(
         config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback

plugins/azure/test/cosmos-db_test.py

@@ -0,0 +1,34 @@
+from conftest import roundtrip_check
+from fix_plugin_azure.resource.base import GraphBuilder
+from fix_plugin_azure.resource.cosmosdb import (
+    AzureCosmosDBCassandraCluster,
+    AzureCosmosDBAccount,
+    AzureCosmosDBRestorableAccount,
+    AzureCosmosDBLocation,
+    AzureCosmosDBMongoDBCluster,
+)
+
+
+def test_cassandra_cluster(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureCosmosDBCassandraCluster, builder)
+    assert len(collected) == 1
+
+
+def test_cosmos_db_account(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureCosmosDBAccount, builder)
+    assert len(collected) == 1
+
+
+def test_restorable_cosmos_db_account(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureCosmosDBRestorableAccount, builder)
+    assert len(collected) == 2
+
+
+def test_cosmos_db_location(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureCosmosDBLocation, builder)
+    assert len(collected) == 2
+
+
+def test_mongo_db_cluster(builder: GraphBuilder) -> None:
+    collected = roundtrip_check(AzureCosmosDBMongoDBCluster, builder)
+    assert len(collected) == 1

plugins/azure/test/files/cosmos-db/cassandraClusters.json

@@ -0,0 +1,65 @@
+{
+    "value": [
+      {
+        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/cassandraClusters",
+        "name": "cassandra-prod",
+        "type": "Microsoft.DocumentDB/cassandraClusters",
+        "location": "West US",
+        "tags": {},
+        "properties": {
+          "provisioningState": "Succeeded",
+          "delegatedManagementSubnetId": "/subscriptions/536e130b-d7d6-4ac7-98a5-de20d69588d2/resourceGroups/customer-vnet-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/management",
+          "cassandraVersion": "3.11",
+          "hoursBetweenBackups": 24,
+          "authenticationMethod": "Cassandra",
+          "externalSeedNodes": [
+            {
+              "ipAddress": "10.52.221.2"
+            },
+            {
+              "ipAddress": "10.52.221.3"
+            },
+            {
+              "ipAddress": "10.52.221.4"
+            }
+          ],
+          "clusterNameOverride": "ClusterNameIllegalForAzureResource",
+          "seedNodes": [
+            {
+              "ipAddress": "10.52.221.2"
+            },
+            {
+              "ipAddress": "10.52.221.3"
+            },
+            {
+              "ipAddress": "10.52.221.4"
+            },
+            {
+              "ipAddress": "192.168.12.2"
+            },
+            {
+              "ipAddress": "192.168.12.3"
+            },
+            {
+              "ipAddress": "192.168.12.4"
+            }
+          ],
+          "clientCertificates": [
+            {
+              "pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
+            }
+          ],
+          "externalGossipCertificates": [
+            {
+              "pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
+            }
+          ],
+          "gossipCertificates": [
+            {
+              "pem": "-----BEGIN CERTIFICATE-----\n...Base64 encoded certificate...\n-----END CERTIFICATE-----"
+            }
+          ]
+        }
+      }
+    ]
+  }

plugins/azure/test/files/cosmos-db/dataCenters.json

@@ -0,0 +1,27 @@
+{
+    "value": [
+      {
+        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cassandra-prod-rg/providers/Microsoft.DocumentDB/cassandraClusters/cassandra-prod/dataCenters",
+        "name": "dc1",
+        "type": "Microsoft.DocumentDB/cassandraClusters/dataCenters",
+        "properties": {
+          "provisioningState": "Succeeded",
+          "dataCenterLocation": "West US 2",
+          "delegatedSubnetId": "/subscriptions/536e130b-d7d6-4ac7-98a5-de20d69588d2/resourceGroups/customer-vnet-rg/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/dc1",
+          "nodeCount": 9,
+          "seedNodes": [
+            {
+              "ipAddress": "192.168.12.2"
+            },
+            {
+              "ipAddress": "192.168.12.3"
+            },
+            {
+              "ipAddress": "192.168.12.4"
+            }
+          ],
+          "base64EncodedCassandraYamlFragment": "Y29tcGFjdGlvbl90aHJvdWdocHV0X21iX3Blcl9zZWM6IDMyCmNvbXBhY3Rpb25fbGFyZ2VfcGFydGl0aW9uX3dhcm5pbmdfdGhyZXNob2xkX21iOiAxMDA="
+        }
+      }
+    ]
+  }

plugins/azure/test/files/cosmos-db/databaseAccounts.json

@@ -0,0 +1,123 @@
+{
+    "value": [
+      {
+        "id": "/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.DocumentDB/databaseAccounts/ddb1",
+        "name": "ddb1",
+        "location": "West US",
+        "type": "Microsoft.DocumentDB/databaseAccounts",
+        "kind": "GlobalDocumentDB",
+        "tags": {},
+        "properties": {
+          "provisioningState": "Succeeded",
+          "documentEndpoint": "https://ddb1.documents.azure.com:443/",
+          "ipRules": [],
+          "databaseAccountOfferType": "Standard",
+          "disableKeyBasedMetadataWriteAccess": false,
+          "consistencyPolicy": {
+            "defaultConsistencyLevel": "Session",
+            "maxIntervalInSeconds": 5,
+            "maxStalenessPrefix": 100
+          },
+          "writeLocations": [
+            {
+              "id": "ddb1-eastus",
+              "locationName": "East US",
+              "documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
+              "provisioningState": "Succeeded",
+              "failoverPriority": 0
+            }
+          ],
+          "readLocations": [
+            {
+              "id": "ddb1-eastus",
+              "locationName": "East US",
+              "documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
+              "provisioningState": "Succeeded",
+              "failoverPriority": 0
+            }
+          ],
+          "locations": [
+            {
+              "id": "ddb1-eastus",
+              "locationName": "East US",
+              "documentEndpoint": "https://ddb1-eastus.documents.azure.com:443/",
+              "provisioningState": "Succeeded",
+              "failoverPriority": 0
+            }
+          ],
+          "failoverPolicies": [
+            {
+              "id": "ddb1-eastus",
+              "locationName": "East US",
+              "failoverPriority": 0
+            }
+          ],
+          "privateEndpointConnections": [
+            {
+              "id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.DocumentDB/databaseAccounts/account1/privateEndpointConnections/pe1",
+              "properties": {
+                "privateEndpoint": {
+                  "id": "/subscriptions/subId/resourceGroups/rg/providers/Microsoft.Network/privateEndpoints/pe1"
+                },
+                "privateLinkServiceConnectionState": {
+                  "status": "Approved",
+                  "actionsRequired": "None"
+                }
+              }
+            }
+          ],
+          "cors": [],
+          "defaultIdentity": "FirstPartyIdentity",
+          "enableFreeTier": false,
+          "apiProperties": {},
+          "enableAnalyticalStorage": true,
+          "enableBurstCapacity": true,
+          "analyticalStorageConfiguration": {
+            "schemaType": "WellDefined"
+          },
+          "instanceId": "d9b26648-2f53-4541-b3d8-3044f4f9810d",
+          "createMode": "Default",
+          "backupPolicy": {
+            "type": "Periodic",
+            "periodicModeProperties": {
+              "backupIntervalInMinutes": 240,
+              "backupRetentionIntervalInHours": 720,
+              "backupStorageRedundancy": "Geo"
+            }
+          },
+          "networkAclBypass": "None",
+          "networkAclBypassResourceIds": [],
+          "keysMetadata": {
+            "primaryMasterKey": {
+              "generationTime": "2022-02-25T20:30:11Z"
+            },
+            "secondaryMasterKey": {
+              "generationTime": "2022-02-25T20:30:11Z"
+            },
+            "primaryReadonlyMasterKey": {
+              "generationTime": "2022-02-25T20:30:11Z"
+            },
+            "secondaryReadonlyMasterKey": {
+              "generationTime": "2022-02-25T20:30:11Z"
+            }
+          },
+          "enablePartitionMerge": true,
+          "minimalTlsVersion": "Tls"
+        },
+        "systemData": {
+          "createdAt": "2021-03-12T22:05:09Z"
+        },
+        "identity": {
+          "type": "SystemAssigned,UserAssigned",
+          "principalId": "52f4fef3-3c3f-4ff3-b52e-b5c9eeb68656",
+          "tenantId": "33e01921-4d64-4f8c-a055-5bdaffd5e33d",
+          "userAssignedIdentities": {
+            "/subscriptions/fa5fc227-a624-475e-b696-cdd604c735bc/resourceGroups/eu2cgroup/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1": {
+              "clientId": "fbe75b66-01c5-4f87-a220-233af3270436",
+              "principalId": "33e01921-4d64-4f8c-a055-5bdaffd5e33d"
+            }
+          }
+        }
+      }
+    ]
+  }

plugins/azure/test/files/cosmos-db/locations.json

@@ -0,0 +1,36 @@
+{
+    "value": [
+      {
+        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/locations/westus",
+        "type": "Microsoft.DocumentDB/locations",
+        "name": "westus",
+        "properties": {
+          "supportsAvailabilityZone": false,
+          "isResidencyRestricted": false,
+          "backupStorageRedundancies": [
+            "Local",
+            "Geo"
+          ],
+          "isSubscriptionRegionAccessAllowedForRegular": true,
+          "isSubscriptionRegionAccessAllowedForAz": false,
+          "status": "Online"
+        }
+      },
+      {
+        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.DocumentDB/locations/centralus",
+        "type": "Microsoft.DocumentDB/locations",
+        "name": "centralus",
+        "properties": {
+          "supportsAvailabilityZone": true,
+          "isResidencyRestricted": false,
+          "backupStorageRedundancies": [
+            "Zone",
+            "Geo"
+          ],
+          "isSubscriptionRegionAccessAllowedForRegular": false,
+          "isSubscriptionRegionAccessAllowedForAz": true,
+          "status": "Online"
+        }
+      }
+    ]
+  }