Merge branch 'main' into nm/access-edges-perforomance-optimization

.github/CODEOWNERS

@@ -1,4 +1 @@
 *       @lloesche @aquamatthias @meln1k
-*.md    @TheCatLady @lloesche @aquamatthias @meln1k
-
-/.github/workflows/publish.yml    @TheCatLady

.github/workflows/publish.yml

@@ -82,7 +82,7 @@ jobs:
         shell: bash
         run: |
           yq '.services.fixcore.environment += "FIXCORE_MODEL_FROM_PLUGINS=true"' docker-compose.yaml > docker-compose-model-gen.yaml
-          PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker-compose -f docker-compose-model-gen.yaml up -d
+          PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker compose -f docker-compose-model-gen.yaml up -d --quiet-pull
           cd ${{ github.workspace }}/docs.fix.security/docs/resources
           python3 ${{ github.workspace }}/docs.fix.security/tools/export_models.py
 
@@ -346,7 +346,7 @@ jobs:
         shell: bash
         run: |
           yq '.services.fixcore.environment += "FIXCORE_MODEL_FROM_PLUGINS=true"' docker-compose.yaml > docker-compose-model-gen.yaml
-          PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker-compose -f docker-compose-model-gen.yaml up -d
+          PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker compose -f docker-compose-model-gen.yaml up -d --quiet-pull
           cd ${{ github.workspace }}/inventory.fix.security/versioned_docs/version-${{ steps.release.outputs.docsVersion }}/reference/unified-data-model
           python3 ${{ github.workspace }}/inventory.fix.security/tools/export_models.py
 

README.md

@@ -11,7 +11,7 @@ Fix Inventory detects compliance and security risks in cloud infrastructure acco
 
 We built Fix Inventory for cloud and security engineers as an open source alternative to proprietary cloud security tools like Orca Security, Prisma Cloud or Wiz.
 
-Check out our [Quick Start Guide](https://inventory.fix.security/docs/getting-started/) for step-by-step instructions on getting started. 
+Check out our [Quick Start Guide](https://fixinventory.org/getting-started) for step-by-step instructions on getting started. 
 
 ## 💡Why Fix Inventory?
 

fixcore/fixcore/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "4.1.0"
+__version__ = "4.2.0"
 
 
 def version() -> str:

fixcore/fixcore/analytics/posthog.py

@@ -27,7 +27,7 @@ def __init__(
         system_data: SystemData,
         flush_at: int = 10000,
         interval: timedelta = timedelta(minutes=1),
-        host: str = "https://analytics.some.engineering",
+        host: Optional[str] = None,  # was: "https://analytics.some.engineering",
         client_flush_interval: float = 0.5,
         client_retries: int = 3,
     ):

fixcore/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventorycore"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 description = "Keeps all the things."
 license = { text="AGPLv3" }
@@ -34,7 +34,7 @@ dependencies = [
     "aiofiles",
     "aiohttp-jinja2",
     "aiohttp-swagger3",
-    "aiohttp[speedups]",
+    "aiohttp[speedups] < 3.11",
     "cryptography",
     "deepdiff",
     "detect_secrets",

fixlib/fixlib/__init__.py

@@ -1 +1 @@
-__version__ = "4.1.0"
+__version__ = "4.2.0"

fixlib/fixlib/baseresources.py

@@ -116,8 +116,7 @@ def get(self) -> Dict[str, Any]:
         return changes
 
 
-# todo: replace to StrEnum once resoto is on 3.11
-class MetricName(str, Enum):
+class MetricName(StrEnum):
     def __str__(self) -> str:
         return self.value
 
@@ -150,6 +149,8 @@ def __str__(self) -> str:
 
     # load balancers
     RequestCount = "request"  # _count will be added to the end because of the unit
+    RequestBytesCount = "request_bytes"  # _count will be added to the end because of the unit
+    ResponseBytesCount = "response_bytes"  # _count will be added to the end because of the unit
     ActiveConnectionCount = "active_connection"  # _count will be added to the end because of the unit
     ALBActiveConnectionCount = "alb_active_connection"  # _count will be added to the end because of the unit
     ConnectionAttemptCount = "connection_attempt"  # _count will be added to the end because of the unit
@@ -195,6 +196,8 @@ def __str__(self) -> str:
     DiskQueueDepth = "disk_queue_depth"
     NetworkReceiveThroughput = "network_receive_throughput"
     NetworkTransmitThroughput = "network_transmit_throughput"
+    NetworkBytesSent = "network_bytes_sent"
+    NetworkBytesReceived = "network_bytes_received"
 
     # serverless
     Invocations = "invocations"
@@ -1109,7 +1112,7 @@ class BaseQueue(BaseResource):
     _categories: ClassVar[List[Category]] = [Category.storage]
     queue_type: Optional[QueueType] = None
     approximate_message_count: Optional[int] = None
-    message_retention_period: Optional[int] = None
+    message_retention_period_days: Optional[int] = None
 
 
 @define(eq=False, slots=False)

fixlib/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventorylib"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{ name = "Some Engineering Inc." }]
 description = "Fix Inventory common library."
 license = { text = "AGPLv3" }

fixmetrics/fixmetrics/__init__.py

@@ -1 +1 @@
-__version__ = "4.1.0"
+__version__ = "4.2.0"

fixmetrics/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventorymetrics"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 description = "Exports Fix Inventory metrics in Prometheus format."
 license = { text="AGPLv3" }
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0"
+    "fixinventorylib==4.2.0"
 ]
 
 [pyproject.optional-dependencies]

fixshell/fixshell/__init__.py

@@ -1 +1 @@
-__version__ = "4.1.0"
+__version__ = "4.2.0"

fixshell/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventoryshell"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 description = "Commandline interpreter to interact with Fix Inventory."
 license = { text="AGPLv3" }
@@ -28,7 +28,7 @@ readme = {file="README.md", content-type="text/markdown"}
 keywords = ["cloud security"]
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "prompt-toolkit",
     "rich",
     "fixinventoryclient",

fixworker/fixworker/__init__.py

@@ -1 +1 @@
-__version__ = "4.1.0"
+__version__ = "4.2.0"

fixworker/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventoryworker"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{ name = "Some Engineering Inc." }]
 description = "Runs collector plugins and sends the result to fixcore."
 license = { text="AGPLv3" }
@@ -9,7 +9,7 @@ classifiers = ["Programming Language :: Python :: 3"]
 readme = { file = "README.md", content-type = "text/markdown" }
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "tenacity",
     "CherryPy",
 ]

plugins/aws/fix_plugin_aws/collector.py

@@ -237,11 +237,11 @@ def get_last_run() -> Optional[datetime]:
                 try:
                     log.info(f"[Aws:{self.account.id}] Collect usage metrics.")
                     self.collect_usage_metrics(global_builder)
+                    shared_queue.wait_for_submitted_work()
                 except Exception as e:
                     log.warning(
                         f"Failed to collect usage metrics on account {self.account.id} in region {global_builder.region.id}: {e}"
                     )
-            shared_queue.wait_for_submitted_work()
 
             # connect nodes
             log.info(f"[Aws:{self.account.id}] Connect resources and create edges.")
@@ -325,8 +325,9 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> None:
                 continue
             # region can be overridden in the query: s3 is global, but need to be queried per region
             if region := cast(AwsRegion, resource.region()):
-                lookup_map[resource.id] = resource
                 resource_queries: List[cloudwatch.AwsCloudwatchQuery] = resource.collect_usage_metrics(builder)
+                if resource_queries:
+                    lookup_map[resource.id] = resource
                 for query in resource_queries:
                     query_region = query.region or region
                     start = query.start_delta or builder.metrics_delta

plugins/aws/fix_plugin_aws/resource/guardduty.py

@@ -699,7 +699,9 @@ def add_finding(
                     "FindingIds",
                     expected_errors=["BadRequestException"],
                     DetectorId=detector_id,
-                    FindingCriteria={"Criterion": {"accountId": {"Eq": [builder.account.id]}}},
+                    FindingCriteria={
+                        "Criterion": {"accountId": {"Eq": [builder.account.id]}, "service.archived": {"Eq": ["false"]}},
+                    },
                 ): detector_id
                 for detector_id in detector_ids
             }

plugins/aws/fix_plugin_aws/resource/inspector.py

@@ -115,8 +115,7 @@ def parse_finding(self, source: Json) -> Finding:
         if self.remediation and self.remediation.recommendation:
             remediation = self.remediation.recommendation.text or ""
         updated_at = self.updated_at
-        details = source.get("packageVulnerabilityDetails", {}) | source.get("codeVulnerabilityDetails", {})
-        return Finding(finding_title, finding_severity, description, remediation, updated_at, details)
+        return Finding(finding_title, finding_severity, description, remediation, updated_at, None)
 
     @classmethod
     def collect_resources(cls, builder: GraphBuilder) -> None:
@@ -151,7 +150,10 @@ def add_finding(
                 action="list-findings",
                 result_name="findings",
                 expected_errors=["AccessDeniedException"],
-                filterCriteria={"awsAccountId": [{"comparison": "EQUALS", "value": f"{builder.account.id}"}]},
+                filterCriteria={
+                    "awsAccountId": [{"comparison": "EQUALS", "value": f"{builder.account.id}"}],
+                    "findingStatus": [{"comparison": "EQUALS", "value": "ACTIVE"}],
+                },
             ):
                 if finding := AwsInspectorFinding.from_api(item, builder):
                     for fr in finding.finding_resources or []:

plugins/aws/fix_plugin_aws/resource/sqs.py

@@ -81,7 +81,7 @@ class AwsSqsQueue(AwsResource, BaseQueue, HasResourcePolicy):
         "sqs_delay_seconds": S("DelaySeconds") >> AsInt(),
         "sqs_receive_message_wait_time_seconds": S("ReceiveMessageWaitTimeSeconds") >> AsInt(),
         "sqs_managed_sse_enabled": S("SqsManagedSseEnabled") >> AsBool(),
-        "message_retention_period": S("MessageRetentionPeriod") >> AsInt(),
+        "message_retention_period_days": S("MessageRetentionPeriod") >> AsInt(),
         "approximate_message_count": S("ApproximateNumberOfMessages") >> AsInt(),
     }
     sqs_queue_url: Optional[str] = field(default=None)

plugins/aws/fix_plugin_aws/resource/ssm.py

@@ -384,7 +384,7 @@ class AwsSSMResourceCompliance(AwsResource, PhantomBaseResource):
     compliance_details: Optional[Dict[str, str]] = field(default=None, metadata={"description": "A Key:Value tag combination for the compliance item."})  # fmt: skip
 
     def parse_finding(self) -> Finding:
-        title = self.title or ""
+        title = self.title or (self.compliance_details or {}).get("DocumentName") or ""
         severity = SEVERITY_MAPPING.get(self.severity or "", Severity.medium)
         details = self.compliance_details
         if self.execution_summary:

plugins/aws/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventory-plugin-aws"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 description = "Runs collector plugins and sends the result to fixcore."
 license = { text="AGPLv3" }
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "fixinventorydata",
     "retrying",
     "boto3",

plugins/azure/fix_plugin_azure/resource/metrics.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from datetime import datetime, timedelta
 from concurrent.futures import as_completed
 import logging
@@ -271,12 +272,13 @@ def _query_for_single(
         interval: str,
     ) -> "Tuple[Optional[AzureMetricData], Optional[str]]":
         try:
+            local_api_spec = deepcopy(api_spec)
             # Set the path for the API call based on the instance ID of the query
-            api_spec.path = f"{query.instance_id}/providers/Microsoft.Insights/metrics"
+            local_api_spec.path = f"{query.instance_id}/providers/Microsoft.Insights/metrics"
             # Retrieve metric data from the API
             aggregation = ",".join(query.aggregation)
             part = builder.client.list(
-                api_spec,
+                local_api_spec,
                 metricnames=query.metric_name,
                 metricNamespace=query.metric_namespace,
                 timespan=timespan,

plugins/azure/fix_plugin_azure/resource/storage.py

@@ -26,6 +26,7 @@
     MetricUnit,
     ModelReference,
     PhantomBaseResource,
+    QueueType,
 )
 from fixlib.json_bender import K, Bender, S, ForallBend, Bend, AsBool
 from fixlib.types import Json
@@ -320,8 +321,8 @@ class AzureStorageQueue(MicrosoftResource, BaseQueue):
         "tags": S("tags", default={}),
         "name": S("name"),
         "queue_metadata": S("properties", "metadata"),
-        "queue_type": K("standard"),
-        "message_retention_period": K(7),
+        "queue_type": K(QueueType.STANDARD),
+        "message_retention_period_days": K(7),
         "approximate_message_count": S("properties", "approximateMessageCount"),
     }
     queue_metadata: Optional[Dict[str, str]] = field(default=None, metadata={'description': 'A name-value pair that represents queue metadata.'})  # fmt: skip

plugins/azure/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixinventory-plugin-azure"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 description = "Runs collector plugins and sends the result to fixcore."
 license = { text="AGPLv3" }
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "retrying",
     "azure-identity",
     "azure-mgmt-resource"

plugins/digitalocean/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "fixinventory-plugin-digitalocean"
 description = "Fix DigitalOcean Collector Plugin"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 license = { text="AGPLv3" }
 requires-python = ">=3.12"
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "boto3",
     "requests",
     "botocore",

plugins/dockerhub/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "fixinventory-plugin-dockerhub"
 description = "Fix Docker Hub Collector Plugin"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 license = { text="AGPLv3" }
 requires-python = ">=3.12"
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
     "requests",
 
 ]

plugins/example_collector/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "fixinventory-plugin-example-collector"
 description = "Fix Example Collector Plugin"
-version = "4.1.0"
+version = "4.2.0"
 authors = [{name="Some Engineering Inc."}]
 license = { text="AGPLv3" }
 requires-python = ">=3.12"
@@ -27,7 +27,7 @@ classifiers = [
 readme = {file="README.md", content-type="text/markdown"}
 
 dependencies = [
-    "fixinventorylib==4.1.0",
+    "fixinventorylib==4.2.0",
 ]
 
 [project.entry-points."fix.plugins"]