Merge branch 'main' into nm/access-edge-creator-process

fixcore/fixcore/cli/command.py

@@ -18,7 +18,7 @@
 from collections import defaultdict
 from contextlib import suppress
 from datetime import timedelta, datetime
-from functools import partial, lru_cache
+from functools import partial, lru_cache, cached_property
 from itertools import dropwhile, chain
 from pathlib import Path
 from typing import (
@@ -2444,8 +2444,12 @@ class PropToShow:
     alternative_path: Optional[List[str]] = None
     override_kind: Optional[str] = None
 
+    @cached_property
+    def path_str(self) -> str:
+        return ".".join(self.path)
+
     def full_path(self) -> str:
-        return self.path_access or ".".join(self.path)
+        return self.path_access or self.path_str
 
     def value(self, node: JsonElement) -> Optional[JsonElement]:
         result = js_value_at(node, self.path)
@@ -2502,8 +2506,9 @@ class ListCommand(CLICommand, OutputTransformer):
     ## Options
 
     - `--csv` [optional]: format the output as CSV. Can't be used together with `--markdown`.
-
     - `--markdown` [optional]: format the output as Markdown table. Can't be used together with `--csv`.
+    - `--json-table` [optional]: format the output as JSON table.
+    - `--with-defaults` [optional]: show the default properties in addition to the defined ones.
 
     ## Examples
 
@@ -2619,6 +2624,7 @@ def args_info(self) -> ArgsInfo:
             ArgInfo("--csv", help_text="format", option_group="format"),
             ArgInfo("--markdown", help_text="format", option_group="format"),
             ArgInfo("--json-table", help_text="format", option_group="format"),
+            ArgInfo("--with-defaults"),
             ArgInfo(
                 expects_value=True,
                 help_text="comma separated list of properties to show",
@@ -2632,6 +2638,7 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa
         output_type.add_argument("--csv", dest="csv", action="store_true")
         output_type.add_argument("--markdown", dest="markdown", action="store_true")
         output_type.add_argument("--json-table", dest="json_table", action="store_true")
+        parser.add_argument("--with-defaults", dest="with_defaults", action="store_true")
         parsed, properties_list = parser.parse_known_args(arg.split() if arg else [])
         properties = " ".join(properties_list) if properties_list else None
         is_aggregate: bool = ctx.query is not None and ctx.query.aggregate is not None
@@ -2766,7 +2773,15 @@ def unique_name(path: List[str], current: str) -> str:
         def props_to_show(
             props_setting: Tuple[List[PropToShow], List[PropToShow], List[PropToShow], List[PropToShow]]
         ) -> List[PropToShow]:
-            props = parse_props_to_show(properties) if properties is not None else default_props_to_show(props_setting)
+            if properties:
+                props = parse_props_to_show(properties)
+                if parsed.with_defaults:
+                    paths = {prop.path_str for prop in props}
+                    for prop in default_props_to_show(props_setting):
+                        if prop.path_str not in paths:
+                            props.append(prop)
+            else:
+                props = default_props_to_show(props_setting)
             return create_unique_names(props)
 
         def fmt_json(elem: Json) -> JsonElement:

fixcore/fixcore/db/arango_query.py

@@ -881,36 +881,56 @@ def inout(
             in_crsr: str, start: int, until: int, edge_type: str, direction: str, edge_filter: Optional[Term]
         ) -> str:
             nonlocal query_part
-            in_c = ctx.next_crs("io_in")
+            start_c = ctx.next_crs("graph_start")
+            in_c = ctx.next_crs("gc")
+            in_edge = f"{in_c}_edge"
+            in_path = f"{in_c}_path"
+            in_r = f"{in_c}_result"
             out = ctx.next_crs("io_out")
-            out_crsr = ctx.next_crs("io_crs")
-            e = ctx.next_crs("io_link")
             unique = "uniqueEdges: 'path'" if with_edges else "uniqueVertices: 'global'"
             dir_bound = "OUTBOUND" if direction == Direction.outbound else "INBOUND"
-            inout_result = (
-                # merge edge and vertex properties - will be split in the output transformer
-                f"MERGE({out_crsr}, {{_from:{e}._from, _to:{e}._to, _link_id:{e}._id, _link_reported:{e}.reported}})"
-                if with_edges
-                else out_crsr
-            )
+
+            # the path array contains the whole path from the start node.
+            # in the case of start > 0, we need to slice the array to get the correct part
+            def slice_or_all(in_p_part: str) -> str:
+                return f"SLICE({in_path}.{in_p_part}, {start})" if start > 0 else f"{in_path}.{in_p_part}"
+
+            # Edge filter: decision to include the source element is not possible while traversing it.
+            #              When the target node is reached and edge properties are available, the decision can be made.
+            #              In case the filter succeeds, we need to select all vertices and edges on the path.
+            # No filter but with_edges: another nested for loop required to return the node and edge
+            # No filter and no with_edges: only the node is returned
+            if edge_filter:
+                # walk the path and return all vertices (and possibly edges)
+                # this means intermediate nodes are returned multiple times and have to be made distinct
+                # since we return nodes first, the edges can always be resolved
+                walk_array = slice_or_all("vertices")
+                walk_array = f'APPEND({walk_array}, {slice_or_all("edges")})' if with_edges else walk_array
+                inout_result = f"FOR {in_r} in {walk_array} RETURN DISTINCT({in_r})"
+            elif with_edges:
+                # return the node and edge via a nested for loop
+                inout_result = f"FOR {in_r} in [{in_c}, {in_edge}] FILTER {in_r}!=null RETURN DISTINCT({in_r})"
+            else:
+                # return only the node
+                inout_result = f"RETURN DISTINCT {in_c}"
+
             if outer_merge and part_idx == 0:
                 graph_cursor = in_crsr
                 outer_for = ""
             else:
-                graph_cursor = in_c
-                outer_for = f"FOR {in_c} in {in_crsr} "
+                graph_cursor = start_c
+                outer_for = f"FOR {start_c} in {in_crsr} "
 
             # optional: add the edge filter to the query
-            pre, fltr, post = term(e, edge_filter) if edge_filter else (None, None, None)
+            pre, fltr, post = term(in_edge, edge_filter) if edge_filter else (None, None, None)
             pre_string = " " + pre if pre else ""
             post_string = f" AND ({post})" if post else ""
             filter_string = "" if not fltr and not post_string else f"{pre_string} FILTER {fltr}{post_string}"
             query_part += (
                 f"LET {out} =({outer_for}"
                 # suggested by jsteemann: use crs._id instead of crs (stored in the view and more efficient)
-                f"FOR {out_crsr}, {e} IN {start}..{until} {dir_bound} {graph_cursor}._id "
-                f"`{db.edge_collection(edge_type)}` OPTIONS {{ bfs: true, {unique} }}{filter_string} "
-                f"RETURN DISTINCT {inout_result})"
+                f"FOR {in_c}, {in_edge}, {in_path} IN {start}..{until} {dir_bound} {graph_cursor}._id "
+                f"`{db.edge_collection(edge_type)}` OPTIONS {{ bfs: true, {unique} }}{filter_string} {inout_result})"
             )
             return out
 

fixcore/fixcore/db/async_arangodb.py

@@ -53,7 +53,6 @@ def __init__(
         self.cursor_exhausted = False
         self.trafo: Callable[[Json], Optional[Any]] = trafo if trafo else identity  # type: ignore
         self.vt_len: Optional[int] = None
-        self.on_hold: Optional[Json] = None
         self.get_next: Callable[[], Awaitable[Optional[Json]]] = (
             self.next_filtered if flatten_nodes_and_edges else self.next_element
         )
@@ -62,11 +61,7 @@ async def __anext__(self) -> Any:
         # if there is an on-hold element: unset and return it
         # background: a graph node contains vertex and edge information.
         # since this method can only return one element at a time, the edge is put on-hold for vertex+edge data.
-        if self.on_hold:
-            res = self.on_hold
-            self.on_hold = None
-            return res
-        elif self.cursor_exhausted:
+        if self.cursor_exhausted:
             return await self.next_deferred_edge()
         else:
             try:
@@ -99,20 +94,10 @@ async def next_element(self) -> Optional[Json]:
 
     async def next_filtered(self) -> Optional[Json]:
         element = await self.next_from_db()
-        vertex: Optional[Json] = None
-        edge = None
         try:
-            _key = element["_key"]
-            if _key not in self.visited_node:
-                self.visited_node.add(_key)
-                vertex = self.trafo(element)
-
-            from_id = element.get("_from")
-            to_id = element.get("_to")
-            link_id = element.get("_link_id")
-            if from_id is not None and to_id is not None and link_id is not None:
-                if link_id not in self.visited_edge:
-                    self.visited_edge.add(link_id)
+            if (from_id := element.get("_from")) and (to_id := element.get("_to")) and (node_id := element.get("_id")):
+                if node_id not in self.visited_edge:
+                    self.visited_edge.add(node_id)
                     if not self.vt_len:
                         self.vt_len = len(re.sub("/.*$", "", from_id)) + 1
                     edge = {
@@ -122,21 +107,22 @@ async def next_filtered(self) -> Optional[Json]:
                         # example: vertex_name/node_id -> node_id
                         "to": to_id[self.vt_len :],  # noqa: E203
                         # example: vertex_name_default/edge_id -> default
-                        "edge_type": re.sub("/.*$", "", link_id[self.vt_len :]),  # noqa: E203
+                        "edge_type": re.sub("/.*$", "", node_id[self.vt_len :]),  # noqa: E203
                     }
-                    if reported := element.get("_link_reported"):
+                    if reported := element.get("reported"):
                         edge["reported"] = reported
                     # make sure that both nodes of the edge have been visited already
                     if from_id not in self.visited_node or to_id not in self.visited_node:
                         self.deferred_edges.append(edge)
-                        edge = None
-            # if the vertex is not returned: return the edge
-            # otherwise return the vertex and remember the edge
-            if vertex:
-                self.on_hold = edge
-                return vertex
+                        return None
+                    else:
+                        return edge
+            elif key := element.get("_key"):
+                if key not in self.visited_node:
+                    self.visited_node.add(key)
+                    return self.trafo(element)
             else:
-                return edge
+                return element
         except Exception as ex:
             log.warning(f"Could not read element {element}: {ex}. Ignore.")
         return None

fixcore/fixcore/model/graph_access.py

@@ -104,9 +104,14 @@ class EdgeTypes:
     # A resource can be deleted, if all outgoing resources are deleted.
     delete: EdgeType = "delete"
 
+    # This edge type defines the IAM relationship.
+    # It models allowed permissions between principals and resources, as well as the inter-principal relationship.
+    # Example: AWS IAM User (principal) has permission to write to an S3 bucket (resource).
+    iam: EdgeType = "iam"
+
     # The set of all allowed edge types.
     # Note: the database schema has to be adapted to support additional edge types.
-    all: Set[EdgeType] = {default, delete}
+    all: Set[EdgeType] = {default, delete, iam}
 
 
 class Direction:

fixcore/fixcore/types.py

@@ -21,7 +21,7 @@
 ValidationResult = Optional[Any]
 ValidationFn = Callable[[Any], ValidationResult]
 
-EdgeType = Literal["default", "delete"]
+EdgeType = Literal["default", "delete", "iam"]
 
 
 # make sure jsons does not do something clever, when a json element needs to be parsed

fixcore/fixcore/web/api.py

@@ -1389,7 +1389,7 @@ async def write_files(mpr: MultipartReader, tmp_dir: str) -> Dict[str, str]:
                 if isinstance(part, MultipartReader):
                     files.update(await write_files(part, tmp_dir))
                 elif isinstance(part, BodyPartReader):
-                    name = part.filename
+                    name = part.name
                     if not name:
                         raise AttributeError("Multipart request: content disposition name is required!")
                     path = os.path.join(tmp_dir, rnd_str())  # use random local path to avoid clashes

fixcore/tests/fixcore/cli/command_test.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import re
 import sqlite3
 from datetime import timedelta
 from functools import partial
@@ -156,7 +157,7 @@ async def test_search_source(cli: CLIService) -> None:
     assert len(result3[0]) == 3
 
     result4 = await cli.execute_cli_command("search --explain --with-edges is(graph_root) -[0:1]->", list_sink)
-    assert result4[0][0]["rating"] in ["simple", "complex"]
+    assert result4[0][0]["rating"] in ["simple", "bad", "complex"]
 
     # use absolute path syntax
     result5 = await cli.execute_cli_command(
@@ -680,6 +681,10 @@ async def test_list_command(cli: CLI) -> None:
         "Region / Zone",
     ]
 
+    # define properties and add default properties
+    result = await cli.execute_cli_command("search is (foo) and id=0 | list --with-defaults kind as k, id", list_sink)
+    assert re.fullmatch("k=foo, id=0, age=.+, cloud=collector, account=sub_root", result[0][0])
+
 
 @pytest.mark.asyncio
 async def test_jq_command(cli: CLI) -> None:

plugins/azure/fix_plugin_azure/resource/authorization.py

@@ -100,7 +100,7 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource):
     condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the deny assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '})  # fmt: skip
     condition_version: Optional[str] = field(default=None, metadata={"description": "Version of the condition."})
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})
+    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
     deny_assignment_name: Optional[str] = field(default=None, metadata={'description': 'The display name of the deny assignment.'})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={'description': 'The description of the deny assignment.'})  # fmt: skip
     do_not_apply_to_child_scopes: Optional[bool] = field(default=None, metadata={'description': 'Determines if the deny assignment applies to child scopes. Default value is false.'})  # fmt: skip
@@ -109,8 +109,8 @@ class AzureAuthorizationDenyAssignment(MicrosoftResource):
     permissions: Optional[List[AzureDenyAssignmentPermission]] = field(default=None, metadata={'description': 'An array of permissions that are denied by the deny assignment.'})  # fmt: skip
     principals: Optional[List[AzurePrincipal]] = field(default=None, metadata={'description': 'Array of principals to which the deny assignment applies.'})  # fmt: skip
     scope: Optional[str] = field(default=None, metadata={"description": "The deny assignment scope."})
-    updated_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})
+    updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
 
 
 @define(eq=False, slots=False)
@@ -173,15 +173,15 @@ class AzureAuthorizationRoleAssignment(MicrosoftResource):
     condition: Optional[str] = field(default=None, metadata={'description': 'The conditions on the role assignment. This limits the resources it can be assigned to. e.g.: @Resource[Microsoft.Storage/storageAccounts/blobServices/containers:ContainerName] StringEqualsIgnoreCase foo_storage_container '})  # fmt: skip
     condition_version: Optional[str] = field(default=None, metadata={'description': 'Version of the condition. Currently the only accepted value is 2.0 '})  # fmt: skip
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})
+    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
     delegated_managed_identity_resource_id: Optional[str] = field(default=None, metadata={'description': 'Id of the delegated managed identity resource'})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={"description": "Description of role assignment"})
     principal_id: Optional[str] = field(default=None, metadata={"description": "The principal ID."})
     principal_type: Optional[str] = field(default=None, metadata={'description': 'The principal type of the assigned principal ID.'})  # fmt: skip
     role_definition_id: Optional[str] = field(default=None, metadata={"description": "The role definition ID."})
     scope: Optional[str] = field(default=None, metadata={"description": "The role assignment scope."})
-    updated_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})
+    updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
 
     def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None:
         # role definition
@@ -251,12 +251,12 @@ class AzureAuthorizationRoleDefinition(MicrosoftResource, BaseRole):
     }
     assignable_scopes: Optional[List[str]] = field(default=None, metadata={'description': 'Role definition assignable scopes.'})  # fmt: skip
     created_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who created the assignment'})  # fmt: skip
-    created_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was created"})
+    created_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was created"})  # fmt: skip
     description: Optional[str] = field(default=None, metadata={"description": "The role definition description."})
     azure_role_permissions: Optional[List[AzurePermission]] = field(default=None, metadata={'description': 'Role definition permissions.'})  # fmt: skip
     role_name: Optional[str] = field(default=None, metadata={"description": "The role name."})
-    updated_by: Optional[str] = field(default=None, metadata={'description': 'Id of the user who updated the assignment'})  # fmt: skip
-    updated_on: Optional[datetime] = field(default=None, metadata={"description": "Time it was updated"})
+    updated_by: Optional[str] = field(default=None, metadata={"ignore_history": True, 'description': 'Id of the user who updated the assignment'})  # fmt: skip
+    updated_on: Optional[datetime] = field(default=None, metadata={"ignore_history": True, "description": "Time it was updated"})  # fmt: skip
 
 
 @define(eq=False, slots=False)