diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index ade3d49e11..70c785e89d 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -1446,7 +1446,7 @@ async def get_db(at: Optional[datetime], graph_name: GraphName) -> Tuple[GraphDB async def load_query_model(db: GraphDB, graph_name: GraphName) -> QueryModel: model = await self.dependencies.model_handler.load_model(graph_name) - query_model = QueryModel(query, model) + query_model = QueryModel(query, model, ctx.env) await db.to_query(query_model) # only here to validate the query itself (can throw) return query_model @@ -3127,7 +3127,7 @@ async def load_element(items: List[JsonElement]) -> AsyncIterator[JsonElement]: .merge_with("ancestors.region", NavigateUntilRoot, P.of_kind("region")) .merge_with("ancestors.zone", NavigateUntilRoot, P.of_kind("zone")) ).rewrite_for_ancestors_descendants(variables) - query_model = QueryModel(query, model) + query_model = QueryModel(query, model, env) async with await self.dependencies.db_access.get_graph_db(GraphName(env["graph"])).search_list( query_model ) as crs: @@ -5889,7 +5889,7 @@ async def sync_database_result(p: Namespace, maybe_stream: Optional[JsStream]) - query = Query(parts=[Part(term=IsTerm(["graph_root"]), navigation=NavigateUntilLeaf)]) graph_db = self.dependencies.db_access.get_graph_db(ctx.graph_name) async with await graph_db.search_graph_gen( - QueryModel(query, resoto_model), timeout=timedelta(weeks=200000) + QueryModel(query, resoto_model, ctx.env), timeout=timedelta(weeks=200000) ) as cursor: await sync_fn(query=query, in_stream=stream.iterate(cursor)) diff --git a/resotocore/resotocore/db/graphdb.py b/resotocore/resotocore/db/graphdb.py index 4a0b55eef3..f302bd8617 100644 --- a/resotocore/resotocore/db/graphdb.py +++ b/resotocore/resotocore/db/graphdb.py @@ -53,6 +53,7 @@ synthetic_metadata_kinds, ) from resotocore.model.resolve_in_graph import NodePath, GraphResolver +from resotocore.model.typed_model import to_js from resotocore.query.model import Query, FulltextTerm, MergeTerm, P, Predicate from resotocore.report import ReportSeverity from resotocore.types import JsonElement, EdgeType @@ -569,7 +570,7 @@ async def search_list( q_string, bind = await self.to_query(query) return await self.db.aql_cursor( query=q_string, - trafo=self.document_to_instance_fn(query.model, query.query), + trafo=self.document_to_instance_fn(query.model, query), count=with_count, bind_vars=bind, batch_size=10000, @@ -605,7 +606,10 @@ async def search_history( None if query.query.aggregate else self.document_to_instance_fn( - query.model, query.query, ["change", "changed_at", "created", "updated", "deleted"], id_column="id" + query.model, + query, + ["change", "changed_at", "created", "updated", "deleted"], + id_column="id", ) ) ttl = cast(Number, int(timeout.total_seconds())) if timeout else None @@ -620,7 +624,7 @@ async def search_graph_gen( query_string, bind = await self.to_query(query, with_edges=True) return await self.db.aql_cursor( query=query_string, - trafo=self.document_to_instance_fn(query.model, query.query), + trafo=self.document_to_instance_fn(query.model, query), bind_vars=bind, count=with_count, batch_size=10000, @@ -655,11 +659,12 @@ async def wipe(self) -> None: @staticmethod def document_to_instance_fn( model: Model, - query: Optional[Query] = None, + query: Optional[QueryModel] = None, additional_root_props: Optional[List[str]] = None, id_column: str = "_key", ) -> Callable[[Json], Json]: synthetic_metadata = model.predefined_synthetic_props(synthetic_metadata_kinds) + with_kinds = query and query.is_set("with-kind") def props(doc: Json, result: Json, definition: Iterable[str]) -> None: for prop in definition: @@ -682,11 +687,13 @@ def render_prop(doc: Json, root_level: bool) -> Json: if "_rev" in doc: result["revision"] = doc["_rev"] props(doc, result, Section.content_ordered) + kind = model.get(doc[Section.reported]) if root_level: props(doc, result, Section.lookup_sections_ordered) if additional_root_props: props(doc, result, additional_root_props) - kind = model.get(doc[Section.reported]) + if with_kinds and kind is not None: + result["kind"] = to_js(kind) if isinstance(kind, ComplexKind): synth_props(doc, result, Section.reported, kind.synthetic_props()) synth_props(doc, result, Section.metadata, synthetic_metadata) @@ -709,7 +716,7 @@ def render_merge_results(doc: Json, result: Json, q: Query) -> Json: def merge_results(doc: Json) -> Json: rendered = render_prop(doc, True) if query: - render_merge_results(doc, rendered, query) + render_merge_results(doc, rendered, query.query) return rendered return merge_results diff --git a/resotocore/resotocore/db/model.py b/resotocore/resotocore/db/model.py index 7da4e69ad4..9234a79e9a 100644 --- a/resotocore/resotocore/db/model.py +++ b/resotocore/resotocore/db/model.py @@ -1,33 +1,37 @@ from __future__ import annotations + from abc import ABC -from typing import Any +from typing import Dict, Any + +from attr import define from resotocore.model.model import Model from resotocore.query.model import Query -class QueryModel(ABC): - def __init__(self, query: Query, model: Model): - self.query = query - self.model = model +@define +class QueryModel: + query: Query + model: Model + env: Dict[str, Any] = {} + + def is_set(self, name: str) -> bool: + if value := self.env.get(name): + if isinstance(value, bool): + return value + elif isinstance(value, str): + return value.lower() in ["1", "true", "yes", "y"] + return False +@define(repr=True, eq=True) class GraphUpdate(ABC): - def __init__( - self, - nodes_created: int = 0, - nodes_updates: int = 0, - nodes_deleted: int = 0, - edges_created: int = 0, - edges_updated: int = 0, - edges_deleted: int = 0, - ): - self.nodes_created = nodes_created - self.nodes_updated = nodes_updates - self.nodes_deleted = nodes_deleted - self.edges_created = edges_created - self.edges_updated = edges_updated - self.edges_deleted = edges_deleted + nodes_created: int = 0 + nodes_updated: int = 0 + nodes_deleted: int = 0 + edges_created: int = 0 + edges_updated: int = 0 + edges_deleted: int = 0 def all_changes(self) -> int: return ( @@ -48,20 +52,3 @@ def __add__(self, other: GraphUpdate) -> GraphUpdate: self.edges_updated + other.edges_updated, self.edges_deleted + other.edges_deleted, ) - - def __repr__(self) -> str: - return ( - f"[[{self.nodes_created},{self.nodes_updated}," - f"{self.nodes_deleted}],[{self.edges_created}," - f"{self.edges_updated},{self.edges_deleted}]]" - ) - - def __str__(self) -> str: - return ( - f"GraphUpdate(nodes_created={self.nodes_created}, nodes_updated={self.nodes_updated}, " - f"nodes_deleted={self.nodes_deleted}, edges_created={self.edges_created}, " - f"edges_updated={self.edges_updated}, edges_deleted={self.edges_deleted})" - ) - - def __eq__(self, other: Any) -> bool: - return self.__dict__ == other.__dict__ if isinstance(other, GraphUpdate) else False diff --git a/resotocore/resotocore/static/api-doc.yaml b/resotocore/resotocore/static/api-doc.yaml index 9f9c89c875..c3708016ac 100644 --- a/resotocore/resotocore/static/api-doc.yaml +++ b/resotocore/resotocore/static/api-doc.yaml @@ -194,13 +194,7 @@ paths: tags: - model parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: flat description: "If true, the hierarchy of complex kinds is flattened, holding all properties and all merged metadata." in: query @@ -256,13 +250,7 @@ paths: tags: - model parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" requestBody: description: "Complete model or part of the model." content: @@ -288,13 +276,7 @@ paths: tags: - model parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: output description: The output format. in: query @@ -413,13 +395,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" responses: "200": description: "The graph with the root node" @@ -434,13 +410,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" responses: "200": description: "The created graph with the root node" @@ -453,13 +423,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: truncate in: query schema: @@ -478,13 +442,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: wait_for_result in: query description: > @@ -528,13 +486,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: batch_id in: query description: > @@ -588,13 +540,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" responses: "200": description: "Ok message" @@ -611,13 +557,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: batch_id in: path description: "A batch identifier is a string that uniquely identifies the batch update." @@ -640,13 +580,7 @@ paths: tags: - graph_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: batch_id in: path description: "A batch identifier is a string that uniquely identifies the batch update." @@ -673,13 +607,7 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" requestBody: description: "The partial object data to patch." content: @@ -738,13 +666,7 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: node_id in: path description: "The identifier of the node" @@ -779,13 +701,7 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: node_id in: path description: "The identifier of the node" @@ -807,13 +723,7 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: node_id in: path description: "The identifier of the node" @@ -842,13 +752,7 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" - name: node_id in: path description: "The identifier of the node" @@ -867,29 +771,14 @@ paths: tags: - node_management parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: node_id in: path description: "The identifier of the node" required: true schema: type: string - - name: section - in: path - description: "The identifier of the section" - required: true - schema: - type: string - enum: - - reported - - desired - - metadata requestBody: description: "The partial object data to patch." content: @@ -916,23 +805,8 @@ paths: tags: - debug parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: at in: query description: "The timestamp to use for the search. If not defined the latest version of the graph is used." @@ -964,23 +838,9 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - example: resoto - description: "The identifier of the graph" - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" + - $ref: "#/components/parameters/with-kind" - name: count in: query description: "Optional parameter to get a Ck-Element-Count header which returns the number of returned json elements" @@ -1174,23 +1034,9 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" + - $ref: "#/components/parameters/with-kind" - name: count in: query description: "Optional parameter to get a Ck-Element-Count header which returns the number of returned json elements" @@ -1390,23 +1236,9 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" + - $ref: "#/components/parameters/with-kind" - name: at in: query description: "The timestamp to use for the search. If not defined the latest version of the graph is used." @@ -1440,23 +1272,8 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: at in: query description: "The timestamp to use for the search. If not defined the latest version of the graph is used." @@ -1577,23 +1394,8 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: at in: query description: "The timestamp to use for the search. If not defined the latest version of the graph is used." @@ -1625,23 +1427,9 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - example: resoto - description: "The identifier of the graph" - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" + - $ref: "#/components/parameters/with-kind" - name: count in: query description: "Optional parameter to get a Ck-Element-Count header which returns the number of returned json elements" @@ -1777,23 +1565,8 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - description: "The identifier of the graph" - example: resoto - required: true - schema: - type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: before in: query description: "Optional parameter to get all history events before the given timestamp" @@ -1849,13 +1622,8 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - example: resoto - description: "The identifier of the graph" - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: prop in: query example: | @@ -1864,16 +1632,6 @@ paths: required: true schema: type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata - name: count in: query description: "Optional parameter to get a Ck-Element-Count header which returns the number of returned json elements" @@ -1909,13 +1667,8 @@ paths: tags: - graph_search parameters: - - name: graph_id - in: path - example: resoto - description: "The identifier of the graph" - required: true - schema: - type: string + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" - name: prop in: query example: | @@ -1924,16 +1677,6 @@ paths: required: true schema: type: string - - name: section - in: query - description: "The name of the section used for all property paths. If not defined root is assumed." - required: false - schema: - type: string - enum: - - reported - - desired - - metadata - name: count in: query description: "Optional parameter to get a Ck-Element-Count header which returns the number of returned json elements" @@ -2930,6 +2673,7 @@ paths: tags: - report parameters: + - $ref: "#/components/parameters/graph_id" - name: benchmark in: path description: "The name of the benchmark to perform" @@ -2937,13 +2681,6 @@ paths: type: string example: "aws_cis_1.5" required: true - - name: graph_id - in: path - description: "The id of the graph to perform this operation." - schema: - type: string - example: "resoto" - required: true - name: accounts description: | Comma separated list of account ids to include in the benchmark. @@ -2985,13 +2722,7 @@ paths: tags: - report parameters: - - name: graph_id - in: path - description: "The ID of the graph to check." - schema: - type: string - example: "resoto" - required: true + - $ref: "#/components/parameters/graph_id" - name: check_id in: path description: "The ID of the check to perform." @@ -3027,13 +2758,7 @@ paths: tags: - report parameters: - - name: graph_id - in: path - description: "The ID of the graph to check." - schema: - type: string - example: "resoto" - required: true + - $ref: "#/components/parameters/graph_id" - name: provider in: query description: "Filter by provider." @@ -3451,6 +3176,34 @@ paths: # endregion components: + parameters: + graph_id: + name: graph_id + in: path + example: resoto + description: "The identifier of the graph" + required: true + schema: + type: string + section: + name: section + in: query + description: "The name of the section used for all property paths. If not defined root is assumed." + required: false + schema: + type: string + enum: + - reported + - desired + - metadata + with-kind: + name: with-kind + description: "Include the kind of the node in the result node." + schema: + type: boolean + default: false + in: query + required: false schemas: AnalyticsEvent: description: "An analytics event." diff --git a/resotocore/resotocore/web/api.py b/resotocore/resotocore/web/api.py index a87f76fc02..3f7e67d314 100644 --- a/resotocore/resotocore/web/api.py +++ b/resotocore/resotocore/web/api.py @@ -27,6 +27,7 @@ Callable, Awaitable, Literal, + cast, ) from urllib.parse import urlencode, urlparse, parse_qs, urlunparse @@ -1017,7 +1018,7 @@ async def graph_query_model_from_request( graph_db = deps.db_access.get_graph_db(graph_name) q = await deps.template_expander.parse_query(query_string, section, **request.query) m = await deps.model_handler.load_model(graph_name) - return graph_db, QueryModel(q, m) + return graph_db, QueryModel(q, m, cast(Dict[str, Any], request.query)) async def raw(self, request: Request, deps: TenantDependencies) -> StreamResponse: graph_db, query_model = await self.graph_query_model_from_request(request, deps) diff --git a/resotocore/tests/resotocore/db/graphdb_test.py b/resotocore/tests/resotocore/db/graphdb_test.py index e3a7cde075..ca51692cbd 100644 --- a/resotocore/tests/resotocore/db/graphdb_test.py +++ b/resotocore/tests/resotocore/db/graphdb_test.py @@ -707,6 +707,17 @@ def test_render_metadata_section(foo_model: Model) -> None: assert "exported_age" in out["metadata"] # exported_age is not part of the document, but should be added +def test_with_kind_section(foo_model: Model) -> None: + qm = QueryModel(Query.by("foo"), foo_model, {"with-kind": "true"}) + p1 = ArangoGraphDB.document_to_instance_fn(foo_model) + p2 = ArangoGraphDB.document_to_instance_fn(foo_model, qm) + # no kind is rendered + assert p1({"_key": "1", "reported": {"kind": "foo"}}) == {"id": "1", "type": "node", "reported": {"kind": "foo"}} + # kind is rendered + wk = p2({"_key": "1", "reported": {"kind": "foo"}}) + assert wk["kind"]["fqn"] == "foo" + + @mark.asyncio async def test_update_security_section(filled_graph_db: GraphDB, foo_model: Model) -> None: async def query_vulnerable() -> List[Json]: diff --git a/resotocore/tests/resotocore/db/model_test.py b/resotocore/tests/resotocore/db/model_test.py new file mode 100644 index 0000000000..b4fbd3687b --- /dev/null +++ b/resotocore/tests/resotocore/db/model_test.py @@ -0,0 +1,18 @@ +from resotocore.db.model import QueryModel, GraphUpdate +from resotocore.model.model import Model +from resotocore.query.model import Query + + +def test_query_model() -> None: + model = QueryModel(Query.by("test"), Model.from_kinds([]), {"a": "1", "b": "True", "c": "false", "d": "true"}) + assert model.is_set("a") + assert model.is_set("b") + assert not model.is_set("c") + assert model.is_set("d") + + +def test_graph_update() -> None: + gu1 = GraphUpdate(1, 2, 3, 4, 5, 6) + gu2 = GraphUpdate(6, 5, 4, 3, 2, 1) + assert gu1.all_changes() == gu2.all_changes() == 21 + assert gu1 + gu2 == GraphUpdate(7, 7, 7, 7, 7, 7)