From c6d86fa5bdd0cf5097ada8e410f33acd738a0941 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Thu, 16 Nov 2023 17:02:46 +0100 Subject: [PATCH] [resotocore][feat] Provide property complete endpoint (#1827) --- resotocore/Makefile | 1 - resotocore/resotocore/cli/command.py | 2 +- resotocore/resotocore/model/model.py | 70 +++++++++++++++- resotocore/resotocore/static/api-doc.yaml | 84 ++++++++++++++++++- resotocore/resotocore/web/api.py | 34 ++++++-- .../tests/resotocore/model/model_test.py | 36 ++++++++ 6 files changed, 213 insertions(+), 14 deletions(-) diff --git a/resotocore/Makefile b/resotocore/Makefile index a6d4985c49..7f843bca8c 100644 --- a/resotocore/Makefile +++ b/resotocore/Makefile @@ -59,7 +59,6 @@ lint: ## static code analysis black --line-length 120 --check resotocore tests flake8 resotocore pylint resotocore - # mypy --python-version 3.9 --strict --install-types --non-interactive resotocore tests mypy --python-version 3.9 --strict resotocore tests test: ## run tests quickly with the default Python diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index 636958a18f..20e53b0ef3 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -5907,7 +5907,7 @@ async def database_synchronize( resoto_model = await self.dependencies.model_handler.load_model(ctx.graph_name) if complete_model: - complex_kinds = resoto_model.complex_kinds() + complex_kinds = list(resoto_model.complex_kinds()) kinds = list(resoto_model.kinds.values()) else: # only export the kinds that are exported by this query diff --git a/resotocore/resotocore/model/model.py b/resotocore/resotocore/model/model.py index a62d524b43..59efaa70e1 100644 --- a/resotocore/resotocore/model/model.py +++ b/resotocore/resotocore/model/model.py @@ -23,6 +23,7 @@ Tuple, Iterable, TypeVar, + Iterator, ) import yaml @@ -1352,8 +1353,15 @@ def get(self, name_or_object: Union[str, Json]) -> Optional[Kind]: else: return None - def complex_kinds(self) -> List[ComplexKind]: - return [k for k in self.kinds.values() if isinstance(k, ComplexKind)] + def complex_kinds(self) -> Iterator[ComplexKind]: + for k in self.kinds.values(): + if isinstance(k, ComplexKind): + yield k + + def aggregate_roots(self) -> Iterator[ComplexKind]: + for k in self.complex_kinds(): + if k.aggregate_root: + yield k def property_by_path(self, path_: str) -> ResolvedProperty: path = PropertyPath.from_path(path_) @@ -1569,6 +1577,64 @@ def add_kind(cpl: ComplexKind) -> None: return Model(kinds, self.__property_kind_by_path) + def complete_path( + self, + path: str, + prop: str, + *, + filter_kinds: Optional[List[str]] = None, + fuzzy: bool = False, + skip: int = 0, + limit: int = 20, + ) -> Tuple[int, Dict[str, str]]: + filter_prop: Callable[[str], bool] = ( + (lambda p: re.match(prop, p) is not None) if fuzzy else (lambda p: p.startswith(prop)) + ) + + def from_prop_path(kinds: Iterable[ComplexKind]) -> Tuple[int, Dict[str, str]]: + props: Dict[str, str] = {} + for kind in kinds: + for p, pk in kind.all_props_with_kind(): + if filter_prop(p.name): + if isinstance(pk, ArrayKind): + props[p.name + "[*]"] = pk.inner.fqn + elif isinstance(pk, TransformKind): + props[p.name] = (pk.source_kind or any_kind).fqn + else: + props[p.name] = pk.fqn + return len(props), {k: props[k] for k in sorted(props)[skip : skip + limit]} + + def from_aggregate_roots() -> Tuple[int, Dict[str, str]]: + roots: Dict[str, str] = {} + for kind in self.aggregate_roots(): + if filter_prop(kind.fqn) and (not filter_kinds or kind.fqn in filter_kinds): + roots[kind.fqn] = "node" + return len(roots), {k: roots[k] for k in sorted(roots)[skip : skip + limit]} + + if path: + parts = path.split(".") + if parts[0] in ("ancestors", "descendants"): + if len(parts) == 1: + return from_aggregate_roots() + elif len(parts) == 2: + return 1, {"reported": "resource"} + elif len(parts) == 3 and parts[2] == "reported" and (kind := self.get(parts[1])): + return from_prop_path([kind]) if isinstance(kind, ComplexKind) else (0, {}) + elif len(parts) > 3 and parts[2] == "reported": + parts = parts[3:] + else: + return 0, {} + elif parts[0] in ("reported", "metadata", "desired"): + parts = parts[1:] + path = ".".join(parts) + + if path: + resolved = self.property_by_path(path) + return from_prop_path([resolved.kind]) if isinstance(resolved.kind, ComplexKind) else (0, {}) + else: + roots = [k for k in self.aggregate_roots() if not filter_kinds or k.fqn in filter_kinds] + return from_prop_path(roots) + @frozen class UsageDatapoint: diff --git a/resotocore/resotocore/static/api-doc.yaml b/resotocore/resotocore/static/api-doc.yaml index c3708016ac..fa8ad58a44 100644 --- a/resotocore/resotocore/static/api-doc.yaml +++ b/resotocore/resotocore/static/api-doc.yaml @@ -226,10 +226,10 @@ paths: in: query schema: type: string - default: null enum: - schema - simple + required: false - name: with_property_kinds description: "Render types of property values. Only together with kind or filter" in: query @@ -772,7 +772,7 @@ paths: - node_management parameters: - $ref: "#/components/parameters/graph_id" - - $ref: "#/components/parameters/section" + - $ref: "#/components/parameters/section-path" - name: node_id in: path description: "The identifier of the node" @@ -1706,6 +1706,75 @@ paths: ] items: type: string + /graph/{graph_id}/property/path/complete: + post: + summary: "Search the graph and return all possible attribute values for given property path." + tags: + - graph_search + parameters: + - $ref: "#/components/parameters/graph_id" + - $ref: "#/components/parameters/section" + requestBody: + content: + application/json: + schema: + description: "Parameters to define a property path completion." + type: object + properties: + prop: + type: string + description: | + The last part of the property name used to filter possible results. + All property names that start with this name are returned. + In fuzzy mode, this property is interpreted as regular expression. + example: "ab" + path: + type: string + description: Already existing path + kinds: + type: array + items: + type: string + description: The list of allowed kinds. + fuzzy: + type: boolean + description: | + If true, the prop parameter is interpreted as regular expression. + If false, the prop parameter is interpreted as prefix. + default: false + limit: + type: integer + description: | + The maximum number of results to return. + If not defined, the default limit is used. + default: 20 + skip: + type: integer + description: | + The number of results to skip. + If not defined, the default skip is used. + default: 0 + responses: + "200": + description: "The result of this search in the defined format" + headers: + Total-Count: + description: "The number of total available elements" + schema: + type: integer + example: 3 + content: + "application/json": + schema: + type: array + example: | + [ + "owner", + "checksum/secret", + "prometheus.io/path", + ] + items: + type: string @@ -3185,6 +3254,17 @@ components: required: true schema: type: string + section-path: + name: section + in: path + required: true + description: "The name of the section used for all property paths. If not defined root is assumed." + schema: + type: string + enum: + - reported + - desired + - metadata section: name: section in: query diff --git a/resotocore/resotocore/web/api.py b/resotocore/resotocore/web/api.py index 7ebe8cee8d..0e28c34060 100644 --- a/resotocore/resotocore/web/api.py +++ b/resotocore/resotocore/web/api.py @@ -90,7 +90,7 @@ from resotocore.model.exportable_model import json_export_simple_schema from resotocore.model.graph_access import Section from resotocore.model.json_schema import json_schema -from resotocore.model.model import Kind +from resotocore.model.model import Kind, Model from resotocore.model.typed_model import to_json, from_js, to_js_str, to_js from resotocore.query.model import Predicate, PathRoot, variable_to_absolute from resotocore.query.query_parser import predicate_term @@ -231,6 +231,7 @@ def __add_routes(self, prefix: str) -> None: web.post(prefix + "/graph/{graph_id}/search/history/aggregate", require(self.query_history, r)), web.post(prefix + "/graph/{graph_id}/property/attributes", require(self.possible_values, r)), web.post(prefix + "/graph/{graph_id}/property/values", require(self.possible_values, r)), + web.post(prefix + "/graph/{graph_id}/property/path/complete", require(self.property_path_complete, r)), # maintain the graph web.patch(prefix + "/graph/{graph_id}/nodes", require(self.update_nodes, a)), web.post(prefix + "/graph/{graph_id}/merge", require(self.merge_graph, a)), @@ -1012,11 +1013,7 @@ async def abort_batch(self, request: Request, deps: TenantDependencies) -> Strea await graph_db.abort_update(batch_id) return web.HTTPOk(body="Batch aborted.") - async def graph_query_model_from_request( - self, request: Request, deps: TenantDependencies - ) -> Tuple[GraphDB, QueryModel]: - section = section_of(request) - query_string = await request.text() + async def graph_model_from_request(self, request: Request, deps: TenantDependencies) -> Tuple[GraphName, Model]: graph_name = GraphName(request.match_info.get("graph_id", "resoto")) raw_at = request.query.get("at") at = date_parser.parse(raw_at) if raw_at else None @@ -1027,10 +1024,17 @@ async def graph_query_model_from_request( raise ValueError(f"No snapshot found for {graph_name} at {at}") graph_name = snapshot_name or graph_name + return graph_name, await deps.model_handler.load_model(graph_name) + + async def graph_query_model_from_request( + self, request: Request, deps: TenantDependencies + ) -> Tuple[GraphDB, QueryModel]: + section = section_of(request) + query_string = await request.text() + graph_name, model = await self.graph_model_from_request(request, deps) graph_db = deps.db_access.get_graph_db(graph_name) q = await deps.template_expander.parse_query(query_string, section, **request.query) - m = await deps.model_handler.load_model(graph_name) - return graph_db, QueryModel(q, m, cast(Dict[str, Any], request.query)) + return graph_db, QueryModel(q, model, cast(Dict[str, Any], request.query)) async def raw(self, request: Request, deps: TenantDependencies) -> StreamResponse: graph_db, query_model = await self.graph_query_model_from_request(request, deps) @@ -1043,6 +1047,20 @@ async def explain(self, request: Request, deps: TenantDependencies) -> StreamRes result = await graph_db.explain(query_model) return web.json_response(to_js(result)) + async def property_path_complete(self, request: Request, deps: TenantDependencies) -> StreamResponse: + _, model = await self.graph_model_from_request(request, deps) + body = await request.json() + path = variable_to_absolute(section_of(request), body.get("path", PathRoot)).rstrip(".\n\t") + prop = body.get("prop", "") + filter_kinds = body.get("kinds") + fuzzy = body.get("fuzzy", False) + limit = body.get("limit", 20) + skip = body.get("skip", 0) + assert skip >= 0, "Skip must be positive" + assert limit > 0, "Limit must be positive" + count, result = model.complete_path(path, prop, filter_kinds=filter_kinds, fuzzy=fuzzy, skip=skip, limit=limit) + return await single_result(request, result, {"Total-Count": str(count)}) + async def possible_values(self, request: Request, deps: TenantDependencies) -> StreamResponse: graph_db, query_model = await self.graph_query_model_from_request(request, deps) section = section_of(request) diff --git a/resotocore/tests/resotocore/model/model_test.py b/resotocore/tests/resotocore/model/model_test.py index 548cb7920d..68c70dfb60 100644 --- a/resotocore/tests/resotocore/model/model_test.py +++ b/resotocore/tests/resotocore/model/model_test.py @@ -582,6 +582,42 @@ def test_filter_model(person_model: Model) -> None: assert address.predecessor_kinds() == {"default": ["Person"], "delete": []} +def test_complete_path(person_model: Model) -> None: + count, all_props = person_model.complete_path("", "", fuzzy=False, skip=0, limit=4) + assert count == 21 # total + assert all_props == {"address": "Address", "addresses[*]": "Address", "age": "duration", "any": "any"} + # filter by prefix + count, all_props = person_model.complete_path("", "ad", fuzzy=False, skip=0, limit=4) + assert all_props == {"address": "Address", "addresses[*]": "Address"} + # filter by prefix fuzzy + count, all_props = person_model.complete_path("", "^a.*s", fuzzy=True, skip=0, limit=4) + assert all_props == {"address": "Address", "addresses[*]": "Address"} + # skip the first 4 + count, all_props = person_model.complete_path("", "", fuzzy=False, skip=4, limit=4) + assert all_props == {"city": "string", "ctime": "datetime", "expires": "datetime", "exported_age": "duration"} + # ask for a nested kind + count, all_props = person_model.complete_path("address", "", fuzzy=False, skip=4, limit=4) + assert all_props == {"mtime": "datetime", "tags": "dictionary[string, string]", "zip": "zip"} + # ask for a nested kind filtered + count, all_props = person_model.complete_path("address", "ta", fuzzy=False, skip=0, limit=4) + assert all_props == {"tags": "dictionary[string, string]"} + # reported section + count, all_props = person_model.complete_path("reported.address", "ta", fuzzy=False, skip=0, limit=4) + assert all_props == {"tags": "dictionary[string, string]"} + # ask for ancestors will list all available kinds + count, all_props = person_model.complete_path("ancestors", "", fuzzy=False, skip=0, limit=4) + assert all_props == {"Address": "node", "Base": "node", "Person": "node", "account": "node"} + # suggest the reported section + count, all_props = person_model.complete_path("ancestors.Address", "", fuzzy=False, skip=0, limit=4) + assert all_props == {"reported": "resource"} + # suggest properties of address + count, all_props = person_model.complete_path("ancestors.Address.reported", "", fuzzy=False, skip=0, limit=4) + assert all_props == {"city": "string", "id": "string", "kind": "string", "list[*]": "string"} + # filter by kinds + count, all_props = person_model.complete_path("", "", filter_kinds=["Address"], skip=5) + assert all_props == {"tags": "dictionary[string, string]", "zip": "zip"} + + @given(json_object_gen) @settings(max_examples=200, suppress_health_check=list(HealthCheck)) def test_yaml_generation(js: Json) -> None: