From 628e830aa2af1fdb48ec94ab5d4f153ce3e10d41 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Mon, 4 Dec 2023 10:34:55 +0100 Subject: [PATCH] [resotocore][feat] Add timeseries command (#1840) --- resotocore/resotocore/cli/cli.py | 2 +- resotocore/resotocore/cli/command.py | 120 ++++++++- resotocore/resotocore/cli/model.py | 4 + resotocore/resotocore/core_config.py | 3 + resotocore/resotocore/db/arango_query.py | 238 +++++++++++------- resotocore/resotocore/db/async_arangodb.py | 24 +- resotocore/resotocore/db/db_access.py | 4 + resotocore/resotocore/db/graphdb.py | 6 + resotocore/resotocore/db/timeseriesdb.py | 95 +++++++ resotocore/resotocore/query/__init__.py | 10 +- resotocore/resotocore/query/query_parser.py | 5 +- .../resotocore/query/template_expander.py | 17 +- .../query/template_expander_service.py | 8 +- .../resotocore/report/inspector_service.py | 4 +- resotocore/resotocore/static/api-doc.yaml | 77 ++++++ resotocore/resotocore/web/api.py | 28 ++- .../tests/resotocore/cli/command_test.py | 31 +++ resotocore/tests/resotocore/conftest.py | 10 + .../tests/resotocore/db/arango_query_test.py | 52 +++- .../tests/resotocore/db/timeseriesdb_test.py | 50 ++++ .../resotocore/query/query_parser_test.py | 2 +- .../query/template_expander_test.py | 4 +- resotocore/tests/resotocore/web/api_test.py | 2 +- 23 files changed, 674 insertions(+), 122 deletions(-) create mode 100644 resotocore/resotocore/db/timeseriesdb.py create mode 100644 resotocore/tests/resotocore/db/timeseriesdb_test.py diff --git a/resotocore/resotocore/cli/cli.py b/resotocore/resotocore/cli/cli.py index c1b4c4616b..6654fe1ac2 100644 --- a/resotocore/resotocore/cli/cli.py +++ b/resotocore/resotocore/cli/cli.py @@ -394,7 +394,7 @@ async def parse_query(query_arg: str) -> Query: query_part = "all" if query_part.strip() == "" else query_part # section expansion is disabled here: it will happen on the final query after all parts have been combined return await self.dependencies.template_expander.parse_query( - "".join(query_part), None, omit_section_expansion=True, **ctx.env + "".join(query_part), None, omit_section_expansion=True, env=ctx.env ) query: Query = Query.by(AllTerm()) diff --git a/resotocore/resotocore/cli/command.py b/resotocore/resotocore/cli/command.py index 87790ed0a1..13f1bcd91c 100644 --- a/resotocore/resotocore/cli/command.py +++ b/resotocore/resotocore/cli/command.py @@ -51,7 +51,7 @@ from attr import evolve from attrs import define, field from dateutil import parser as date_parser -from parsy import Parser, string +from parsy import Parser, string, ParseError from resotoclient.models import Model as RCModel, Kind as RCKind from resotodatalink import EngineConfig from resotodatalink.batch_stream import BatchStream @@ -140,7 +140,7 @@ NavigateUntilLeaf, IsTerm, ) -from resotocore.query.query_parser import parse_query, aggregate_parameter_parser +from resotocore.query.query_parser import parse_query, aggregate_parameter_parser, predicate_term from resotocore.query.template_expander import tpl_props_p from resotocore.report import BenchmarkConfigPrefix, ReportSeverity from resotocore.report.benchmark_renderer import respond_benchmark_result @@ -170,6 +170,7 @@ ) from resotocore.worker_task_queue import WorkerTask, WorkerTaskName from resotolib.core import CLIEnvelope +from resotolib.durations import parse_duration from resotolib.parse_util import ( double_quoted_or_simple_string_dp, space_dp, @@ -1431,7 +1432,7 @@ def parse_at(x: Optional[str]) -> Optional[datetime]: at: Optional[datetime] = parse_at(parsed.get("at", None)) # all templates are expanded at this point, so we can call the parser directly. - query = parse_query(rest, **ctx.env) + query = parse_query(rest, ctx.env) async def get_db(at: Optional[datetime], graph_name: GraphName) -> Tuple[GraphDB, GraphName]: db_access = self.dependencies.db_access @@ -3904,7 +3905,7 @@ async def put_template(name: str, template_query: str) -> AsyncIterator[str]: # try to render_console the template with dummy values and see if the search can be parsed try: rendered_query = self.dependencies.template_expander.render(template_query, defaultdict(lambda: True)) - parse_query(rendered_query, **ctx.env) + parse_query(rendered_query, ctx.env) except Exception as ex: raise CLIParseError(f"Given template does not define a valid search: {template_query}") from ex await self.dependencies.template_expander.put_template(Template(name, template_query)) @@ -6036,6 +6037,115 @@ def key_fn(node: Json) -> Union[str, Tuple[str, str]]: raise AttributeError("Wrong or insufficient arguments. Execute `help db` to get more information.") +class TimeSeriesCommand(CLICommand): + """ + ``` + timeseries snapshot --name