diff --git a/fixcore/.pylintrc b/fixcore/.pylintrc index 91536aa44b..94fd5c5778 100644 --- a/fixcore/.pylintrc +++ b/fixcore/.pylintrc @@ -246,7 +246,7 @@ ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local, aiostream.pipe +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular diff --git a/fixcore/fixcore/cli/__init__.py b/fixcore/fixcore/cli/__init__.py index cb93c9334d..3af2a601e3 100644 --- a/fixcore/fixcore/cli/__init__.py +++ b/fixcore/fixcore/cli/__init__.py @@ -15,13 +15,12 @@ AsyncIterable, ) -from aiostream import stream -from aiostream.core import Stream from parsy import Parser, regex, string from fixcore.model.graph_access import Section from fixcore.types import JsonElement, Json from fixcore.util import utc, parse_utc, AnyT +from fixlib.asynchronous.stream import Stream from fixlib.durations import parse_duration, DurationRe from fixlib.parse_util import ( make_parser, @@ -47,7 +46,7 @@ # A sink function takes a stream and creates a result Sink = Callable[[JsStream], Awaitable[T]] -list_sink: Callable[[JsGen], Awaitable[Any]] = stream.list # type: ignore +list_sink: Callable[[JsGen], Awaitable[List[Any]]] = Stream.as_list @make_parser diff --git a/fixcore/fixcore/cli/cli.py b/fixcore/fixcore/cli/cli.py index 8c535bef82..178bb7233c 100644 --- a/fixcore/fixcore/cli/cli.py +++ b/fixcore/fixcore/cli/cli.py @@ -10,14 +10,13 @@ from typing import Dict, List, Tuple, Union, Sequence from typing import Optional, Any, TYPE_CHECKING -from aiostream import stream from attrs import evolve from parsy import Parser from rich.padding import Padding from fixcore import version from fixcore.analytics import CoreEvent -from fixcore.cli import cmd_with_args_parser, key_values_parser, T, Sink, args_values_parser, JsGen +from fixcore.cli import cmd_with_args_parser, key_values_parser, T, Sink, args_values_parser, JsStream from fixcore.cli.command import ( SearchPart, PredecessorsPart, @@ -78,6 +77,7 @@ from fixcore.types import JsonElement from fixcore.user.model import Permission from fixcore.util import group_by +from fixlib.asynchronous.stream import Stream from fixlib.parse_util import make_parser, pipe_p, semicolon_p if TYPE_CHECKING: @@ -104,7 +104,7 @@ def command_line_parser() -> Parser: return ParsedCommands(commands, maybe_env if maybe_env else {}) -# multiple piped commands are separated by semicolon +# semicolon separates multiple piped commands multi_command_parser = command_line_parser.sep_by(semicolon_p) @@ -187,7 +187,7 @@ def overview() -> str: logo = ctx.render_console(Padding(WelcomeCommand.ck, pad=(0, 0, 0, middle))) if ctx.supports_color() else "" return headline + logo + ctx.render_console(result) - def help_command() -> JsGen: + def help_command() -> JsStream: if not arg: result = overview() elif arg == "placeholders": @@ -209,7 +209,7 @@ def help_command() -> JsGen: else: result = f"No command found with this name: {arg}" - return stream.just(result) + return Stream.just(result) return CLISource.single(help_command, required_permissions={Permission.read}) @@ -352,11 +352,11 @@ def command( self, name: str, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any ) -> ExecutableCommand: """ - Create an executable command for given command name, args and context. - :param name: the name of the command to execute (must be a known command) - :param arg: the arg of the command (must be parsable by the command) - :param ctx: the context of this command. - :return: the ready to run executable command. + Create an executable command for given command name, args, and context. + :param name: The name of the command to execute (must be a known command). + :param arg: The arg of the command (must be parsable by the command). + :param ctx: The context of this command. + :return: The ready to run executable command. :raises: CLIParseError: if the name of the command is not known, or the argument fails to parse. """ @@ -377,9 +377,9 @@ async def create_query( Takes a list of query part commands and combine them to a single executable query command. This process can also introduce new commands that should run after the query is finished. Therefore, a list of executable commands is returned. - :param commands: the incoming executable commands, which actions are all instances of SearchCLIPart. - :param ctx: the context to execute within. - :return: the resulting list of commands to execute. + :param commands: The incoming executable commands, which actions are all instances of SearchCLIPart. + :param ctx: The context to execute within. + :return: The resulting list of commands to execute. """ # Pass parsed options to execute query @@ -484,8 +484,8 @@ async def parse_query(query_arg: str) -> Query: first_head_tail_in_a_row = None head_tail_keep_order = True - # Define default sort order, if not already defined - # A sort order is required to always return the result in a deterministic way to the user. + # Define default sort order, if not already defined. + # A sort order is required to always return the result deterministically to the user. # Deterministic order is required for head/tail to work if query.is_simple_fulltext_search(): # Do not define any additional sort order for fulltext searches @@ -494,7 +494,7 @@ async def parse_query(query_arg: str) -> Query: parts = [pt if pt.sort else evolve(pt, sort=default_sort) for pt in query.parts] query = evolve(query, parts=parts) - # If the last part is a navigation, we need to add sort which will ingest a new part. + # If the last part is a navigation, we need to add a sort which will ingest a new part. with_sort = query.set_sort(*default_sort) if query.current_part.navigation else query section = ctx.env.get("section", PathRoot) # If this is an aggregate query, the default sort needs to be changed @@ -534,7 +534,7 @@ def rewrite_command_line(cmds: List[ExecutableCommand], ctx: CLIContext) -> List Rules: - add the list command if no output format is defined - add a format to write commands if no output format is defined - - report benchmark run will be formatted as benchmark result automatically + - report benchmark run will be formatted as a benchmark result automatically """ if ctx.env.get("no_rewrite") or len(cmds) == 0: return cmds diff --git a/fixcore/fixcore/cli/command.py b/fixcore/fixcore/cli/command.py index b582cba90d..2645658266 100644 --- a/fixcore/fixcore/cli/command.py +++ b/fixcore/fixcore/cli/command.py @@ -29,7 +29,6 @@ Optional, Any, AsyncIterator, - Iterable, Callable, Awaitable, cast, @@ -46,9 +45,6 @@ import yaml from aiofiles.tempfile import TemporaryDirectory from aiohttp import ClientTimeout, JsonPayload, BasicAuth -from aiostream import stream, pipe -from aiostream.aiter_utils import is_async_iterable -from aiostream.core import Stream from attr import evolve, frozen from attrs import define, field from dateutil import parser as date_parser @@ -178,6 +174,7 @@ respond_cytoscape, ) from fixcore.worker_task_queue import WorkerTask, WorkerTaskName +from fixlib.asynchronous.stream import Stream from fixlib.core import CLIEnvelope from fixlib.durations import parse_duration from fixlib.parse_util import ( @@ -946,14 +943,13 @@ def group(keys: tuple[Any, ...]) -> Json: return result async def aggregate_data(content: JsStream) -> AsyncIterator[JsonElement]: - async with content.stream() as in_stream: - for key, value in (await self.aggregate_in(in_stream, var_names, aggregate.group_func)).items(): - entry: Json = {"group": group(key)} - for fn_name, (fn_val, fn_count) in value.fn_values.items(): - if fn_by_name.get(fn_name) == "avg" and fn_val is not None and fn_count > 0: - fn_val = fn_val / fn_count # type: ignore - entry[fn_name] = fn_val - yield entry + for key, value in (await self.aggregate_in(content, var_names, aggregate.group_func)).items(): + entry: Json = {"group": group(key)} + for fn_name, (fn_val, fn_count) in value.fn_values.items(): + if fn_by_name.get(fn_name) == "avg" and fn_val is not None and fn_count > 0: + fn_val = fn_val / fn_count # type: ignore + entry[fn_name] = fn_val + yield entry # noinspection PyTypeChecker return CLIFlow(aggregate_data) @@ -1000,7 +996,7 @@ def info(self) -> str: def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIAction: size = self.parse_size(arg) - return CLIFlow(lambda in_stream: in_stream | pipe.take(size)) + return CLIFlow(lambda in_stream: Stream(in_stream).take(size)) def args_info(self) -> ArgsInfo: return [ArgInfo(expects_value=True, help_text="number of elements to take")] @@ -1054,7 +1050,7 @@ def args_info(self) -> ArgsInfo: def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIAction: size = HeadCommand.parse_size(arg) - return CLIFlow(lambda in_stream: in_stream | pipe.takelast(size)) + return CLIFlow(lambda in_stream: Stream(in_stream).take_last(size)) class CountCommand(SearchCLIPart): @@ -1145,9 +1141,8 @@ def inc_identity(_: Any) -> None: fn = inc_prop if arg else inc_identity async def count_in_stream(content: JsStream) -> AsyncIterator[JsonElement]: - async with content.stream() as in_stream: - async for element in in_stream: - fn(element) + async for element in content: + fn(element) for key, value in sorted(counter.items(), key=lambda x: x[1]): yield f"{key}: {value}" @@ -1194,7 +1189,7 @@ def args_info(self) -> ArgsInfo: def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLISource: return CLISource.single( - lambda: stream.just(strip_quotes(arg if arg else "")), required_permissions={Permission.read} + lambda: Stream.just(strip_quotes(arg if arg else "")), required_permissions={Permission.read} ) @@ -1256,7 +1251,7 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa else: raise AttributeError(f"json does not understand {arg}.") return CLISource.with_count( - lambda: stream.iterate(elements), len(elements), required_permissions={Permission.read} + lambda: Stream.iterate(elements), len(elements), required_permissions={Permission.read} ) @@ -1339,19 +1334,17 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa async def to_count(in_stream: JsStream) -> AsyncIterator[JsonElement]: null_value = 0 total = 0 - in_streamer = in_stream if isinstance(in_stream, Stream) else stream.iterate(in_stream) - async with in_streamer.stream() as streamer: - async for elem in streamer: - name = js_value_at(elem, name_path) - count = js_value_get(elem, count_path, 0) - if name is None: - null_value = count - else: - total += count - yield f"{name}: {count}" - tm, tu = (total, null_value) if arg else (null_value + total, 0) - yield f"total matched: {tm}" - yield f"total unmatched: {tu}" + async for elem in in_stream: + name = js_value_at(elem, name_path) + count = js_value_get(elem, count_path, 0) + if name is None: + null_value = count + else: + total += count + yield f"{name}: {count}" + tm, tu = (total, null_value) if arg else (null_value + total, 0) + yield f"total matched: {tm}" + yield f"total unmatched: {tu}" return CLIFlow(to_count) @@ -1550,7 +1543,7 @@ def args_info(self) -> ArgsInfo: return [] def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLISource: - return CLISource.with_count(lambda: stream.just(ctx.env), len(ctx.env), required_permissions={Permission.read}) + return CLISource.with_count(lambda: Stream.just(ctx.env), len(ctx.env), required_permissions={Permission.read}) class ChunkCommand(CLICommand): @@ -1599,7 +1592,10 @@ def args_info(self) -> ArgsInfo: def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIFlow: size = int(arg) if arg else 100 - return CLIFlow(lambda in_stream: in_stream | pipe.chunks(size), required_permissions={Permission.read}) + return CLIFlow( + lambda in_stream: Stream(in_stream).chunks(size), + required_permissions={Permission.read}, + ) class FlattenCommand(CLICommand): @@ -1646,13 +1642,7 @@ def args_info(self) -> ArgsInfo: return [] def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIFlow: - def iterable(it: Any) -> bool: - return False if isinstance(it, str) else isinstance(it, Iterable) - - def iterate(it: Any) -> JsGen: - return stream.iterate(it) if is_async_iterable(it) or iterable(it) else stream.just(it) - - return CLIFlow(lambda i: i | pipe.flatmap(iterate), required_permissions={Permission.read}) # type: ignore + return CLIFlow(lambda i: Stream(i).flatten(), required_permissions={Permission.read}) class UniqCommand(CLICommand): @@ -1709,7 +1699,7 @@ def has_not_seen(item: Any) -> bool: visited.add(item) return True - return CLIFlow(lambda in_stream: stream.filter(in_stream, has_not_seen), required_permissions={Permission.read}) + return CLIFlow(lambda in_stream: Stream(in_stream).filter(has_not_seen), required_permissions={Permission.read}) class JqCommand(CLICommand, OutputTransformer): @@ -1809,7 +1799,7 @@ def process(in_json: JsonElement) -> JsonElement: result = out[0] if len(out) == 1 else out return cast(Json, result) - return CLIFlow(lambda i: i | pipe.map(process), required_permissions={Permission.read}) # type: ignore + return CLIFlow(lambda i: Stream(i).map(process), required_permissions={Permission.read}) class KindsCommand(CLICommand, PreserveOutputFormat): @@ -1962,16 +1952,16 @@ def show(k: ComplexKind) -> bool: result: JsonElement = ( kind_to_js(model, model[kind]) if kind in model else f"No kind with this name: {kind}" ) - return 1, stream.just(result) + return 1, Stream.just(result) elif args.property_path: no_section = Section.without_section(args.property_path) result = kind_to_js(model, model.kind_by_path(no_section)) if appears_in := property_defined_in(model, no_section): result["appears_in"] = appears_in - return 1, stream.just(result) + return 1, Stream.just(result) else: result = sorted([k.fqn for k in model.kinds.values() if isinstance(k, ComplexKind) and show(k)]) - return len(model.kinds), stream.iterate(result) + return len(model.kinds), Stream.iterate(result) return CLISource.only_count(source, required_permissions={Permission.read}) @@ -1986,7 +1976,8 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa buffer_size = 1000 func = partial(self.set_desired, arg, ctx.graph_name, self.patch(arg, ctx)) return CLIFlow( - lambda i: i | pipe.chunks(buffer_size) | pipe.flatmap(func), required_permissions={Permission.write} + lambda i: Stream(i).chunks(buffer_size).flatmap(func, task_limit=10, ordered=False), + required_permissions={Permission.write}, ) async def set_desired( @@ -2124,7 +2115,8 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa buffer_size = 1000 func = partial(self.set_metadata, ctx.graph_name, self.patch(arg, ctx)) return CLIFlow( - lambda i: i | pipe.chunks(buffer_size) | pipe.flatmap(func), required_permissions={Permission.write} + lambda i: Stream(i).chunks(buffer_size).flatmap(func, task_limit=10, ordered=False), + required_permissions={Permission.write}, ) async def set_metadata(self, graph_name: GraphName, patch: Json, items: List[Json]) -> AsyncIterator[JsonElement]: @@ -2331,9 +2323,8 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa use = next(iter(format_to_use)) async def render_single(converter: ConvertFn, iss: JsStream) -> JsGen: - async with iss.stream() as streamer: - async for elem in converter(streamer): - yield elem + async for elem in converter(iss): + yield elem async def format_stream(in_stream: JsStream) -> JsGen: if use: @@ -2344,7 +2335,7 @@ async def format_stream(in_stream: JsStream) -> JsGen: else: raise ValueError(f"Unknown format: {use}") elif formatting_string: - return in_stream | pipe.map(ctx.formatter(arg)) if arg else in_stream # type: ignore + return in_stream.map(ctx.formatter(arg)) if arg else in_stream # type: ignore else: return in_stream @@ -2817,14 +2808,13 @@ def to_csv_string(lst: List[Any]) -> str: header_values = [prop.name for prop in props] yield to_csv_string(header_values) - async with in_stream.stream() as s: - async for elem in s: - if is_node(elem) or is_aggregate: - result = [] - for prop in props: - value = prop.value(elem) - result.append(value) - yield to_csv_string(result) + async for elem in in_stream: + if is_node(elem) or is_aggregate: + result = [] + for prop in props: + value = prop.value(elem) + result.append(value) + yield to_csv_string(result) async def json_table_stream(in_stream: JsStream, model: QueryModel) -> JsGen: def kind_of(path: str) -> Kind: @@ -2857,13 +2847,12 @@ def render_prop(elem: JsonElement) -> JsonElement: ], } # data columns - async with in_stream.stream() as s: - async for elem in s: - if isinstance(elem, dict) and (is_node(elem) or is_aggregate): - yield { - "id": None if is_aggregate else elem["id"], # aggregates have no id - "row": {prop.name: render_prop(prop.value(elem)) for prop in props}, - } + async for elem in in_stream: + if isinstance(elem, dict) and (is_node(elem) or is_aggregate): + yield { + "id": None if is_aggregate else elem["id"], # aggregates have no id + "row": {prop.name: render_prop(prop.value(elem)) for prop in props}, + } def markdown_stream(in_stream: JsStream) -> JsGen: chunk_size = 500 @@ -2922,12 +2911,11 @@ def to_str(elem: Any) -> str: # noinspection PyUnresolvedReferences markdown_chunks = ( - in_stream - | pipe.filter(lambda x: is_node(x) or is_aggregate) - | pipe.map(extract_values) # type: ignore - | pipe.chunks(chunk_size) - | pipe.enumerate() - | pipe.flatmap(generate_markdown) # type: ignore + in_stream.filter(lambda x: is_node(x) or is_aggregate) + .map(extract_values) + .chunks(chunk_size) + .enumerate() + .flatmap(generate_markdown) ) return markdown_chunks @@ -2943,12 +2931,9 @@ async def load_model() -> QueryModel: model = await self.dependencies.model_handler.load_model(ctx.graph_name) return QueryModel(ctx.query or Query.empty(), model, ctx.env) - return stream.call(load_model) | pipe.flatmap(partial(json_table_stream, in_stream)) # type: ignore + return Stream.call(load_model).flatmap(partial(json_table_stream, in_stream)) # type: ignore else: - return stream.map( - in_stream, - lambda elem: fmt_json(elem) if isinstance(elem, dict) else str(elem), # type: ignore - ) + return Stream(in_stream).map(lambda elem: fmt_json(elem) if isinstance(elem, dict) else str(elem)) return CLIFlow(fmt, produces=MediaType.String, required_permissions={Permission.read}) @@ -3208,7 +3193,7 @@ async def activate_deactivate_job(job_id: str, active: bool) -> AsyncIterator[Js async def running_jobs() -> Tuple[int, JsStream]: tasks = await self.dependencies.task_handler.running_tasks() - return len(tasks), stream.iterate( + return len(tasks), Stream.iterate( {"job": t.descriptor.id, "started_at": to_json(t.task_started_at), "task-id": t.id} for t in tasks if isinstance(t.descriptor, Job) @@ -3271,7 +3256,7 @@ async def send_to_queue(task_name: str, task_args: Dict[str, str], data: Json) - await self.dependencies.forked_tasks.put((result_task, f"WorkerTask {task_name}:{task.id}")) return f"Spawned WorkerTask {task_name}:{task.id}" - return in_stream | pipe.starmap(send_to_queue, ordered=False, task_limit=self.task_limit()) # type: ignore + return in_stream.starmap(send_to_queue, ordered=False, task_limit=self.task_limit()) def load_by_id_merged( self, @@ -3307,7 +3292,7 @@ async def load_element(items: List[JsonElement]) -> AsyncIterator[JsonElement]: async for a in crs: yield a - return stream.chunks(in_stream, 1000) | pipe.flatmap(load_element) # type: ignore + return in_stream.chunks(1000).flatmap(load_element, task_limit=10) async def no_update(self, _: WorkerTask, future_result: Future[Json]) -> Json: return await future_result @@ -3448,17 +3433,17 @@ def setup_stream(in_stream: JsStream) -> JsStream: def with_dependencies(model: Model) -> JsStream: load = self.load_by_id_merged(model, in_stream, variables, allowed_on_kind, **ctx.env) handler = self.update_node_in_graphdb(model, **ctx.env) if expect_node_result else self.no_update - return self.send_to_queue_stream(load | pipe.map(fn), handler, True) # type: ignore + return self.send_to_queue_stream(load.map(fn), handler, True) # type: ignore # dependencies are not resolved directly (no async function is allowed here) async def load_model() -> Model: return await self.dependencies.model_handler.load_model(ctx.graph_name) - return stream.call(load_model) | pipe.flatmap(with_dependencies) # type: ignore + return Stream.call(load_model).flatmap(with_dependencies) # type: ignore def setup_source() -> JsStream: arg = {"args": args_parts_unquoted_parser.parse(formatter({}))} - return self.send_to_queue_stream(stream.just((command_name, {}, arg)), self.no_update, True) + return self.send_to_queue_stream(Stream.just((command_name, {}, arg)), self.no_update, True) return ( CLISource.single(setup_source, required_permissions={Permission.write}) @@ -3575,13 +3560,13 @@ def setup_stream(in_stream: JsStream) -> JsStream: def with_dependencies(model: Model) -> JsStream: load = self.load_by_id_merged(model, in_stream, variables, **ctx.env) result_handler = self.update_node_in_graphdb(model, **ctx.env) - return self.send_to_queue_stream(load | pipe.map(fn), result_handler, not ns.nowait) # type: ignore + return self.send_to_queue_stream(load.map(fn), result_handler, not ns.nowait) # type: ignore async def load_model() -> Model: return await self.dependencies.model_handler.load_model(ctx.graph_name) # dependencies are not resolved directly (no async function is allowed here) - return stream.call(load_model) | pipe.flatmap(with_dependencies) # type: ignore + return Stream.call(load_model).flatmap(with_dependencies) # type: ignore return CLIFlow(setup_stream, required_permissions={Permission.write}) @@ -3594,7 +3579,7 @@ def file_command() -> JsStream: elif not os.path.exists(arg): raise AttributeError(f"file does not exist: {arg}!") else: - return stream.just(arg if arg else "") + return Stream.just(arg if arg else "") return CLISource.single(file_command, MediaType.FilePath, required_permissions={Permission.admin}) @@ -3618,7 +3603,7 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa def upload_command() -> JsStream: if file_id in ctx.uploaded_files: file = ctx.uploaded_files[file_id] - return stream.just(f"Received file {file} of size {os.path.getsize(file)}") + return Stream.just(f"Received file {file} of size {os.path.getsize(file)}") else: raise AttributeError(f"file was not uploaded: {arg}!") @@ -3932,19 +3917,17 @@ async def write_result_to_file(ctx: CLIContext, in_stream: JsStream, file_name: async with TemporaryDirectory() as temp_dir: path = file_name if ctx.intern else os.path.join(temp_dir, uuid_str()) async with aiofiles.open(path, "w") as f: - async with in_stream.stream() as streamer: - async for out in streamer: - if isinstance(out, str): - await f.write(out + "\n") - else: - raise AttributeError("No output format is defined! Consider to use the format command.") + async for out in in_stream: + if isinstance(out, str): + await f.write(out + "\n") + else: + raise AttributeError("No output format is defined! Consider to use the format command.") yield FilePath.user_local(user=file_name, local=path).json() @staticmethod async def already_file_stream(in_stream: JsStream, file_name: str) -> AsyncIterator[JsonElement]: - async with in_stream.stream() as streamer: - async for out in streamer: - yield evolve(FilePath.from_path(out), user=Path(file_name)).json() + async for out in in_stream: + yield evolve(FilePath.from_path(out), user=Path(file_name)).json() def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwargs: Any) -> CLIAction: if arg is None: @@ -4040,7 +4023,7 @@ async def get_template(name: str) -> AsyncIterator[JsonElement]: async def list_templates() -> Tuple[int, Stream[str]]: templates = await self.dependencies.template_expander.list_templates() - return len(templates), stream.iterate(template_str(t) for t in templates) + return len(templates), Stream.iterate(template_str(t) for t in templates) async def put_template(name: str, template_query: str) -> AsyncIterator[str]: # try to render_console the template with dummy values and see if the search can be parsed @@ -4283,10 +4266,9 @@ async def perform_request(e: JsonElement) -> int: async def iterate_stream(in_stream: JsStream) -> AsyncIterator[JsonElement]: results: Dict[int, int] = defaultdict(lambda: 0) - async with in_stream.stream() as streamer: - async for elem in streamer: - status_code = await perform_request(elem) - results[status_code] += 1 + async for elem in in_stream: + status_code = await perform_request(elem) + results[status_code] += 1 summary = ", ".join(f"{count} requests with status {status}" for status, count in results.items()) if results: yield f"{summary} sent." @@ -4514,18 +4496,18 @@ def info(rt: RunningTask) -> JsonElement: **progress, } - return len(tasks), stream.iterate(info(t) for t in tasks if isinstance(t.descriptor, Workflow)) + return len(tasks), Stream.iterate(info(t) for t in tasks if isinstance(t.descriptor, Workflow)) async def show_log(wf_id: str) -> Tuple[int, JsStream]: rtd = await self.dependencies.db_access.running_task_db.get(wf_id) if rtd: messages = [msg.info() for msg in rtd.info_messages()] if messages: - return len(messages), stream.iterate(messages) + return len(messages), Stream.iterate(messages) else: - return 0, stream.just("No error messages for this run.") + return 0, Stream.just("No error messages for this run.") else: - return 0, stream.just(f"No workflow task with this id: {wf_id}") + return 0, Stream.just(f"No workflow task with this id: {wf_id}") def running_task_data(rtd: RunningTaskData) -> Json: result = { @@ -4539,7 +4521,7 @@ def running_task_data(rtd: RunningTaskData) -> Json: async def history_aggregation() -> JsStream: info = await self.dependencies.db_access.running_task_db.aggregated_history() - return stream.just(info) + return Stream.just(info) async def history_of(history_args: List[str]) -> Tuple[int, JsStream]: parser = NoExitArgumentParser() @@ -4558,7 +4540,7 @@ async def history_of(history_args: List[str]) -> Tuple[int, JsStream]: ) cursor: AsyncCursor = context.cursor try: - return cursor.count() or 0, stream.map(cursor, running_task_data) # type: ignore + return cursor.count() or 0, Stream(cursor).map(running_task_data) finally: cursor.close() @@ -4591,7 +4573,7 @@ async def stop_workflow(task_id: TaskId) -> AsyncIterator[str]: return CLISource.only_count(list_workflows, required_permissions={Permission.read}) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -4763,7 +4745,7 @@ async def update_config(cfg_id: ConfigId) -> AsyncIterator[str]: async def list_configs() -> Tuple[int, JsStream]: ids = [i async for i in self.dependencies.config_handler.list_config_ids()] - return len(ids), stream.iterate(ids) + return len(ids), Stream.iterate(ids) args = re.split("\\s+", arg, maxsplit=2) if arg else [] if arg and len(args) == 2 and (args[0] == "show" or args[0] == "get"): @@ -4800,7 +4782,7 @@ async def list_configs() -> Tuple[int, JsStream]: return CLISource.only_count(list_configs, required_permissions={Permission.read}) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -4889,7 +4871,7 @@ async def welcome() -> str: res = ctx.render_console(grid) return res - return CLISource.single(lambda: stream.just(welcome()), required_permissions={Permission.read}) # type: ignore + return CLISource.single(lambda: Stream.just(welcome()), required_permissions={Permission.read}) class TipOfTheDayCommand(CLICommand): @@ -4926,7 +4908,7 @@ async def totd() -> str: res = ctx.render_console(info) return res - return CLISource.single(lambda: stream.just(totd()), required_permissions={Permission.read}) # type: ignore + return CLISource.single(lambda: Stream.just(totd()), required_permissions={Permission.read}) class CertificateCommand(CLICommand): @@ -5004,7 +4986,7 @@ async def create_certificate( ) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -5435,9 +5417,8 @@ async def app_run( raise ValueError(f"Config {config} not found.") async def stream_to_iterator() -> AsyncIterator[JsonElement]: - async with in_stream.stream() as streamer: - async for item in streamer: - yield item + async for item in in_stream: + yield item stdin = stream_to_iterator() if dry_run: @@ -5531,7 +5512,7 @@ async def stream_to_iterator() -> AsyncIterator[JsonElement]: return CLISource.no_count( partial( app_run, - in_stream=stream.empty(), + in_stream=Stream.empty(), app_name=InfraAppName(parsed.app_name), dry_run=parsed.dry_run, config=parsed.config, @@ -5552,7 +5533,7 @@ async def stream_to_iterator() -> AsyncIterator[JsonElement]: ) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -5757,7 +5738,7 @@ def parse(self, arg: Optional[str] = None, ctx: CLIContext = EmptyContext, **kwa return CLISource.no_count(partial(self.show_user, args[1]), required_permissions={Permission.read}) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -5950,7 +5931,7 @@ async def lines_iterator() -> AsyncIterator[str]: else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -6102,7 +6083,7 @@ async def sync_database_result(p: Namespace, maybe_stream: Optional[JsStream]) - async with await graph_db.search_graph_gen( QueryModel(query, fix_model, ctx.env), timeout=timedelta(weeks=200000) ) as cursor: - await sync_fn(query=query, in_stream=stream.iterate(cursor)) + await sync_fn(query=query, in_stream=Stream.iterate(cursor)) if file_output is not None: assert p.database, "No database name provided. Use the --database argument." @@ -6160,11 +6141,10 @@ def key_fn(node: Json) -> Union[str, Tuple[str, str]]: kind_by_id[node["id"]] = node["reported"]["kind"] return cast(str, node["reported"]["kind"]) - async with in_stream.stream() as streamer: - batched = BatchStream(streamer, key_fn, engine_config.batch_size, engine_config.batch_size * 10) - await update_sql( - engine_config, rcm, batched, edges, swap_temp_tables=True, drop_existing_tables=drop_existing_tables - ) + batched = BatchStream(in_stream, key_fn, engine_config.batch_size, engine_config.batch_size * 10) + await update_sql( + engine_config, rcm, batched, edges, swap_temp_tables=True, drop_existing_tables=drop_existing_tables + ) args = arg.split(maxsplit=1) if arg else [] if len(args) == 2 and args[0] == "sync": @@ -6339,16 +6319,16 @@ def parse_duration_or_int(s: str) -> Union[int, timedelta]: async def list_ts() -> Tuple[int, JsGen]: ts = await self.dependencies.db_access.time_series_db.list_time_series() - return len(ts), stream.iterate([to_js(a) for a in ts]) + return len(ts), Stream.iterate([to_js(a) for a in ts]) async def downsample() -> Tuple[int, JsGen]: ts = await self.dependencies.db_access.time_series_db.downsample() if isinstance(ts, str): - return 1, stream.just(ts) + return 1, Stream.just(ts) elif ts: - return len(ts), stream.iterate([{k: v} for k, v in ts.items()]) + return len(ts), Stream.iterate([{k: v} for k, v in ts.items()]) else: - return 1, stream.just("No time series to downsample.") + return 1, Stream.just("No time series to downsample.") args = re.split("\\s+", arg, maxsplit=1) if arg else [] if arg and len(args) == 2 and args[0] == "snapshot": @@ -6363,7 +6343,7 @@ async def downsample() -> Tuple[int, JsGen]: return CLISource.only_count(downsample, required_permissions={Permission.read}) else: return CLISource.single( - lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} + lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read} ) @@ -6459,29 +6439,28 @@ def walk_element(el: JsonElement) -> Iterator[Tuple[str, PotentialSecret]]: if r.startswith("True"): yield el, secret - async def detect_secrets_in(content: JsStream) -> JsGen: + async def detect_secrets_in(in_stream: JsStream) -> JsGen: self.configure_detect() # make sure all plugins are loaded - async with content.stream() as in_stream: - async for element in in_stream: - paths = [p for pl in parsed.path for p in pl] - paths = paths or [PropertyPath.from_list([ctx.section]) if is_node(element) else EmptyPath] - found_secrets = False - for path in paths: - if to_check_js := path.value_in(element): - for secret_string, possible_secret in walk_element(to_check_js): - found_secrets = True - if isinstance(element, dict): - element["info"] = { - "secret_detected": True, - "potential_secret": secret_string, - "secret_type": possible_secret.type, - } - yield element - break - if found_secrets: - break # no need to check other paths - if not found_secrets and not parsed.with_secrets: - yield element + async for element in in_stream: + paths = [p for pl in parsed.path for p in pl] + paths = paths or [PropertyPath.from_list([ctx.section]) if is_node(element) else EmptyPath] + found_secrets = False + for path in paths: + if to_check_js := path.value_in(element): + for secret_string, possible_secret in walk_element(to_check_js): + found_secrets = True + if isinstance(element, dict): + element["info"] = { + "secret_detected": True, + "potential_secret": secret_string, + "secret_type": possible_secret.type, + } + yield element + break + if found_secrets: + break # no need to check other paths + if not found_secrets and not parsed.with_secrets: + yield element return CLIFlow(detect_secrets_in) @@ -6535,7 +6514,7 @@ async def load_model() -> Model: def setup_stream(in_stream: JsStream) -> JsStream: def with_dependencies(model: Model) -> JsStream: - async def process_element(el: JsonElement) -> JsonElement: + def process_element(el: JsonElement) -> JsonElement: if ( is_node(el) and (fqn := value_in_path(el, NodePath.reported_kind)) @@ -6546,9 +6525,9 @@ async def process_element(el: JsonElement) -> JsonElement: set_value_in_path(refinement.value, refinement.path, el) # type: ignore return el - return in_stream | pipe.map(process_element) # type: ignore + return in_stream.map(process_element) - return stream.call(load_model) | pipe.flatmap(with_dependencies) # type: ignore + return Stream.call(load_model).flatmap(with_dependencies) # type: ignore return CLIFlow(setup_stream, required_permissions={Permission.read}) @@ -6590,7 +6569,7 @@ async def delete_node(node_id: NodeId, keep_history: bool) -> AsyncIterator[str] fn=partial(delete_node, node_id=parsed.node_id, keep_history=parsed.keep_history), required_permissions={Permission.write}, ) - return CLISource.single(lambda: stream.just(self.rendered_help(ctx)), required_permissions={Permission.read}) + return CLISource.single(lambda: Stream.just(self.rendered_help(ctx)), required_permissions={Permission.read}) def all_commands(d: TenantDependencies) -> List[CLICommand]: diff --git a/fixcore/fixcore/cli/model.py b/fixcore/fixcore/cli/model.py index f4afcde957..ec2cac1e45 100644 --- a/fixcore/fixcore/cli/model.py +++ b/fixcore/fixcore/cli/model.py @@ -26,8 +26,6 @@ TYPE_CHECKING, ) -from aiostream import stream -from aiostream.core import Stream from attrs import define, field from parsy import test_char, string from rich.jupyter import JupyterMixin @@ -42,6 +40,7 @@ from fixcore.query.template_expander import render_template from fixcore.types import Json, JsonElement from fixcore.util import AccessJson, uuid_str, from_utc, utc, utc_str +from fixlib.asynchronous.stream import Stream from fixlib.parse_util import l_curly_dp, r_curly_dp from fixlib.utils import get_local_tzinfo @@ -236,7 +235,7 @@ def __init__( @staticmethod def make_stream(in_stream: JsGen) -> JsStream: - return in_stream if isinstance(in_stream, Stream) else stream.iterate(in_stream) + return in_stream if isinstance(in_stream, Stream) else Stream.iterate(in_stream) @define @@ -316,7 +315,7 @@ def single( @staticmethod def empty() -> CLISource: - return CLISource.with_count(stream.empty, 0) + return CLISource.with_count(Stream.empty, 0) class CLIFlow(CLIAction): @@ -739,7 +738,7 @@ async def execute(self) -> Tuple[CLISourceContext, JsStream]: flow = await flow_action.flow(flow) return context, flow else: - return CLISourceContext(count=0), stream.empty() + return CLISourceContext(count=0), Stream.empty() class CLI(ABC): diff --git a/fixcore/fixcore/db/graphdb.py b/fixcore/fixcore/db/graphdb.py index 14b997c8a5..0424414776 100644 --- a/fixcore/fixcore/db/graphdb.py +++ b/fixcore/fixcore/db/graphdb.py @@ -23,7 +23,6 @@ Union, ) -from aiostream import stream, pipe from arango import AnalyzerGetError from arango.collection import VertexCollection, StandardCollection, EdgeCollection from arango.graph import Graph @@ -67,6 +66,7 @@ set_value_in_path, if_set, ) +from fixlib.asynchronous.stream import Stream log = logging.getLogger(__name__) @@ -675,9 +675,8 @@ async def move_security_temp_to_proper() -> None: try: # stream updates to the temp collection - async with (stream.iterate(iterator) | pipe.chunks(1000)).stream() as streamer: - async for part in streamer: - await update_chunk(dict(part)) + async for part in Stream.iterate(iterator).chunks(1000): + await update_chunk(dict(part)) # move temp collection to proper and history collection await move_security_temp_to_proper() finally: diff --git a/fixcore/fixcore/infra_apps/local_runtime.py b/fixcore/fixcore/infra_apps/local_runtime.py index c2d2376291..4ca160e8a1 100644 --- a/fixcore/fixcore/infra_apps/local_runtime.py +++ b/fixcore/fixcore/infra_apps/local_runtime.py @@ -3,7 +3,6 @@ from pydoc import locate from typing import List, AsyncIterator, Type, Optional, Any -from aiostream import stream, pipe from jinja2 import Environment from fixcore.cli import NoExitArgumentParser, JsStream, JsGen @@ -14,6 +13,7 @@ from fixcore.infra_apps.runtime import Runtime from fixcore.service import Service from fixcore.types import Json, JsonElement +from fixlib.asynchronous.stream import Stream from fixlib.asynchronous.utils import async_lines from fixlib.durations import parse_optional_duration @@ -46,9 +46,8 @@ async def execute( Runtime implementation that runs the app locally. """ async for line in self.generate_template(graph, manifest, config, stdin, argv): - async with (await self._interpret_line(line, ctx)).stream() as streamer: - async for item in streamer: - yield item + async for item in await self._interpret_line(line, ctx): + yield item async def generate_template( self, @@ -117,4 +116,4 @@ async def _interpret_line(self, line: str, ctx: CLIContext) -> JsStream: total_nr_outputs = total_nr_outputs + (src_ctx.count or 0) command_streams.append(command_output_stream) - return stream.iterate(command_streams) | pipe.concat(task_limit=1) + return Stream.iterate(command_streams).concat() # type: ignore diff --git a/fixcore/fixcore/model/db_updater.py b/fixcore/fixcore/model/db_updater.py index cab00efb37..9a29c95573 100644 --- a/fixcore/fixcore/model/db_updater.py +++ b/fixcore/fixcore/model/db_updater.py @@ -13,11 +13,9 @@ from multiprocessing import Process, Queue from pathlib import Path from queue import Empty -from typing import Optional, Union, Any, Generator, List, AsyncIterator, Dict +from typing import Optional, Union, Any, List, AsyncIterator, Dict import aiofiles -from aiostream import stream, pipe -from aiostream.core import Stream from attrs import define from fixcore.analytics import AnalyticsEventSender, InMemoryEventSender, AnalyticsEvent @@ -36,6 +34,7 @@ from fixcore.system_start import db_access, setup_process, reset_process_start_method from fixcore.types import Json from fixcore.util import utc, uuid_str, shutdown_process +from fixlib.asynchronous.stream import Stream log = logging.getLogger(__name__) @@ -56,9 +55,9 @@ class ReadFile(ProcessAction): path: Path task_id: Optional[str] - def jsons(self) -> Generator[Json, Any, None]: - with open(self.path, "r", encoding="utf-8") as f: - for line in f: + async def jsons(self) -> AsyncIterator[Json]: + async with aiofiles.open(self.path, "r", encoding="utf-8") as f: + async for line in f: if line.strip(): yield json.loads(line) @@ -75,8 +74,8 @@ class ReadElement(ProcessAction): elements: List[Union[bytes, Json]] task_id: Optional[str] - def jsons(self) -> Generator[Json, Any, None]: - return (e if isinstance(e, dict) else json.loads(e) for e in self.elements) + def jsons(self) -> AsyncIterator[Json]: + return Stream.iterate(self.elements).map(lambda e: e if isinstance(e, dict) else json.loads(e)) @define @@ -125,15 +124,15 @@ def get_value(self) -> GraphUpdate: class DbUpdaterProcess(Process): """ - This update class implements Process and is supposed to run as separate process. + This update class implements Process and is supposed to run as a separate process. Note: default starting method is supposed to be "spawn". This process has 2 queues to read input from and write output to. - All elements in either queues are of type ProcessAction. + All elements in all queues are of type ProcessAction. The parent process should stream the raw commands of graph to this process via ReadElement objects. Once the MergeGraph action is received, the graph gets imported. - From here the parent expects result messages from the child. + From here, the parent expects result messages from the child. All events happen in the child are forwarded to the parent via EmitEvent. Once the graph update is done, a result is send. The result is either an exception in case of failure or a graph update in success case. @@ -156,8 +155,8 @@ def __init__( def next_action(self) -> ProcessAction: try: - # graph is read into memory. If the sender does not send data in a given amount of time, - # we raise an exception and abort the update. + # The graph is read into memory. + # If the sender does not send data in a given amount of time, we raise an exception and abort the update. return self.read_queue.get(True, 90) except Empty as ex: raise ImportAborted("Merge process did not receive any data for more than 90 seconds. Abort.") from ex @@ -168,12 +167,12 @@ async def merge_graph(self, db: DbAccess) -> GraphUpdate: # type: ignore builder = GraphBuilder(model, self.change_id) nxt = self.next_action() if isinstance(nxt, ReadFile): - for element in nxt.jsons(): + async for element in nxt.jsons(): builder.add_from_json(element) nxt = self.next_action() elif isinstance(nxt, ReadElement): while isinstance(nxt, ReadElement): - for element in nxt.jsons(): + async for element in nxt.jsons(): builder.add_from_json(element) log.debug(f"Read {int(BatchSize / 1000)}K elements in process") nxt = self.next_action() @@ -276,16 +275,11 @@ async def __process_item(self, item: GraphUpdateTask) -> Union[GraphUpdate, Exce async def start(self) -> None: async def wait_for_update() -> None: log.info("Start waiting for graph updates") - fl = ( - stream.call(self.update_queue.get) # type: ignore - | pipe.cycle() - | pipe.map(self.__process_item, task_limit=self.config.graph.parallel_imports) # type: ignore - ) + fl = Stream.for_ever(self.update_queue.get).map(self.__process_item, task_limit=self.config.graph.parallel_imports) # type: ignore # noqa with suppress(CancelledError): - async with fl.stream() as streamer: - async for update in streamer: - if isinstance(update, GraphUpdate): - log.info(f"Finished spawned graph merge: {update}") + async for update in fl: + if isinstance(update, GraphUpdate): + log.info(f"Finished spawned graph merge: {update}") self.handler_task = asyncio.create_task(wait_for_update()) @@ -373,19 +367,17 @@ async def read_forever() -> GraphUpdate: task: Optional[Task[GraphUpdate]] = None result: Optional[GraphUpdate] = None try: - reset_process_start_method() # other libraries might have tampered the value in the mean time + reset_process_start_method() # other libraries might have tampered the value in the meantime updater.start() task = read_results() # concurrently read result queue # Either send a file or stream the content directly if isinstance(content, Path): await send_to_child(ReadFile(content, task_id)) else: - chunked: Stream[List[Union[bytes, Json]]] = stream.chunks(content, BatchSize) # type: ignore - async with chunked.stream() as streamer: - async for lines in streamer: - if not await send_to_child(ReadElement(lines, task_id)): - # in case the child is dead, we should stop - break + async for lines in Stream.iterate(content).chunks(BatchSize): + if not await send_to_child(ReadElement(lines, task_id)): + # in case the child is dead, we should stop + break await send_to_child(MergeGraph(db.name, change_id, maybe_batch is not None, task_id)) result = await task # wait for final result await self.model_handler.load_model(db.name, force=True) # reload model to get the latest changes diff --git a/fixcore/fixcore/report/benchmark_renderer.py b/fixcore/fixcore/report/benchmark_renderer.py index af40975da7..babd149d5d 100644 --- a/fixcore/fixcore/report/benchmark_renderer.py +++ b/fixcore/fixcore/report/benchmark_renderer.py @@ -1,6 +1,5 @@ from typing import AsyncGenerator, List, AsyncIterable -from aiostream import stream from networkx import DiGraph from rich._emoji_codes import EMOJI @@ -91,27 +90,26 @@ def render_check_result(check_result: CheckResult, account: str) -> str: async def respond_benchmark_result(gen: AsyncIterable[JsonElement]) -> AsyncGenerator[str, None]: - # step 1: read graph + # step 1: read graph graph = DiGraph() - async with stream.iterate(gen).stream() as streamer: - async for item in streamer: - if isinstance(item, dict): - type_name = item.get("type") - if type_name == "node": - uid = value_in_path(item, NodePath.node_id) - reported = value_in_path(item, NodePath.reported) - kind = value_in_path(item, NodePath.reported_kind) - if uid and reported and kind and (reader := kind_reader.get(kind)): - graph.add_node(uid, data=reader(item)) - elif type_name == "edge": - from_node = value_in_path(item, NodePath.from_node) - to_node = value_in_path(item, NodePath.to_node) - if from_node and to_node: - graph.add_edge(from_node, to_node) - else: - raise AttributeError(f"Expect json object but got: {type(item)}: {item}") + async for item in gen: + if isinstance(item, dict): + type_name = item.get("type") + if type_name == "node": + uid = value_in_path(item, NodePath.node_id) + reported = value_in_path(item, NodePath.reported) + kind = value_in_path(item, NodePath.reported_kind) + if uid and reported and kind and (reader := kind_reader.get(kind)): + graph.add_node(uid, data=reader(item)) + elif type_name == "edge": + from_node = value_in_path(item, NodePath.from_node) + to_node = value_in_path(item, NodePath.to_node) + if from_node and to_node: + graph.add_edge(from_node, to_node) else: raise AttributeError(f"Expect json object but got: {type(item)}: {item}") + else: + raise AttributeError(f"Expect json object but got: {type(item)}: {item}") # step 2: read benchmark result from graph def traverse(node_id: str, collection: CheckCollectionResult) -> None: diff --git a/fixcore/fixcore/report/inspector_service.py b/fixcore/fixcore/report/inspector_service.py index ceb9c8f5b4..f3b70cc77e 100644 --- a/fixcore/fixcore/report/inspector_service.py +++ b/fixcore/fixcore/report/inspector_service.py @@ -3,8 +3,6 @@ from functools import lru_cache from typing import Optional, List, Dict, Tuple, Callable, AsyncIterator, cast, Set -from aiostream import stream, pipe -from aiostream.core import Stream from attr import define from fixcore.analytics import CoreEvent @@ -40,6 +38,7 @@ from fixcore.service import Service from fixcore.types import Json from fixcore.util import value_in_path, uuid_str, value_in_path_get +from fixlib.asynchronous.stream import Stream from fixlib.json_bender import Bender, S, bend log = logging.getLogger(__name__) @@ -380,7 +379,7 @@ async def list_failing_resources( async def __list_failing_resources( self, graph: GraphName, model: Model, inspection: ReportCheck, context: CheckContext ) -> AsyncIterator[Json]: - # final environment: defaults are coming from the check and are eventually overriden in the config + # final environment: defaults are coming from the check and are eventually overridden in the config env = inspection.environment(context.override_values()) account_id_prop = "ancestors.account.reported.id" ignore_prop = "metadata.security_ignore" @@ -484,7 +483,7 @@ def to_result(cc: CheckCollection) -> CheckCollectionResult: node_id=next_node_id(), ) - async def __perform_checks( # type: ignore + async def __perform_checks( self, graph: GraphName, checks: List[ReportCheck], context: CheckContext ) -> Dict[str, SingleCheckResult]: # load model @@ -493,11 +492,10 @@ async def __perform_checks( # type: ignore async def perform_single(check: ReportCheck) -> Tuple[str, SingleCheckResult]: return check.id, await self.__perform_check(graph, model, check, context) - check_results: Stream[Tuple[str, SingleCheckResult]] = stream.iterate(checks) | pipe.map( - perform_single, ordered=False, task_limit=context.parallel_checks # type: ignore + check_results: Stream[Tuple[str, SingleCheckResult]] = Stream.iterate(checks).map( + perform_single, ordered=False, task_limit=context.parallel_checks ) - async with check_results.stream() as streamer: - return {key: value async for key, value in streamer} + return {key: value async for key, value in check_results} async def __perform_check( self, graph: GraphName, model: Model, inspection: ReportCheck, context: CheckContext diff --git a/fixcore/fixcore/task/task_handler.py b/fixcore/fixcore/task/task_handler.py index d8677fee54..4d7073afe1 100644 --- a/fixcore/fixcore/task/task_handler.py +++ b/fixcore/fixcore/task/task_handler.py @@ -8,7 +8,6 @@ from copy import copy from datetime import timedelta from typing import Optional, Any, Callable, Union, Sequence, Dict, List, Tuple -from aiostream import stream from attrs import evolve from fixcore.analytics import AnalyticsEventSender, CoreEvent @@ -57,6 +56,7 @@ ) from fixcore.util import first, Periodic, group_by, utc_str, utc, partition_by from fixcore.types import Json +from fixlib.asynchronous.stream import Stream log = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def __init__( # note: the waiting queue is kept in memory and lost when the service is restarted. self.start_when_done: Dict[str, TaskDescription] = {} - # Step1: define all workflows and jobs in code: later it will be persisted and read from database + # Step1: define all workflows and jobs in code: later it will be persisted and read from the database self.task_descriptions: Sequence[TaskDescription] = [*self.known_workflows(config), *self.known_jobs()] self.tasks: Dict[TaskId, RunningTask] = {} self.message_bus_watcher: Optional[Task[None]] = None @@ -496,7 +496,7 @@ async def execute_commands() -> None: results[command] = None elif isinstance(command, ExecuteOnCLI): ctx = evolve(self.cli_context, env={**command.env, **wi.descriptor.environment}) - result = await self.cli.execute_cli_command(command.command, stream.list, ctx) # type: ignore + result = await self.cli.execute_cli_command(command.command, Stream.as_list, ctx) results[command] = result else: raise AttributeError(f"Does not understand this command: {wi.descriptor.name}: {command}") diff --git a/fixcore/fixcore/web/api.py b/fixcore/fixcore/web/api.py index e7c830160b..06b95ccd77 100644 --- a/fixcore/fixcore/web/api.py +++ b/fixcore/fixcore/web/api.py @@ -54,7 +54,6 @@ from aiohttp.web_fileresponse import FileResponse from aiohttp.web_response import json_response from aiohttp_swagger3 import SwaggerFile, SwaggerUiSettings -from aiostream import stream from attrs import evolve from dateutil import parser as date_parser from multidict import MultiDict @@ -134,6 +133,7 @@ WorkerTaskResult, WorkerTaskInProgress, ) +from fixlib.asynchronous.stream import Stream from fixlib.asynchronous.web.ws_handler import accept_websocket, clean_ws_handler from fixlib.durations import parse_duration from fixlib.jwt import encode_jwt @@ -664,7 +664,7 @@ async def perform_benchmark_on_checks(self, request: Request, deps: TenantDepend ) return await single_result(request, to_js(result)) - async def perform_benchmark(self, request: Request, deps: TenantDependencies) -> StreamResponse: # type: ignore + async def perform_benchmark(self, request: Request, deps: TenantDependencies) -> StreamResponse: benchmark = request.match_info["benchmark"] graph = GraphName(request.match_info["graph_id"]) acc = request.query.get("accounts") @@ -677,8 +677,8 @@ async def perform_benchmark(self, request: Request, deps: TenantDependencies) -> else: raise ValueError(f"Unknown action {action}. One of run or load is expected.") result_graph = results[benchmark].to_graph() - async with stream.iterate(result_graph).stream() as streamer: - return await self.stream_response_from_gen(request, streamer, count=len(result_graph)) + stream = Stream.iterate(result_graph) + return await self.stream_response_from_gen(request, stream, count=len(result_graph)) async def inspection_checks(self, request: Request, deps: TenantDependencies) -> StreamResponse: provider = request.query.get("provider") @@ -1433,7 +1433,7 @@ async def write_files(mpr: MultipartReader, tmp_dir: str) -> Dict[str, str]: if temp_dir: shutil.rmtree(temp_dir) - async def execute_parsed( # type: ignore + async def execute_parsed( self, request: Request, command: str, parsed: List[ParsedCommandLine], ctx: CLIContext ) -> StreamResponse: # what is the accepted content type @@ -1455,43 +1455,41 @@ async def execute_parsed( # type: ignore first_result = parsed[0] src_ctx, generator = await first_result.execute() # flat the results from 0 or 1 - async with generator.stream() as streamer: - gen = await force_gen(streamer) - if first_result.produces.text: - text_gen = ctx.text_generator(first_result, gen) - return await self.stream_response_from_gen( - request, - text_gen, - count=src_ctx.count, - total_count=src_ctx.total_count, - query_stats=src_ctx.stats, - additional_header=first_result.envelope, - ) - elif first_result.produces.file_path: - await mp_response.prepare(request) - await Api.multi_file_response(first_result, gen, boundary, mp_response) - await Api.close_multi_part_response(mp_response, boundary) - return mp_response - else: - raise AttributeError(f"Can not handle type: {first_result.produces}") + gen = await force_gen(generator) + if first_result.produces.text: + text_gen = ctx.text_generator(first_result, gen) + return await self.stream_response_from_gen( + request, + text_gen, + count=src_ctx.count, + total_count=src_ctx.total_count, + query_stats=src_ctx.stats, + additional_header=first_result.envelope, + ) + elif first_result.produces.file_path: + await mp_response.prepare(request) + await Api.multi_file_response(first_result, gen, boundary, mp_response) + await Api.close_multi_part_response(mp_response, boundary) + return mp_response + else: + raise AttributeError(f"Can not handle type: {first_result.produces}") elif len(parsed) > 1: await mp_response.prepare(request) for single in parsed: _, generator = await single.execute() - async with generator.stream() as streamer: - gen = await force_gen(streamer) - if single.produces.text: - with MultipartWriter(repr(single.produces), boundary) as mp: - text_gen = ctx.text_generator(single, gen) - content_type, result_stream = await result_binary_gen(request, text_gen) - mp.append_payload( - AsyncIterablePayload(result_stream, content_type=content_type, headers=single.envelope) - ) - await mp.write(mp_response, close_boundary=False) - elif single.produces.file_path: - await Api.multi_file_response(single, gen, boundary, mp_response) - else: - raise AttributeError(f"Can not handle type: {single.produces}") + gen = await force_gen(generator) + if single.produces.text: + with MultipartWriter(repr(single.produces), boundary) as mp: + text_gen = ctx.text_generator(single, gen) + content_type, result_stream = await result_binary_gen(request, text_gen) + mp.append_payload( + AsyncIterablePayload(result_stream, content_type=content_type, headers=single.envelope) + ) + await mp.write(mp_response, close_boundary=False) + elif single.produces.file_path: + await Api.multi_file_response(single, gen, boundary, mp_response) + else: + raise AttributeError(f"Can not handle type: {single.produces}") await Api.close_multi_part_response(mp_response, boundary) return mp_response else: diff --git a/fixcore/pyproject.toml b/fixcore/pyproject.toml index 5c85874c80..2ef84831f1 100644 --- a/fixcore/pyproject.toml +++ b/fixcore/pyproject.toml @@ -35,7 +35,6 @@ dependencies = [ "aiohttp-jinja2", "aiohttp-swagger3", "aiohttp[speedups]", - "aiostream", "cryptography", "deepdiff", "detect_secrets", diff --git a/fixcore/tests/fixcore/cli/command_test.py b/fixcore/tests/fixcore/cli/command_test.py index cf85a7ec38..514d0fc2c3 100644 --- a/fixcore/tests/fixcore/cli/command_test.py +++ b/fixcore/tests/fixcore/cli/command_test.py @@ -13,9 +13,9 @@ from _pytest.logging import LogCaptureFixture from aiohttp import ClientTimeout from aiohttp.web import Request -from aiostream import stream, pipe from attrs import evolve from pytest import fixture + from fixcore import version from fixcore.cli import is_node, JsStream, list_sink from fixcore.cli.cli import CLIService @@ -48,6 +48,7 @@ from fixcore.user import UsersConfigId from fixcore.util import AccessJson, utc_str, utc from fixcore.worker_task_queue import WorkerTask +from fixlib.asynchronous.stream import Stream from tests.fixcore.util_test import not_in_path @@ -279,7 +280,7 @@ async def test_list_sink(cli: CLI, dependencies: TenantDependencies) -> None: async def test_flat_sink(cli: CLI) -> None: parsed = await cli.evaluate_cli_command("json [1,2,3] | dump; json [4,5,6] | dump; json [7,8,9] | dump") expected = [1, 2, 3, 4, 5, 6, 7, 8, 9] - assert await stream.list(stream.iterate((await p.execute())[1] for p in parsed) | pipe.concat()) == expected + assert expected == await Stream.iterate(await (await p.execute())[1].collect() for p in parsed).flatten().collect() # type: ignore @pytest.mark.asyncio @@ -315,7 +316,7 @@ async def test_format(cli: CLI) -> None: async def test_workflows_command(cli: CLIService, task_handler: TaskHandlerService, test_workflow: Workflow) -> None: async def execute(cmd: str) -> List[JsonElement]: ctx = CLIContext(cli.cli_env) - return (await cli.execute_cli_command(cmd, list_sink, ctx))[0] # type: ignore + return (await cli.execute_cli_command(cmd, list_sink, ctx))[0] assert await execute("workflows list") == ["sleep_workflow", "wait_for_collect_done", "test_workflow"] assert await execute("workflows show test_workflow") == [to_js(test_workflow)] @@ -754,15 +755,14 @@ async def test_aggregation_to_count_command(cli: CLI) -> None: @pytest.mark.asyncio async def test_system_backup_command(cli: CLI) -> None: async def check_backup(res: JsStream) -> None: - async with res.stream() as streamer: - only_one = True - async for s in streamer: - path = FilePath.from_path(s) - assert path.local.exists() - # backup should have size between 30k and 1500k (adjust size if necessary) - assert 30000 < path.local.stat().st_size < 1500000 - assert only_one - only_one = False + only_one = True + async for s in res: + path = FilePath.from_path(s) + assert path.local.exists() + # backup should have size between 30k and 1500k (adjust size if necessary) + assert 30000 < path.local.stat().st_size < 1500000 + assert only_one + only_one = False await cli.execute_cli_command("system backup create", check_backup) @@ -781,10 +781,9 @@ async def test_system_restore_command(cli: CLI, tmp_directory: str) -> None: backup = os.path.join(tmp_directory, "backup") async def move_backup(res: JsStream) -> None: - async with res.stream() as streamer: - async for s in streamer: - path = FilePath.from_path(s) - os.rename(path.local, backup) + async for s in res: + path = FilePath.from_path(s) + os.rename(path.local, backup) await cli.execute_cli_command("system backup create", move_backup) ctx = CLIContext(uploaded_files={"backup": backup}) @@ -802,11 +801,10 @@ async def test_configs_command(cli: CLI, tmp_directory: str) -> None: config_file = os.path.join(tmp_directory, "config.yml") async def check_file_is_yaml(res: JsStream) -> None: - async with res.stream() as streamer: - async for s in streamer: - assert isinstance(s, str) - with open(s, "r") as file: - yaml.safe_load(file.read()) + async for s in res: + assert isinstance(s, str) + with open(s, "r") as file: + yaml.safe_load(file.read()) # create a new config entry create_result = await cli.execute_cli_command("configs set test_config t1=1, t2=2, t3=3 ", list_sink) @@ -865,19 +863,18 @@ async def test_templates_command(cli: CLI) -> None: @pytest.mark.asyncio async def test_write_command(cli: CLI) -> None: async def check_file(res: JsStream, check_content: Optional[str] = None) -> None: - async with res.stream() as streamer: - only_one = True - async for s in streamer: - fp = FilePath.from_path(s) - assert fp.local.exists() and fp.local.is_file() - assert 1 < fp.local.stat().st_size < 100000 - assert fp.user.name.startswith("write_test") - assert only_one - only_one = False - if check_content: - with open(fp.local, "r") as file: - data = file.read() - assert data == check_content + only_one = True + async for s in res: + fp = FilePath.from_path(s) + assert fp.local.exists() and fp.local.is_file() + assert 1 < fp.local.stat().st_size < 100000 + assert fp.user.name.startswith("write_test") + assert only_one + only_one = False + if check_content: + with open(fp.local, "r") as file: + data = file.read() + assert data == check_content # result can be read as json await cli.execute_cli_command("search all limit 3 | format --json | write write_test.json ", check_file) @@ -1095,14 +1092,12 @@ async def history_count(cmd: str) -> int: @pytest.mark.asyncio async def test_aggregate(dependencies: TenantDependencies) -> None: - in_stream = stream.iterate( - [{"a": 1, "b": 1, "c": 1}, {"a": 2, "b": 1, "c": 1}, {"a": 3, "b": 2, "c": 1}, {"a": 4, "b": 2, "c": 1}] - ) - - async def aggregate(agg_str: str) -> List[JsonElement]: # type: ignore + async def aggregate(agg_str: str) -> List[JsonElement]: + in_stream = Stream.iterate( + [{"a": 1, "b": 1, "c": 1}, {"a": 2, "b": 1, "c": 1}, {"a": 3, "b": 2, "c": 1}, {"a": 4, "b": 2, "c": 1}] + ) res = AggregateCommand(dependencies).parse(agg_str) - async with (await res.flow(in_stream)).stream() as flow: - return [s async for s in flow] + return [s async for s in (await res.flow(in_stream))] assert await aggregate("b as bla, c, r.d.f.name: sum(1) as count, min(a) as min, max(a) as max") == [ {"group": {"bla": 1, "c": 1, "r.d.f.name": None}, "count": 2, "min": 1, "max": 2}, @@ -1161,11 +1156,10 @@ async def execute(cmd: str, _: Type[T]) -> List[T]: return cast(List[T], result[0]) async def check_file_is_yaml(res: JsStream) -> None: - async with res.stream() as streamer: - async for s in streamer: - assert isinstance(s, str) - with open(s, "r") as file: - yaml.safe_load(file.read()) + async for s in res: + assert isinstance(s, str) + with open(s, "r") as file: + yaml.safe_load(file.read()) # install a package assert "installed successfully" in (await execute("apps install cleanup-untagged", str))[0] @@ -1235,7 +1229,7 @@ async def check_file_is_yaml(res: JsStream) -> None: async def test_user(cli: CLI) -> None: async def execute(cmd: str) -> List[JsonElement]: all_results = await cli.execute_cli_command(cmd, list_sink) - return all_results[0] # type: ignore + return all_results[0] # remove all existing users await cli.dependencies.config_handler.delete_config(UsersConfigId) @@ -1355,10 +1349,9 @@ async def execute(cmd: str, _: Type[T]) -> List[T]: dump = os.path.join(tmp_directory, "dump") async def move_dump(res: JsStream) -> None: - async with res.stream() as streamer: - async for s in streamer: - fp = FilePath.from_path(s) - os.rename(fp.local, dump) + async for s in res: + fp = FilePath.from_path(s) + os.rename(fp.local, dump) # graph export works await cli.execute_cli_command("graph export graphtest dump", move_dump) @@ -1387,28 +1380,27 @@ async def sync_and_check( ) -> Json: result: List[Json] = [] - async def check(in_: JsStream) -> None: - async with in_.stream() as streamer: - async for s in streamer: - assert isinstance(s, dict) - path = FilePath.from_path(s) - # open sqlite database - conn = sqlite3.connect(path.local) - c = conn.cursor() - tables = { - row[0] for row in c.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'").fetchall() - } - if expected_tables is not None: - assert tables == expected_tables - if expected_table_count is not None: - assert len(tables) == expected_table_count - if expected_table is not None: - for table in tables: - count = c.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] - assert expected_table(table, count), f"Table {table} has {count} rows" - c.close() - conn.close() - result.append(s) + async def check(streamer: JsStream) -> None: + async for s in streamer: + assert isinstance(s, dict) + path = FilePath.from_path(s) + # open sqlite database + conn = sqlite3.connect(path.local) + c = conn.cursor() + tables = { + row[0] for row in c.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'").fetchall() + } + if expected_tables is not None: + assert tables == expected_tables + if expected_table_count is not None: + assert len(tables) == expected_table_count + if expected_table is not None: + for table in tables: + count = c.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + assert expected_table(table, count), f"Table {table} has {count} rows" + c.close() + conn.close() + result.append(s) await cli.execute_cli_command(cmd, check) assert len(result) == 1 diff --git a/fixcore/tests/fixcore/db/graphdb_test.py b/fixcore/tests/fixcore/db/graphdb_test.py index f5bc1b393b..a38a9d768b 100644 --- a/fixcore/tests/fixcore/db/graphdb_test.py +++ b/fixcore/tests/fixcore/db/graphdb_test.py @@ -291,7 +291,7 @@ async def check_usage() -> bool: expected = {"min": 42, "avg": 42, "max": 42} return node_usage == expected - await eventually(check_usage) + await eventually(check_usage, timeout=timedelta(seconds=30)) # exactly the same graph is updated: expect no changes assert await graph_db.merge_graph(create("yes or no"), foo_model) == (p, GraphUpdate(0, 0, 0, 0, 0, 0)) diff --git a/fixcore/tests/fixcore/hypothesis_extension.py b/fixcore/tests/fixcore/hypothesis_extension.py index 297376f6ea..db2cecf6b4 100644 --- a/fixcore/tests/fixcore/hypothesis_extension.py +++ b/fixcore/tests/fixcore/hypothesis_extension.py @@ -1,9 +1,7 @@ import string from datetime import datetime -from typing import TypeVar, Callable, Any, cast, Optional, List, Generator +from typing import TypeVar, Any, cast, Optional, List, Generator -from aiostream import stream -from aiostream.core import Stream from hypothesis.strategies import ( SearchStrategy, just, @@ -20,6 +18,7 @@ from fixcore.model.resolve_in_graph import NodePath from fixcore.types import JsonElement, Json from fixcore.util import value_in_path, interleave +from fixlib.asynchronous.stream import Stream T = TypeVar("T") @@ -71,4 +70,4 @@ def from_node() -> Generator[Json, Any, None]: for from_n, to_n in interleave(node_ids): yield {"type": "edge", "from": from_n, "to": to_n} - return stream.iterate(from_node()) + return Stream.iterate(from_node()) diff --git a/fixcore/tests/fixcore/report/benchmark_renderer_test.py b/fixcore/tests/fixcore/report/benchmark_renderer_test.py index 080740695c..f34c711124 100644 --- a/fixcore/tests/fixcore/report/benchmark_renderer_test.py +++ b/fixcore/tests/fixcore/report/benchmark_renderer_test.py @@ -1,16 +1,16 @@ import pytest -from aiostream import stream from fixcore.report.benchmark_renderer import respond_benchmark_result from fixcore.report.inspector_service import InspectorService from fixcore.ids import GraphName +from fixlib.asynchronous.stream import Stream @pytest.mark.asyncio async def test_benchmark_renderer(inspector_service: InspectorService) -> None: bench_results = await inspector_service.perform_benchmarks(GraphName("ns"), ["test"]) bench_result = bench_results["test"] - render_result = [elem async for elem in respond_benchmark_result(stream.iterate(bench_result.to_graph()))] + render_result = [elem async for elem in respond_benchmark_result(Stream.iterate(bench_result.to_graph()))] assert len(render_result) == 1 assert ( render_result[0] diff --git a/fixcore/tests/fixcore/util_test.py b/fixcore/tests/fixcore/util_test.py index bf64f33d8e..5a688f3366 100644 --- a/fixcore/tests/fixcore/util_test.py +++ b/fixcore/tests/fixcore/util_test.py @@ -5,7 +5,6 @@ import pytest import pytz -from aiostream import stream from fixcore.util import ( AccessJson, @@ -21,6 +20,7 @@ utc_str, parse_utc, ) +from fixlib.asynchronous.stream import Stream def not_in_path(name: str, *other: str) -> bool: @@ -107,17 +107,9 @@ def test_del_value_in_path() -> None: @pytest.mark.asyncio async def test_async_gen() -> None: - async with stream.empty().stream() as empty: - async for _ in await force_gen(empty): - pass - - with pytest.raises(Exception): - async with stream.throw(Exception(";)")).stream() as err: - async for _ in await force_gen(err): - pass - - async with stream.iterate(range(0, 100)).stream() as elems: - assert [x async for x in await force_gen(elems)] == list(range(0, 100)) + async for _ in await force_gen(Stream.empty()): + pass + assert [x async for x in await force_gen(Stream.iterate(range(0, 100)))] == list(range(0, 100)) def test_deep_merge() -> None: diff --git a/fixcore/tests/fixcore/web/content_renderer_test.py b/fixcore/tests/fixcore/web/content_renderer_test.py index 4d3c5c6724..f37cf276e5 100644 --- a/fixcore/tests/fixcore/web/content_renderer_test.py +++ b/fixcore/tests/fixcore/web/content_renderer_test.py @@ -4,7 +4,6 @@ import pytest import yaml -from aiostream import stream from hypothesis import given, settings, HealthCheck from hypothesis.strategies import lists @@ -18,6 +17,7 @@ respond_cytoscape, respond_graphml, ) +from fixlib.asynchronous.stream import Stream from tests.fixcore.hypothesis_extension import ( json_array_gen, json_simple_element_gen, @@ -30,79 +30,72 @@ @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_json(elements: List[JsonElement]) -> None: - async with stream.iterate(elements).stream() as streamer: - result = "" - async for elem in respond_json(streamer): - result += elem - assert json.loads(result) == elements + result = "" + async for elem in respond_json(Stream.iterate(elements)): + result += elem + assert json.loads(result) == elements @given(json_array_gen) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_ndjson(elements: List[JsonElement]) -> None: - async with stream.iterate(elements).stream() as streamer: - result = [] - async for elem in respond_ndjson(streamer): - result.append(json.loads(elem.strip())) - assert result == elements + result = [] + async for elem in respond_ndjson(Stream.iterate(elements)): + result.append(json.loads(elem.strip())) + assert result == elements @given(json_array_gen) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_yaml(elements: List[JsonElement]) -> None: - async with stream.iterate(elements).stream() as streamer: - result = "" - async for elem in respond_yaml(streamer): - result += elem + "\n" - assert [a for a in yaml.full_load_all(result)] == elements + result = "" + async for elem in respond_yaml(Stream.iterate(elements)): + result += elem + "\n" + assert [a for a in yaml.full_load_all(result)] == elements @given(lists(json_simple_element_gen, min_size=1, max_size=10)) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_text_simple_elements(elements: List[JsonElement]) -> None: - async with stream.iterate(elements).stream() as streamer: - result = "" - async for elem in respond_text(streamer): - result += elem + "\n" - # every element is rendered as one or more line (string with \n is rendered as multiple lines) - assert len(elements) + 1 <= len(result.split("\n")) + result = "" + async for elem in respond_text(Stream.iterate(elements)): + result += elem + "\n" + # every element is rendered as one or more line (string with \n is rendered as multiple lines) + assert len(elements) + 1 <= len(result.split("\n")) @given(lists(node_gen(), min_size=1, max_size=10)) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_text_complex_elements(elements: List[JsonElement]) -> None: - async with stream.iterate(elements).stream() as streamer: - result = "" - async for elem in respond_text(streamer): - result += elem - # every element is rendered as yaml with --- as object deliminator - assert len(elements) == len(result.split("---")) + result = "" + async for elem in respond_text(Stream.iterate(elements)): + result += elem + # every element is rendered as yaml with --- as object deliminator + assert len(elements) == len(result.split("---")) @given(lists(node_gen(), min_size=1, max_size=10)) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_cytoscape(elements: List[Json]) -> None: - async with graph_stream(elements).stream() as streamer: - result = "" - async for elem in respond_cytoscape(streamer): - result += elem - # The resulting string can be parsed as json - assert json.loads(result) + result = "" + async for elem in respond_cytoscape(Stream.iterate(elements)): + result += elem + # The resulting string can be parsed as json + assert json.loads(result) @given(lists(node_gen(), min_size=1, max_size=10)) @settings(max_examples=20, suppress_health_check=list(HealthCheck), deadline=1000) @pytest.mark.asyncio async def test_graphml(elements: List[Json]) -> None: - async with graph_stream(elements).stream() as streamer: - result = "" - async for elem in respond_graphml(streamer): - result += elem + result = "" + async for elem in respond_graphml(Stream.iterate(elements)): + result += elem # The resulting string can be parsed as xml assert ElementTree.fromstring(result) is not None @@ -119,30 +112,29 @@ def edge(from_node: str, to_node: str) -> Json: nodes = [node("a", "acc1"), node("b", "acc1"), node("c", "acc2")] edges = [edge("a", "b"), edge("a", "c"), edge("b", "c")] - async with stream.iterate(nodes + edges).stream() as streamer: - result = "" - async for elem in respond_dot(streamer): - result += elem + "\n" - expected = ( - "digraph {\n" - "rankdir=LR\n" - "overlap=false\n" - "splines=true\n" - "node [shape=Mrecord colorscheme=paired12]\n" - "edge [arrowsize=0.5]\n" - ' "a" [label="a|a", style=filled fillcolor=1];\n' - ' "b" [label="b|b", style=filled fillcolor=2];\n' - ' "c" [label="c|c", style=filled fillcolor=3];\n' - ' "a" -> "b" [label="delete"]\n' - ' "a" -> "c" [label="delete"]\n' - ' "b" -> "c" [label="delete"]\n' - ' subgraph "acc1" {\n' - ' "a"\n' - ' "b"\n' - " }\n" - ' subgraph "acc2" {\n' - ' "c"\n' - " }\n" - "}\n" - ) - assert result == expected + result = "" + async for elem in respond_dot(Stream.iterate(nodes + edges)): + result += elem + "\n" + expected = ( + "digraph {\n" + "rankdir=LR\n" + "overlap=false\n" + "splines=true\n" + "node [shape=Mrecord colorscheme=paired12]\n" + "edge [arrowsize=0.5]\n" + ' "a" [label="a|a", style=filled fillcolor=1];\n' + ' "b" [label="b|b", style=filled fillcolor=2];\n' + ' "c" [label="c|c", style=filled fillcolor=3];\n' + ' "a" -> "b" [label="delete"]\n' + ' "a" -> "c" [label="delete"]\n' + ' "b" -> "c" [label="delete"]\n' + ' subgraph "acc1" {\n' + ' "a"\n' + ' "b"\n' + " }\n" + ' subgraph "acc2" {\n' + ' "c"\n' + " }\n" + "}\n" + ) + assert result == expected diff --git a/fixlib/fixlib/asynchronous/stream.py b/fixlib/fixlib/asynchronous/stream.py new file mode 100644 index 0000000000..a191b90305 --- /dev/null +++ b/fixlib/fixlib/asynchronous/stream.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import asyncio +from asyncio import TaskGroup, Task +from collections import deque +from typing import AsyncIterable, AsyncIterator, TypeVar, Optional, List, Dict, Callable, Generic, ParamSpec, TypeAlias +from typing import Iterable, Awaitable, Never, Tuple, Union + +T = TypeVar("T") +R = TypeVar("R", covariant=True) +P = ParamSpec("P") + +DirectOrAwaitable: TypeAlias = Union[T, Awaitable[T]] +IterOrAsyncIter: TypeAlias = Union[Iterable[T], AsyncIterable[T]] + + +def _async_iter(x: Iterable[T]) -> AsyncIterator[T]: + async def gen() -> AsyncIterator[T]: + for item in x: + yield item + + return gen() + + +def _to_async_iter(x: IterOrAsyncIter[T]) -> AsyncIterable[T]: + if isinstance(x, AsyncIterable): + return x + else: + return _async_iter(x) + + +def _flatmap( + source: AsyncIterable[IterOrAsyncIter[DirectOrAwaitable[T]]], + task_limit: Optional[int], + ordered: bool, +) -> AsyncIterator[T]: + if task_limit is None or task_limit == 1: + return _flatmap_direct(source) + elif ordered: + return _flatmap_ordered(source, task_limit) + else: + return _flatmap_unordered(source, task_limit) + + +async def _flatmap_direct( + source: AsyncIterable[IterOrAsyncIter[DirectOrAwaitable[T]]], +) -> AsyncIterator[T]: + async for sub_iter in source: + if isinstance(sub_iter, AsyncIterable): + async for item in sub_iter: + if isinstance(item, Awaitable): + item = await item + yield item + else: + for item in sub_iter: + if isinstance(item, Awaitable): + item = await item + yield item + + +async def _flatmap_unordered( + source: AsyncIterable[IterOrAsyncIter[DirectOrAwaitable[T]]], + task_limit: int, +) -> AsyncIterator[T]: + semaphore = asyncio.Semaphore(task_limit) + queue: asyncio.Queue[T | Exception] = asyncio.Queue() + tasks_in_flight = 0 + ingest_done = False + + async def worker(sub_iter: IterOrAsyncIter[DirectOrAwaitable[T]]) -> None: + nonlocal tasks_in_flight + try: + if isinstance(sub_iter, AsyncIterable): + async for si in sub_iter: + if isinstance(si, Awaitable): + si = await si + await queue.put(si) + else: + for si in sub_iter: + if isinstance(si, Awaitable): + si = await si + await queue.put(si) + except Exception as e: + await queue.put(e) # exception: put it in the queue to be handled + finally: + semaphore.release() + tasks_in_flight -= 1 + + async with TaskGroup() as tg: + + async def ingest_tasks() -> None: + nonlocal tasks_in_flight, ingest_done + # Start worker tasks + async for src in source: + await semaphore.acquire() + tg.create_task(worker(src)) + tasks_in_flight += 1 + ingest_done = True + + # Consume items from the queue and yield them + tg.create_task(ingest_tasks()) + while True: + if ingest_done and tasks_in_flight == 0 and queue.empty(): + break + try: + item = await queue.get() + if isinstance(item, Exception): + raise item + yield item + except asyncio.CancelledError: + break + + +async def _flatmap_ordered( + source: AsyncIterable[IterOrAsyncIter[DirectOrAwaitable[T]]], + task_limit: int, +) -> AsyncIterator[T]: + semaphore = asyncio.Semaphore(task_limit) + tasks: Dict[int, Task[None]] = {} + results: Dict[int, List[T] | Exception] = {} + next_index_to_yield = 0 + source_iter = aiter(source) + max_index_started = -1 # Highest index of tasks started + source_exhausted = False + + async def worker(sub_iter: IterOrAsyncIter[T | Awaitable[T]], index: int) -> None: + items = [] + try: + if isinstance(sub_iter, AsyncIterable): + async for item in sub_iter: + if isinstance(item, Awaitable): + item = await item + items.append(item) + else: + for item in sub_iter: + if isinstance(item, Awaitable): + item = await item + items.append(item) + results[index] = items + except Exception as e: + results[index] = e # Store exception to be raised later + finally: + semaphore.release() + + async with TaskGroup() as tg: + while True: + # Start new tasks up to task_limit ahead of next_index_to_yield + while (not source_exhausted) and (max_index_started - next_index_to_yield + 1) < task_limit: + try: + await semaphore.acquire() + si = await anext(source_iter) + max_index_started += 1 + tasks[max_index_started] = tg.create_task(worker(_to_async_iter(si), max_index_started)) + except StopAsyncIteration: + source_exhausted = True + break + + if next_index_to_yield in results: + result = results.pop(next_index_to_yield) + if isinstance(result, Exception): + raise result + else: + for res in result: + yield res + # Remove completed task + tasks.pop(next_index_to_yield, None) # noqa + next_index_to_yield += 1 + else: + # Wait for the next task to complete + if next_index_to_yield in tasks: + task = tasks[next_index_to_yield] + await asyncio.wait({task}) + elif not tasks and source_exhausted: + # No more tasks to process + break + else: + # Yield control to the event loop + await asyncio.sleep(0.01) + + +class Stream(Generic[T], AsyncIterator[T]): + def __init__(self, iterator: AsyncIterator[T]): + self.iterator = iterator + + def __aiter__(self) -> AsyncIterator[T]: + return self + + async def __anext__(self) -> T: + return await anext(self.iterator) + + def filter(self, fn: Callable[[T], DirectOrAwaitable[bool]]) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + async for item in self: + af = fn(item) + flag = await af if isinstance(af, Awaitable) else af + if flag: + yield item + + return Stream(gen()) + + def starmap( + self, + fn: Callable[..., DirectOrAwaitable[R]], + task_limit: Optional[int] = None, + ordered: bool = True, + ) -> Stream[R]: + return self.map(lambda args: fn(*args), task_limit, ordered) # type: ignore + + def map( + self, + fn: Callable[[T], DirectOrAwaitable[R]], + task_limit: Optional[int] = None, + ordered: bool = True, + ) -> Stream[R]: + async def gen() -> AsyncIterator[IterOrAsyncIter[DirectOrAwaitable[R]]]: + async for item in self: + res = fn(item) + yield [res] + + # in the case of a synchronous function, task_limit is ignored + task_limit = task_limit if asyncio.iscoroutinefunction(fn) else 1 + return Stream(_flatmap(gen(), task_limit, ordered)) + + def flatmap( + self, + fn: Callable[[T], DirectOrAwaitable[IterOrAsyncIter[DirectOrAwaitable[R]]]], + task_limit: Optional[int] = None, + ordered: bool = True, + ) -> Stream[R]: + async def gen() -> AsyncIterator[IterOrAsyncIter[DirectOrAwaitable[R]]]: + async for item in self: + res = fn(item) + if isinstance(res, Awaitable): + res = await res + yield res + + # in the case of a synchronous function, task_limit is ignored + task_limit = task_limit if asyncio.iscoroutinefunction(fn) else 1 + return Stream(_flatmap(gen(), task_limit, ordered)) + + def concat(self: Stream[Stream[T]], task_limit: Optional[int] = None, ordered: bool = True) -> Stream[T]: + return self.flatmap(lambda x: x, task_limit, ordered) + + def skip(self, num: int) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + count = 0 + async for item in self: + if count < num: + count += 1 + continue + yield item + + return Stream(gen()) + + def take(self, num: int) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + count = 0 + async for item in self: + if count >= num: + break + yield item + count += 1 + + return Stream(gen()) + + def take_last(self, num: int) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + queue: deque[T] = deque(maxlen=num) + async for item in self: + queue.append(item) + for item in queue: + yield item + + return Stream(gen()) + + def enumerate(self) -> Stream[Tuple[int, T]]: + async def gen() -> AsyncIterator[Tuple[int, T]]: + i = 0 + async for item in self: + yield i, item + i += 1 + + return Stream(gen()) + + def chunks(self, num: int) -> Stream[List[T]]: + async def gen() -> AsyncIterator[List[T]]: + while True: + chunk_items: List[T] = [] + try: + for _ in range(num): + item = await anext(self.iterator) + chunk_items.append(item) + yield chunk_items + except StopAsyncIteration: + if chunk_items: + yield chunk_items + break + + return Stream(gen()) + + def flatten(self) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + async for item in self: + if isinstance(item, AsyncIterator) or hasattr(item, "__aiter__"): + async for subitem in item: + yield subitem + elif isinstance(item, Iterable): + for subitem in item: + yield subitem + else: + yield item + + return Stream(gen()) + + async def collect(self) -> List[T]: + return [item async for item in self] + + @staticmethod + def just(x: T | Awaitable[T]) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + if isinstance(x, Awaitable): + yield await x + else: + yield x + + return Stream(gen()) + + @staticmethod + def iterate(x: Iterable[T] | AsyncIterable[T] | AsyncIterator[T]) -> Stream[T]: + if isinstance(x, AsyncIterator): + return Stream(x) + elif isinstance(x, AsyncIterable): + return Stream(aiter(x)) + else: + return Stream(_async_iter(x)) + + @staticmethod + def empty() -> Stream[T]: + async def empty() -> AsyncIterator[Never]: + if False: + yield # noqa + + return Stream(empty()) + + @staticmethod + def for_ever(fn: Callable[P, Awaitable[R]] | Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> Stream[T]: + async def gen() -> AsyncIterator[T]: + while True: + if asyncio.iscoroutinefunction(fn): + yield await fn(*args, **kwargs) + else: + yield fn(*args, **kwargs) # type: ignore + + return Stream(gen()) + + @staticmethod + def call(fn: Callable[P, Awaitable[R]] | Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> Stream[R]: + async def gen() -> AsyncIterator[R]: + if asyncio.iscoroutinefunction(fn): + yield await fn(*args, **kwargs) + else: + yield fn(*args, **kwargs) # type: ignore + + return Stream(gen()) + + @staticmethod + async def as_list(x: Iterable[T] | AsyncIterable[T] | AsyncIterator[T]) -> List[T]: + if isinstance(x, AsyncIterator): + return [item async for item in x] + elif isinstance(x, AsyncIterable): + return [item async for item in aiter(x)] + else: + return [item for item in x] diff --git a/fixlib/fixlib/baseresources.py b/fixlib/fixlib/baseresources.py index 0032907e1d..45ffb32d67 100644 --- a/fixlib/fixlib/baseresources.py +++ b/fixlib/fixlib/baseresources.py @@ -1089,6 +1089,16 @@ class BaseBucket(BaseResource): _metadata: ClassVar[Dict[str, Any]] = {"icon": "bucket", "group": "storage"} _categories: ClassVar[List[Category]] = [Category.storage] + encryption_enabled: Optional[bool] = None + versioning_enabled: Optional[bool] = None + + +@unique +class QueueType(Enum): + kind: ClassVar[str] = "queue_type" + STANDARD = "standard" + FIFO = "fifo" + @define(eq=False, slots=False) class BaseQueue(BaseResource): @@ -1097,6 +1107,9 @@ class BaseQueue(BaseResource): _kind_description: ClassVar[str] = "A storage queue." _metadata: ClassVar[Dict[str, Any]] = {"icon": "queue", "group": "storage"} _categories: ClassVar[List[Category]] = [Category.storage] + queue_type: Optional[QueueType] = None + approximate_message_count: Optional[int] = None + message_retention_period: Optional[int] = None @define(eq=False, slots=False) @@ -1125,6 +1138,8 @@ class BaseServerlessFunction(BaseResource): _metadata: ClassVar[Dict[str, Any]] = {"icon": "function", "group": "compute"} _categories: ClassVar[List[Category]] = [Category.compute] + memory_size: Optional[int] = None + @define(eq=False, slots=False) class BaseNetwork(BaseResource): @@ -1134,6 +1149,8 @@ class BaseNetwork(BaseResource): _metadata: ClassVar[Dict[str, Any]] = {"icon": "network", "group": "networking"} _categories: ClassVar[List[Category]] = [Category.networking] + cidr_blocks: List[str] = field(factory=list) + @define(eq=False, slots=False) class BaseNetworkQuota(BaseQuota): @@ -1215,6 +1232,8 @@ class BaseSubnet(BaseResource): _metadata: ClassVar[Dict[str, Any]] = {"icon": "subnet", "group": "networking"} _categories: ClassVar[List[Category]] = [Category.networking] + cidr_block: Optional[str] = None + @define(eq=False, slots=False) class BaseGateway(BaseResource): @@ -1374,8 +1393,8 @@ class BaseAccessKey(BaseResource): _kind_display: ClassVar[str] = "Access Key" _kind_description: ClassVar[str] = "An access key." _metadata: ClassVar[Dict[str, Any]] = {"icon": "key", "group": "access_control"} - access_key_status: str = "" _categories: ClassVar[List[Category]] = [Category.access_control, Category.security] + access_key_status: Optional[str] = None @define(eq=False, slots=False) @@ -1404,10 +1423,10 @@ class BaseStack(BaseResource): _kind_display: ClassVar[str] = "Stack" _kind_description: ClassVar[str] = "A stack." _metadata: ClassVar[Dict[str, Any]] = {"icon": "stack", "group": "management"} + _categories: ClassVar[List[Category]] = [Category.devops, Category.management] stack_status: str = "" stack_status_reason: str = "" stack_parameters: Dict[str, str] = field(factory=dict) - _categories: ClassVar[List[Category]] = [Category.devops, Category.management] @define(eq=False, slots=False) @@ -1453,6 +1472,7 @@ class BaseDNSZone(BaseResource): _kind_description: ClassVar[str] = "A DNS zone." _metadata: ClassVar[Dict[str, Any]] = {"icon": "dns", "group": "networking"} _categories: ClassVar[List[Category]] = [Category.dns, Category.networking] + private_zone: Optional[bool] = None @define(eq=False, slots=False) @@ -1574,6 +1594,19 @@ class BaseManagedKubernetesClusterProvider(BaseResource): endpoint: Optional[str] = field(default=None, metadata={"description": "The kubernetes API endpoint"}) +class AIJobStatus(Enum): + PENDING = "pending" + PREPARING = "preparing" + RUNNING = "running" + STOPPING = "stopping" + STOPPED = "stopped" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + PAUSED = "paused" + UNKNOWN = "unknown" + + @define(eq=False, slots=False) class BaseAIResource(BaseResource): kind: ClassVar[str] = "ai_resource" @@ -1590,6 +1623,8 @@ class BaseAIJob(BaseAIResource): _kind_description: ClassVar[str] = "An AI Job resource." _metadata: ClassVar[Dict[str, Any]] = {"icon": "job", "group": "ai"} + ai_job_status: Optional[AIJobStatus] = field(default=None, metadata={"description": "Current status of the AI job"}) + @define(eq=False, slots=False) class BaseAIModel(BaseAIResource): diff --git a/fixlib/fixlib/units.py b/fixlib/fixlib/units.py index 08fb20c4b2..2e6e2b21c2 100644 --- a/fixlib/fixlib/units.py +++ b/fixlib/fixlib/units.py @@ -11,10 +11,6 @@ reg.define("Ki = 1 KiB") reg.define("KB = 1000 B") -# globally define or register units - -bytes_u: pint.Quantity = reg.byte - def parse(s: str) -> pint.Quantity: return reg.parse_expression(s) diff --git a/fixlib/test/asynchronous/stream_test.py b/fixlib/test/asynchronous/stream_test.py new file mode 100644 index 0000000000..342f7ad834 --- /dev/null +++ b/fixlib/test/asynchronous/stream_test.py @@ -0,0 +1,123 @@ +import asyncio +from typing import AsyncIterator, Iterator + +from fixlib.asynchronous.stream import Stream + + +async def example_gen() -> AsyncIterator[int]: + for i in range(5, 0, -1): + yield i + + +def example_stream() -> Stream: + return Stream(example_gen()) + + +async def test_just() -> None: + assert await Stream.just(1).collect() == [1] + + +async def test_iterate() -> None: + assert await Stream.iterate([1, 2, 3]).collect() == [1, 2, 3] + assert await Stream.iterate(example_gen()).collect() == [5, 4, 3, 2, 1] + assert await Stream.iterate(example_stream()).collect() == [5, 4, 3, 2, 1] + + +async def test_filter() -> None: + assert await example_stream().filter(lambda x: x % 2).collect() == [5, 3, 1] + assert await example_stream().filter(lambda x: x is None).collect() == [] + assert await example_stream().filter(lambda x: True).collect() == [5, 4, 3, 2, 1] + + +async def test_map() -> None: + invoked = 0 + max_invoked = 0 + + def sync_fn(x: int) -> int: + return x * 2 + + async def async_fn(x: int) -> int: + await asyncio.sleep(x / 100) + return x * 2 + + async def count_invoked_fn(x: int) -> int: + nonlocal invoked, max_invoked + invoked += 1 + await asyncio.sleep(0.003) + max_invoked = max(max_invoked, invoked) + await asyncio.sleep(0.003) + invoked -= 1 + return x + + assert await example_stream().map(lambda x: x * 2).collect() == [10, 8, 6, 4, 2] + assert await example_stream().map(sync_fn).collect() == [10, 8, 6, 4, 2] + assert await example_stream().map(async_fn).collect() == [10, 8, 6, 4, 2] + # The function will wait depending on the streamed value. + # Since we start from biggest to smallest, the result should be reversed + # High chance of being flaky, since it relies on timing. + assert await example_stream().map(async_fn, task_limit=100, ordered=False).collect() == [2, 4, 6, 8, 10] + # All items are processed in parallel, while the order is preserved. + assert await example_stream().map(async_fn, task_limit=100, ordered=True).collect() == [10, 8, 6, 4, 2] + # Make sure all items are processed in parallel. + max_invoked = invoked = 0 + assert await example_stream().map(count_invoked_fn, task_limit=100, ordered=False).collect() + assert max_invoked == 5 + # Limit the number of parallel tasks to 2. + max_invoked = invoked = 0 + assert await example_stream().map(count_invoked_fn, task_limit=2, ordered=False).collect() + assert max_invoked == 2 + # Make sure all items are processed in parallel. + max_invoked = invoked = 0 + assert await example_stream().map(count_invoked_fn, task_limit=100, ordered=True).collect() + assert max_invoked == 5 + # Limit the number of parallel tasks to 2. + max_invoked = invoked = 0 + assert await example_stream().map(count_invoked_fn, task_limit=2, ordered=True).collect() + assert max_invoked == 2 + + +async def test_flatmap() -> None: + def sync_gen(x: int) -> Iterator[int]: + for i in range(2): + yield x * 2 + + async def async_gen(x: int) -> AsyncIterator[int]: + await asyncio.sleep(0) + for i in range(2): + yield x * 2 + + assert await example_stream().flatmap(sync_gen).collect() == [10, 10, 8, 8, 6, 6, 4, 4, 2, 2] + assert await example_stream().flatmap(async_gen).collect() == [10, 10, 8, 8, 6, 6, 4, 4, 2, 2] + assert await Stream.empty().flatmap(sync_gen).collect() == [] + assert await Stream.empty().flatmap(async_gen).collect() == [] + assert await Stream.iterate([]).flatmap(sync_gen).collect() == [] + assert await Stream.iterate([]).flatmap(async_gen).collect() == [] + + +async def test_take() -> None: + assert await example_stream().take(3).collect() == [5, 4, 3] + + +async def test_take_last() -> None: + assert await example_stream().take_last(3).collect() == [3, 2, 1] + + +async def test_skip() -> None: + assert await example_stream().skip(2).collect() == [3, 2, 1] + assert await example_stream().skip(10).collect() == [] + + +async def test_call() -> None: + def fn(foo: int, bla: str) -> int: + return 123 + + def with_int(foo: int) -> int: + return foo + 1 + + assert await Stream.call(fn, 1, "bla").map(with_int).collect() == [124] + + +async def test_chunks() -> None: + assert len([chunk async for chunk in example_stream().chunks(2)]) == 3 + assert [chunk async for chunk in example_stream().chunks(2)] == await example_stream().chunks(2).collect() + assert await example_stream().chunks(2).collect() == [[5, 4], [3, 2], [1]] diff --git a/fixshell/.pylintrc b/fixshell/.pylintrc index fd1655a604..b2bce42c1c 100644 --- a/fixshell/.pylintrc +++ b/fixshell/.pylintrc @@ -245,7 +245,7 @@ ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). -ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local, aiostream.pipe +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular diff --git a/plugins/aws/fix_plugin_aws/resource/acm.py b/plugins/aws/fix_plugin_aws/resource/acm.py index 8a12298791..9b7e4c0115 100644 --- a/plugins/aws/fix_plugin_aws/resource/acm.py +++ b/plugins/aws/fix_plugin_aws/resource/acm.py @@ -80,6 +80,7 @@ class AwsAcmCertificate(AwsResource, BaseCertificate): "tags": S("Tags", default=[]) >> ToDict(), "name": S("DomainName"), "ctime": S("CreatedAt"), + "mtime": S("RenewalSummary", "UpdatedAt"), "arn": S("CertificateArn"), "subject_alternative_names": S("SubjectAlternativeNames", default=[]), "domain_validation_options": S("DomainValidationOptions", default=[]) diff --git a/plugins/aws/fix_plugin_aws/resource/bedrock.py b/plugins/aws/fix_plugin_aws/resource/bedrock.py index d4aecec800..c8e5ba4fff 100644 --- a/plugins/aws/fix_plugin_aws/resource/bedrock.py +++ b/plugins/aws/fix_plugin_aws/resource/bedrock.py @@ -12,9 +12,9 @@ from fix_plugin_aws.resource.lambda_ import AwsLambdaFunction from fix_plugin_aws.resource.s3 import AwsS3Bucket from fix_plugin_aws.resource.rds import AwsRdsCluster, AwsRdsInstance -from fixlib.baseresources import BaseAIJob, ModelReference, BaseAIModel +from fixlib.baseresources import AIJobStatus, BaseAIJob, ModelReference, BaseAIModel from fixlib.graph import Graph -from fixlib.json_bender import Bender, S, ForallBend, Bend, Sort +from fixlib.json_bender import Bender, S, ForallBend, Bend, MapEnum, Sort from fixlib.types import Json log = logging.getLogger("fix.plugins.aws") @@ -82,6 +82,16 @@ def service_name(cls) -> str: return service_name +AWS_BEDROCK_JOB_STATUS_MAPPING = { + "InProgress": AIJobStatus.RUNNING, + "Completed": AIJobStatus.COMPLETED, + "Failed": AIJobStatus.FAILED, + "Stopping": AIJobStatus.STOPPING, + "Stopped": AIJobStatus.STOPPED, + "Deleting": AIJobStatus.STOPPING, +} + + @define(eq=False, slots=False) class AwsBedrockFoundationModel(BaseAIModel, AwsResource): kind: ClassVar[str] = "aws_bedrock_foundation_model" @@ -553,7 +563,7 @@ class AwsBedrockModelCustomizationJob(BedrockTaggable, BaseAIJob, AwsResource): "output_model_arn": S("outputModelArn"), "client_request_token": S("clientRequestToken"), "role_arn": S("roleArn"), - "status": S("status"), + "status": S("status") >> MapEnum(AWS_BEDROCK_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "failure_message": S("failureMessage"), "creation_time": S("creationTime"), "last_modified_time": S("lastModifiedTime"), @@ -575,7 +585,6 @@ class AwsBedrockModelCustomizationJob(BedrockTaggable, BaseAIJob, AwsResource): output_model_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the output model."}) # fmt: skip client_request_token: Optional[str] = field(default=None, metadata={"description": "The token that you specified in the CreateCustomizationJob request."}) # fmt: skip role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM role."}) # fmt: skip - status: Optional[str] = field(default=None, metadata={"description": "The status of the job. A successful job transitions from in-progress to completed when the output model is ready to use. If the job failed, the failure message contains information about why the job failed."}) # fmt: skip failure_message: Optional[str] = field(default=None, metadata={"description": "Information about why the job failed."}) # fmt: skip creation_time: Optional[datetime] = field(default=None, metadata={"description": "Time that the resource was created."}) # fmt: skip last_modified_time: Optional[datetime] = field(default=None, metadata={"description": "Time that the resource was last modified."}) # fmt: skip @@ -777,7 +786,7 @@ class AwsBedrockEvaluationJob(BedrockTaggable, BaseAIJob, AwsResource): "ctime": S("creationTime"), "mtime": S("lastModifiedTime"), "job_name": S("jobName"), - "status": S("status"), + "status": S("status") >> MapEnum(AWS_BEDROCK_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "job_arn": S("jobArn"), "job_description": S("jobDescription"), "role_arn": S("roleArn"), @@ -791,7 +800,6 @@ class AwsBedrockEvaluationJob(BedrockTaggable, BaseAIJob, AwsResource): "failure_messages": S("failureMessages", default=[]), } job_name: Optional[str] = field(default=None, metadata={"description": "The name of the model evaluation job."}) # fmt: skip - status: Optional[str] = field(default=None, metadata={"description": "The status of the model evaluation job."}) # fmt: skip job_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the model evaluation job."}) # fmt: skip job_description: Optional[str] = field(default=None, metadata={"description": "The description of the model evaluation job."}) # fmt: skip role_arn: Optional[str] = field(default=None, metadata={"description": "The Amazon Resource Name (ARN) of the IAM service role used in the model evaluation job."}) # fmt: skip diff --git a/plugins/aws/fix_plugin_aws/resource/cognito.py b/plugins/aws/fix_plugin_aws/resource/cognito.py index c2cdd3e0ae..ebc39fbdbd 100644 --- a/plugins/aws/fix_plugin_aws/resource/cognito.py +++ b/plugins/aws/fix_plugin_aws/resource/cognito.py @@ -113,6 +113,7 @@ class AwsCognitoUser(AwsResource, BaseUser): "enabled": S("Enabled"), "user_status": S("UserStatus"), "mfa_options": S("MFAOptions", default=[]) >> ForallBend(AwsCognitoMFAOptionType.mapping), + "username": S("Username"), } user_attributes: List[AwsCognitoAttributeType] = field(factory=list) enabled: Optional[bool] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/dynamodb.py b/plugins/aws/fix_plugin_aws/resource/dynamodb.py index c068aaf921..92199b8c2b 100644 --- a/plugins/aws/fix_plugin_aws/resource/dynamodb.py +++ b/plugins/aws/fix_plugin_aws/resource/dynamodb.py @@ -8,9 +8,16 @@ from fix_plugin_aws.resource.kinesis import AwsKinesisStream from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import HasResourcePolicy, ModelReference, PolicySource, PolicySourceKind +from fixlib.baseresources import ( + BaseDatabase, + DatabaseInstanceStatus, + HasResourcePolicy, + ModelReference, + PolicySource, + PolicySourceKind, +) from fixlib.graph import Graph -from fixlib.json_bender import S, Bend, Bender, ForallBend, bend +from fixlib.json_bender import S, Bend, Bender, ForallBend, bend, K, MapValue from fixlib.types import Json from fixlib.json import sort_json @@ -356,7 +363,7 @@ class AwsDynamoDbContinuousBackup: @define(eq=False, slots=False) -class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): +class AwsDynamoDbTable(DynamoDbTaggable, BaseDatabase, AwsResource, HasResourcePolicy): kind: ClassVar[str] = "aws_dynamodb_table" _kind_display: ClassVar[str] = "AWS DynamoDB Table" _kind_description: ClassVar[str] = "AWS DynamoDB Table is a fully managed NoSQL database service that stores and retrieves data. It supports key-value and document data models, offering automatic scaling and low-latency performance. DynamoDB Tables handle data storage, indexing, and querying, providing consistent read and write throughput. They offer data encryption, backup, and recovery features for secure and reliable data management." # fmt: skip @@ -396,6 +403,25 @@ class AwsDynamoDbTable(DynamoDbTaggable, AwsResource, HasResourcePolicy): "dynamodb_sse_description": S("SSEDescription") >> Bend(AwsDynamoDbSSEDescription.mapping), "dynamodb_archival_summary": S("ArchivalSummary") >> Bend(AwsDynamoDbArchivalSummary.mapping), "dynamodb_table_class_summary": S("TableClassSummary") >> Bend(AwsDynamoDbTableClassSummary.mapping), + "db_type": K("dynamodb"), + "db_status": S("TableStatus") + >> MapValue( + { + "CREATING": DatabaseInstanceStatus.BUSY, + "UPDATING": DatabaseInstanceStatus.BUSY, + "DELETING": DatabaseInstanceStatus.BUSY, + "ACTIVE": DatabaseInstanceStatus.AVAILABLE, + "INACCESSIBLE_ENCRYPTION_CREDENTIALS": DatabaseInstanceStatus.FAILED, + "ARCHIVING": DatabaseInstanceStatus.BUSY, + "ARCHIVED": DatabaseInstanceStatus.STOPPED, + }, + default=DatabaseInstanceStatus.UNKNOWN, + ), + "volume_encrypted": S("SSEDescription", "Status") + >> MapValue( + {"ENABLING": True, "ENABLED": True, "DISABLING": False, "DISABLED": False, "UPDATING": None}, + default=None, + ), } arn: Optional[str] = field(default=None) dynamodb_attribute_definitions: List[AwsDynamoDbAttributeDefinition] = field(factory=list) diff --git a/plugins/aws/fix_plugin_aws/resource/ec2.py b/plugins/aws/fix_plugin_aws/resource/ec2.py index 1a2209dceb..2a21b2f097 100644 --- a/plugins/aws/fix_plugin_aws/resource/ec2.py +++ b/plugins/aws/fix_plugin_aws/resource/ec2.py @@ -23,7 +23,6 @@ from fix_plugin_aws.resource.kms import AwsKmsKey from fix_plugin_aws.resource.s3 import AwsS3Bucket from fix_plugin_aws.utils import ToDict, TagsValue -from fix_plugin_aws.aws_client import AwsClient from fixlib.baseresources import ( BaseInstance, BaseKeyPair, @@ -2155,6 +2154,7 @@ class AwsEc2Vpc(EC2Taggable, AwsResource, BaseNetwork): "vpc_cidr_block_association_set": S("CidrBlockAssociationSet", default=[]) >> ForallBend(AwsEc2VpcCidrBlockAssociation.mapping), "vpc_is_default": S("IsDefault"), + "cidr_blocks": S("CidrBlockAssociationSet", default=[]) >> ForallBend(S("CidrBlock")), } vpc_cidr_block: Optional[str] = field(default=None) vpc_dhcp_options_id: Optional[str] = field(default=None) @@ -2506,6 +2506,7 @@ class AwsEc2Subnet(EC2Taggable, AwsResource, BaseSubnet): "subnet_ipv6_native": S("Ipv6Native"), "subnet_private_dns_name_options_on_launch": S("PrivateDnsNameOptionsOnLaunch") >> Bend(AwsEc2PrivateDnsNameOptionsOnLaunch.mapping), + "cidr_block": S("CidrBlock"), } subnet_availability_zone: Optional[str] = field(default=None) subnet_availability_zone_id: Optional[str] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/iam.py b/plugins/aws/fix_plugin_aws/resource/iam.py index bdf6b15a39..ced7704b64 100644 --- a/plugins/aws/fix_plugin_aws/resource/iam.py +++ b/plugins/aws/fix_plugin_aws/resource/iam.py @@ -654,6 +654,7 @@ class AwsIamUser(AwsResource, BaseUser, BaseIamPrincipal): "arn": S("Arn"), "user_policies": S("UserPolicyList", default=[]) >> ForallBend(AwsIamPolicyDetail.mapping), "user_permissions_boundary": S("PermissionsBoundary") >> Bend(AwsIamAttachedPermissionsBoundary.mapping), + "username": S("UserName"), } path: Optional[str] = field(default=None) user_policies: List[AwsIamPolicyDetail] = field(factory=list) diff --git a/plugins/aws/fix_plugin_aws/resource/lambda_.py b/plugins/aws/fix_plugin_aws/resource/lambda_.py index 599bef09ce..b5476fbe57 100644 --- a/plugins/aws/fix_plugin_aws/resource/lambda_.py +++ b/plugins/aws/fix_plugin_aws/resource/lambda_.py @@ -251,6 +251,7 @@ class AwsLambdaFunction(AwsResource, BaseServerlessFunction, HasResourcePolicy): "function_signing_job_arn": S("SigningJobArn"), "function_architectures": S("Architectures", default=[]), "function_ephemeral_storage": S("EphemeralStorage", "Size"), + "memory_size": S("MemorySize"), } function_runtime: Optional[str] = field(default=None) function_role: Optional[str] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/route53.py b/plugins/aws/fix_plugin_aws/resource/route53.py index c92f2c4659..9cf65df94e 100644 --- a/plugins/aws/fix_plugin_aws/resource/route53.py +++ b/plugins/aws/fix_plugin_aws/resource/route53.py @@ -81,12 +81,13 @@ class AwsRoute53Zone(AwsResource, BaseDNSZone): "name": S("Name"), "zone_caller_reference": S("CallerReference"), "zone_config": S("Config") >> Bend(AwsRoute53ZoneConfig.mapping), - "zone_resource_record_set_count": S("ResourceRecordSetCount"), "zone_linked_service": S("LinkedService") >> Bend(AwsRoute53LinkedService.mapping), + "private_zone": S("Config", "PrivateZone"), + "zone_resource_record_set_count": S("ResourceRecordSetCount"), } + zone_resource_record_set_count: Optional[int] = field(default=None, metadata=dict(ignore_history=True)) zone_caller_reference: Optional[str] = field(default=None) zone_config: Optional[AwsRoute53ZoneConfig] = field(default=None) - zone_resource_record_set_count: Optional[int] = field(default=None, metadata=dict(ignore_history=True)) zone_linked_service: Optional[AwsRoute53LinkedService] = field(default=None) zone_logging_config: Optional[AwsRoute53LoggingConfig] = field(default=None) diff --git a/plugins/aws/fix_plugin_aws/resource/s3.py b/plugins/aws/fix_plugin_aws/resource/s3.py index bb52c3b8f9..4a22979eaf 100644 --- a/plugins/aws/fix_plugin_aws/resource/s3.py +++ b/plugins/aws/fix_plugin_aws/resource/s3.py @@ -1,3 +1,4 @@ +from functools import partial import logging from collections import defaultdict from datetime import timedelta @@ -234,6 +235,7 @@ def add_bucket_encryption(bck: AwsS3Bucket) -> None: mapped = bend(AwsS3ServerSideEncryptionRule.mapping, raw) if rule := parse_json(mapped, AwsS3ServerSideEncryptionRule, builder): bck.bucket_encryption_rules.append(rule) + bck.encryption_enabled = len(bck.bucket_encryption_rules) > 0 def add_bucket_policy(bck: AwsS3Bucket) -> None: with builder.suppress(f"{service_name}.get-bucket-policy"): @@ -266,9 +268,11 @@ def add_bucket_versioning(bck: AwsS3Bucket) -> None: ): bck.bucket_versioning = raw_versioning.get("Status") == "Enabled" bck.bucket_mfa_delete = raw_versioning.get("MFADelete") == "Enabled" + bck.versioning_enabled = bck.bucket_versioning else: bck.bucket_versioning = False bck.bucket_mfa_delete = False + bck.versioning_enabled = False def add_public_access(bck: AwsS3Bucket) -> None: with builder.suppress(f"{service_name}.get-public-access-block"): @@ -363,9 +367,22 @@ def _get_tags(self, client: AwsClient) -> Dict[str, str]: return tags_as_dict(tag_list) # type: ignore def collect_usage_metrics(self, builder: GraphBuilder) -> List[AwsCloudwatchQuery]: + def _calculate_total_size(bucket_instance: AwsS3Bucket) -> None: + # Calculate the total bucket size for each bucket by summing up the sizes of all storage types + bucket_size: Dict[str, float] = defaultdict(float) + for metric_name, metric_values in bucket_instance._resource_usage.items(): + if metric_name.endswith("_bucket_size_bytes"): + for name, value in metric_values.items(): + bucket_size[name] += value + if bucket_size: + bucket_instance._resource_usage["bucket_size_bytes"] = dict(bucket_size) + # Filter out metrics with the 'aws-controltower' dimension value if "aws-controltower" in self.safe_name: return [] + + # calculate all bucket sizes after usage metrics collection + builder.after_collect_actions.append(partial(_calculate_total_size, self)) storage_types = { "StandardStorage": "standard_storage", "IntelligentTieringStorage": "intelligent_tiering_storage", @@ -415,16 +432,6 @@ def collect_usage_metrics(self, builder: GraphBuilder) -> List[AwsCloudwatchQuer ) return queries - def complete_graph(self, builder: GraphBuilder, source: Json) -> None: - # Calculate the total bucket size for each bucket by summing up the sizes of all storage types - bucket_size: Dict[str, float] = defaultdict(float) - for metric_name, metric_values in self._resource_usage.items(): - if metric_name.endswith("_bucket_size_bytes"): - for name, value in metric_values.items(): - bucket_size[name] += value - if bucket_size: - self._resource_usage["bucket_size_bytes"] = dict(bucket_size) - def update_resource_tag(self, client: AwsClient, key: str, value: str) -> bool: tags = self._get_tags(client) tags[key] = value diff --git a/plugins/aws/fix_plugin_aws/resource/sqs.py b/plugins/aws/fix_plugin_aws/resource/sqs.py index ebaa8923be..dd4b87a2e0 100644 --- a/plugins/aws/fix_plugin_aws/resource/sqs.py +++ b/plugins/aws/fix_plugin_aws/resource/sqs.py @@ -15,6 +15,7 @@ ModelReference, PolicySource, PolicySourceKind, + QueueType, ) from fixlib.graph import Graph from fixlib.json_bender import F, Bender, S, AsInt, AsBool, Bend, ParseJson, Sorted @@ -80,6 +81,8 @@ class AwsSqsQueue(AwsResource, BaseQueue, HasResourcePolicy): "sqs_delay_seconds": S("DelaySeconds") >> AsInt(), "sqs_receive_message_wait_time_seconds": S("ReceiveMessageWaitTimeSeconds") >> AsInt(), "sqs_managed_sse_enabled": S("SqsManagedSseEnabled") >> AsBool(), + "message_retention_period": S("MessageRetentionPeriod") >> AsInt(), + "approximate_message_count": S("ApproximateNumberOfMessages") >> AsInt(), } sqs_queue_url: Optional[str] = field(default=None) sqs_approximate_number_of_messages: Optional[int] = field(default=None, metadata=dict(ignore_history=True)) @@ -118,7 +121,7 @@ def called_collect_apis(cls) -> List[AwsApiSpec]: ] @classmethod - def collect(cls: Type[AwsResource], json: List[Json], builder: GraphBuilder) -> None: + def collect(cls, json: List[Json], builder: GraphBuilder) -> None: def add_instance(queue_url: str) -> None: queue_attributes = builder.client.get( service_name, "get-queue-attributes", "Attributes", QueueUrl=queue_url, AttributeNames=["All"] @@ -126,8 +129,9 @@ def add_instance(queue_url: str) -> None: if queue_attributes is not None: queue_attributes["QueueUrl"] = queue_url queue_attributes["QueueName"] = queue_url.rsplit("/", 1)[-1] - if instance := cls.from_api(queue_attributes, builder): + if instance := AwsSqsQueue.from_api(queue_attributes, builder): builder.add_node(instance, queue_attributes) + instance.queue_type = QueueType.FIFO if instance.sqs_fifo_queue else QueueType.STANDARD builder.submit_work(service_name, add_tags, instance) def add_tags(queue: AwsSqsQueue) -> None: diff --git a/plugins/aws/fix_plugin_aws/resource/ssm.py b/plugins/aws/fix_plugin_aws/resource/ssm.py index dd7fcf8498..38d7446545 100644 --- a/plugins/aws/fix_plugin_aws/resource/ssm.py +++ b/plugins/aws/fix_plugin_aws/resource/ssm.py @@ -1,4 +1,5 @@ -import json +from functools import partial +from json import loads as json_loads import logging from datetime import datetime from typing import ClassVar, Dict, Optional, List, Type, Any @@ -12,9 +13,10 @@ from fix_plugin_aws.resource.ec2 import AwsEc2Instance from fix_plugin_aws.resource.s3 import AwsS3Bucket from fix_plugin_aws.utils import ToDict -from fixlib.baseresources import ModelReference -from fixlib.json_bender import Bender, S, Bend, AsDateString, ForallBend, K +from fixlib.baseresources import SEVERITY_MAPPING, Finding, ModelReference, PhantomBaseResource, Severity +from fixlib.json_bender import Bender, S, Bend, AsDateString, ForallBend from fixlib.types import Json +from fixlib.utils import chunks log = logging.getLogger("fix.plugins.aws") service_name = "ssm" @@ -239,7 +241,7 @@ def collect_document(name: str) -> None: and (instance := cls.from_api(js, builder)) ): if content_format == "JSON": - instance.content = json.loads(content) + instance.content = json_loads(content) elif content_format == "YAML": instance.content = yaml.safe_load(content) else: @@ -341,7 +343,7 @@ class AwsSSMNonCompliantSummary: severity_summary: Optional[AwsSSMSeveritySummary] = field(default=None, metadata={"description": "A summary of the non-compliance severity by compliance type"}) # fmt: skip -ResourceTypeLookup = { +ResourceTypeLookup: Dict[str, Type[AwsResource]] = { "ManagedInstance": AwsEc2Instance, "AWS::EC2::Instance": AwsEc2Instance, "AWS::DynamoDB::Table": AwsDynamoDbTable, @@ -351,45 +353,88 @@ class AwsSSMNonCompliantSummary: @define(eq=False, slots=False) -class AwsSSMResourceCompliance(AwsResource): +class AwsSSMResourceCompliance(AwsResource, PhantomBaseResource): kind: ClassVar[str] = "aws_ssm_resource_compliance" - _kind_display: ClassVar[str] = "AWS SSM Resource Compliance" - _kind_description: ClassVar[str] = "AWS SSM Resource Compliance is a feature within AWS Systems Manager that evaluates and reports on the compliance status of AWS resources. It checks resources against predefined or custom rules, identifying non-compliant configurations and security issues. Users can view compliance data, generate reports, and take corrective actions to maintain resource adherence to organizational standards and best practices." # fmt: skip - _docs_url: ClassVar[str] = ( - "https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-compliance-about.html" - ) - _kind_service: ClassVar[Optional[str]] = service_name - _metadata: ClassVar[Dict[str, Any]] = {"icon": "resource", "group": "management"} - _aws_metadata: ClassVar[Dict[str, Any]] = {"arn_tpl": "arn:{partition}:ssm:{region}:{account}:resource-compliance/{id}"} # fmt: skip + _model_export: ClassVar[bool] = False # do not export this class, since there will be no instances of it api_spec: ClassVar[AwsApiSpec] = AwsApiSpec( - "ssm", "list-resource-compliance-summaries", "ResourceComplianceSummaryItems" + "ssm", + "list-resource-compliance-summaries", + "ResourceComplianceSummaryItems", + {"Filters": [{"Key": "Status", "Values": ["NON_COMPLIANT"], "Type": "EQUAL"}]}, ) - _reference_kinds: ClassVar[ModelReference] = { - "successors": {"default": ["aws_ec2_instance", "aws_dynamodb_table", "aws_s3_bucket", "aws_ssm_document"]} - } mapping: ClassVar[Dict[str, Bender]] = { - "id": S("ComplianceType") + K("_") + S("ResourceType") + K("_") + S("ResourceId"), + "id": S("Id"), + "name": S("title"), "compliance_type": S("ComplianceType"), "resource_type": S("ResourceType"), "resource_id": S("ResourceId"), + "title": S("Title"), "status": S("Status"), - "overall_severity": S("OverallSeverity"), + "severity": S("Severity"), "execution_summary": S("ExecutionSummary") >> Bend(AwsSSMComplianceExecutionSummary.mapping), - "compliant_summary": S("CompliantSummary") >> Bend(AwsSSMCompliantSummary.mapping), - "non_compliant_summary": S("NonCompliantSummary") >> Bend(AwsSSMNonCompliantSummary.mapping), + "compliance_details": S("Details"), } - compliance_type: Optional[str] = field(default=None, metadata={"description": "The compliance type."}) # fmt: skip - resource_type: Optional[str] = field(default=None, metadata={"description": "The resource type."}) # fmt: skip - resource_id: Optional[str] = field(default=None, metadata={"description": "The resource ID."}) # fmt: skip - status: Optional[str] = field(default=None, metadata={"description": "The compliance status for the resource."}) # fmt: skip - overall_severity: Optional[str] = field(default=None, metadata={"description": "The highest severity item found for the resource. The resource is compliant for this item."}) # fmt: skip - execution_summary: Optional[AwsSSMComplianceExecutionSummary] = field(default=None, metadata={"ignore_history": True, "description": "Information about the execution."}) # fmt: skip - compliant_summary: Optional[AwsSSMCompliantSummary] = field(default=None, metadata={"description": "A list of items that are compliant for the resource."}) # fmt: skip - non_compliant_summary: Optional[AwsSSMNonCompliantSummary] = field(default=None, metadata={"description": "A list of items that aren't compliant for the resource."}) # fmt: skip + compliance_type: Optional[str] = field(default=None, metadata={"description": "The compliance type. For example, Association (for a State Manager association), Patch, or Custom:string are all valid compliance types."}) # fmt: skip + resource_type: Optional[str] = field(default=None, metadata={"description": "The type of resource. ManagedInstance is currently the only supported resource type."}) # fmt: skip + resource_id: Optional[str] = field(default=None, metadata={"description": "An ID for the resource. For a managed node, this is the node ID."}) # fmt: skip + title: Optional[str] = field(default=None, metadata={"description": "A title for the compliance item. For example, if the compliance item is a Windows patch, the title could be the title of the KB article for the patch; for example: Security Update for Active Directory Federation Services."}) # fmt: skip + status: Optional[str] = field(default=None, metadata={"description": "The status of the compliance item. An item is either COMPLIANT, NON_COMPLIANT, or an empty string (for Windows patches that aren't applicable)."}) # fmt: skip + severity: Optional[str] = field(default=None, metadata={"description": "The severity of the compliance status. Severity can be one of the following: Critical, High, Medium, Low, Informational, Unspecified."}) # fmt: skip + execution_summary: Optional[AwsSSMComplianceExecutionSummary] = field(default=None, metadata={"description": "A summary for the compliance item. The summary includes an execution ID, the execution type (for example, command), and the execution time."}) # fmt: skip + compliance_details: Optional[Dict[str, str]] = field(default=None, metadata={"description": "A Key:Value tag combination for the compliance item."}) # fmt: skip + + def parse_finding(self) -> Finding: + title = self.title or "" + severity = SEVERITY_MAPPING.get(self.severity or "", Severity.medium) + details = self.compliance_details + if self.execution_summary: + updated_at = self.execution_summary.execution_time + else: + updated_at = None + return Finding(title, severity, None, None, updated_at, details) - def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: - if (rt := self.resource_type) and (rid := self.resource_id) and (clazz := ResourceTypeLookup.get(rt)): - builder.add_edge(self, clazz=clazz, id=rid) + @classmethod + def collect(cls, json: List[Json], builder: GraphBuilder) -> None: + def add_finding( + provider: str, finding: Finding, clazz: Optional[Type[AwsResource]] = None, **node: Any + ) -> None: + if resource := builder.node(clazz=clazz, **node): + resource.add_finding(provider, finding) + + def collect_compliance_items(jsons: List[Json]) -> None: + spec = AwsApiSpec("ssm", "list-compliance-items", "ComplianceItems") + compliance_ids = [item["ResourceId"] for item in jsons] + for result in builder.client.list( + aws_service="ssm", + action=spec.api_action, + result_name=spec.result_property, + expected_errors=spec.expected_errors, + ResourceIds=compliance_ids, + ): + if finding := AwsSSMResourceCompliance.from_api(result, builder): + if ( + (rt := finding.resource_type) + and (rid := finding.resource_id) + and (clazz := ResourceTypeLookup.get(rt)) + ): + # append the finding when all resources have been collected + builder.after_collect_actions.append( + partial( + add_finding, + "amazon_ssm_compliance", + finding.parse_finding(), + clazz, + id=rid, + ) + ) + + # we can request only 40 items per request + for jsons in chunks(json, 39): + builder.submit_work("ssm", collect_compliance_items, jsons) + + @classmethod + def called_collect_apis(cls) -> List[AwsApiSpec]: + return [cls.api_spec, AwsApiSpec("ssm", "list-compliance-items", "ComplianceItems")] resources: List[Type[AwsResource]] = [AwsSSMInstance, AwsSSMDocument, AwsSSMResourceCompliance] diff --git a/plugins/aws/test/collector_test.py b/plugins/aws/test/collector_test.py index 9de124602e..76497e5a9f 100644 --- a/plugins/aws/test/collector_test.py +++ b/plugins/aws/test/collector_test.py @@ -37,8 +37,8 @@ def count_kind(clazz: Type[AwsResource]) -> int: # make sure all threads have been joined assert len(threading.enumerate()) == 1 # ensure the correct number of nodes and edges - assert count_kind(AwsResource) == 260 - assert len(account_collector.graph.edges) == 574 + assert count_kind(AwsResource) == 257 + assert len(account_collector.graph.edges) == 571 assert len(account_collector.graph.deferred_edges) == 2 for node in account_collector.graph.nodes: if isinstance(node, AwsRegion): diff --git a/plugins/aws/test/resources/cloudfront_test.py b/plugins/aws/test/resources/cloudfront_test.py index 8dfb163dde..fffda5a6e4 100644 --- a/plugins/aws/test/resources/cloudfront_test.py +++ b/plugins/aws/test/resources/cloudfront_test.py @@ -46,14 +46,14 @@ def validate_delete_args(**kwargs: Any) -> Any: def test_functions() -> None: - first, builder = round_trip_for(AwsCloudFrontFunction) + first, builder = round_trip_for(AwsCloudFrontFunction, "memory_size") assert len(builder.resources_of(AwsCloudFrontFunction)) == 1 assert len(first.tags) == 1 assert first.arn == "arn" def test_function_deletion() -> None: - func, _ = round_trip_for(AwsCloudFrontFunction) + func, _ = round_trip_for(AwsCloudFrontFunction, "memory_size") def validate_delete_args(**kwargs: Any) -> Any: assert kwargs["action"] == "delete-function" diff --git a/plugins/aws/test/resources/dynamodb_test.py b/plugins/aws/test/resources/dynamodb_test.py index ea0b22eb86..3f39f4cc97 100644 --- a/plugins/aws/test/resources/dynamodb_test.py +++ b/plugins/aws/test/resources/dynamodb_test.py @@ -7,13 +7,17 @@ def test_tables() -> None: - first, builder = round_trip_for(AwsDynamoDbTable, "dynamodb_policy") + first, builder = round_trip_for( + AwsDynamoDbTable, "dynamodb_policy", "db_version", "db_publicly_accessible", "volume_size", "volume_iops" + ) assert len(builder.resources_of(AwsDynamoDbTable)) == 1 assert len(first.tags) == 1 def test_tagging_tables() -> None: - table, _ = round_trip_for(AwsDynamoDbTable, "dynamodb_policy") + table, _ = round_trip_for( + AwsDynamoDbTable, "dynamodb_policy", "db_version", "db_publicly_accessible", "volume_size", "volume_iops" + ) def validate_update_args(**kwargs: Any) -> Any: if kwargs["action"] == "list-tags-of-resource": @@ -37,7 +41,9 @@ def validate_delete_args(**kwargs: Any) -> Any: def test_delete_tables() -> None: - table, _ = round_trip_for(AwsDynamoDbTable, "dynamodb_policy") + table, _ = round_trip_for( + AwsDynamoDbTable, "dynamodb_policy", "db_version", "db_publicly_accessible", "volume_size", "volume_iops" + ) def validate_delete_args(**kwargs: Any) -> Any: assert kwargs["action"] == "delete-table" diff --git a/plugins/aws/test/resources/files/cloudwatch/get-metric-data__2024_05_01_12_00_00_00_00_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_a.json b/plugins/aws/test/resources/files/cloudwatch/get-metric-data__2020_05_30_17_45_30_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_aws_s3_.json similarity index 60% rename from plugins/aws/test/resources/files/cloudwatch/get-metric-data__2024_05_01_12_00_00_00_00_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_a.json rename to plugins/aws/test/resources/files/cloudwatch/get-metric-data__2020_05_30_17_45_30_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_aws_s3_.json index 11273b8167..956409f9a4 100644 --- a/plugins/aws/test/resources/files/cloudwatch/get-metric-data__2024_05_01_12_00_00_00_00_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_a.json +++ b/plugins/aws/test/resources/files/cloudwatch/get-metric-data__2020_05_30_17_45_30_numberofobjects_aws_s3_bucketname_bucket_1_storagetype_allstoragetypes_average_AWS_S3_NumberOfObjects_BucketName_bucket_1_StorageType_AllStorageTypes_86400_Average_Count_True_bucketsizebytes_aws_s3_.json @@ -3,24 +3,36 @@ { "Id": "bucketsizebytes_aws_s3_bucketname_bucket_1_storagetype_standardstorage_average", "Label": "BucketSizeBytes", - "Timestamps": [ "2024-04-30T12:50:00+00:00" ], - "Values": [ 1 ], + "Timestamps": [ + "2024-04-30T12:50:00+00:00" + ], + "Values": [ + 1 + ], "StatusCode": "Complete" }, { "Id": "bucketsizebytes_aws_s3_bucketname_bucket_1_storagetype_intelligenttieringstorage_average", "Label": "BucketSizeBytes", - "Timestamps": [ "2024-04-30T12:50:00+00:00" ], - "Values": [ 2 ], + "Timestamps": [ + "2024-04-30T12:50:00+00:00" + ], + "Values": [ + 2 + ], "StatusCode": "Complete" }, { "Id": "bucketsizebytes_aws_s3_bucketname_bucket_1_storagetype_standardiastorage_average", "Label": "BucketSizeBytes", - "Timestamps": [ "2024-04-30T12:50:00+00:00" ], - "Values": [ 3 ], + "Timestamps": [ + "2024-04-30T12:50:00+00:00" + ], + "Values": [ + 3 + ], "StatusCode": "Complete" } ], "Messages": [] -} +} \ No newline at end of file diff --git a/plugins/aws/test/resources/files/ssm/list-compliance-items__foo_foo_foo.json b/plugins/aws/test/resources/files/ssm/list-compliance-items__foo_foo_foo.json new file mode 100644 index 0000000000..d35fbf6047 --- /dev/null +++ b/plugins/aws/test/resources/files/ssm/list-compliance-items__foo_foo_foo.json @@ -0,0 +1,23 @@ +{ + "ComplianceItems": [ + { + "ComplianceType": "Association", + "ResourceType": "ManagedInstance", + "ResourceId": "i-1", + "Id": "SSM-Association-1", + "Title": "State Manager Association Compliance", + "Status": "NON_COMPLIANT", + "Severity": "HIGH", + "ExecutionSummary": { + "ExecutionTime": "2024-10-01T12:34:56Z", + "ExecutionId": "xyz5678-execution-id", + "ExecutionType": "Association" + }, + "Details": { + "LastExecutionStatus": "Failed", + "ErrorDetails": "Failed to apply association" + } + } + ], + "NextToken": "next-token-value" +} \ No newline at end of file diff --git a/plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries.json b/plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries__Status_NON_COMPLIANT_EQUAL.json similarity index 99% rename from plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries.json rename to plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries__Status_NON_COMPLIANT_EQUAL.json index ce93307dbc..1b15cb376d 100644 --- a/plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries.json +++ b/plugins/aws/test/resources/files/ssm/list-resource-compliance-summaries__Status_NON_COMPLIANT_EQUAL.json @@ -104,4 +104,4 @@ } ], "NextToken": "foo" -} +} \ No newline at end of file diff --git a/plugins/aws/test/resources/s3_test.py b/plugins/aws/test/resources/s3_test.py index 645a0f4c47..fd9dd264f8 100644 --- a/plugins/aws/test/resources/s3_test.py +++ b/plugins/aws/test/resources/s3_test.py @@ -1,9 +1,14 @@ -from fixlib.graph import Graph -from test.resources import round_trip_for +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta from types import SimpleNamespace -from typing import cast, Any, Callable +from typing import cast, Any, Callable, List +from fix_plugin_aws.resource.base import AwsRegion, GraphBuilder +from fix_plugin_aws.resource.cloudwatch import update_resource_metrics, AwsCloudwatchMetricData, AwsCloudwatchQuery from fix_plugin_aws.aws_client import AwsClient from fix_plugin_aws.resource.s3 import AwsS3Bucket, AwsS3AccountSettings +from fixlib.threading import ExecutorQueue +from fixlib.graph import Graph +from test.resources import round_trip_for def test_buckets() -> None: @@ -62,14 +67,42 @@ def validate_delete_args(aws_service: str, fn: Callable[[Any], None]) -> Any: bucket.delete_resource(client, Graph()) -# TODO: fix 'RuntimeError: cannot schedule new futures after shutdown' -# def test_s3_usage_metrics(account_collector: AwsAccountCollector) -> None: -# bucket, builder = round_trip_for(AwsS3Bucket) -# builder.all_regions.update({"us-east-1": AwsRegion(id="us-east-1", name="us-east-1")}) -# account_collector.collect_usage_metrics(builder) -# bucket.complete_graph(builder, {}) -# assert bucket._resource_usage["standard_storage_bucket_size_bytes"]["avg"] == 1.0 -# assert bucket._resource_usage["intelligent_tiering_storage_bucket_size_bytes"]["avg"] == 2.0 -# assert bucket._resource_usage["standard_ia_storage_bucket_size_bytes"]["avg"] == 3.0 -# # This values is computed internally using the other values. If the number does not match, the logic is broken! -# assert bucket._resource_usage["bucket_size_bytes"]["avg"] == 6.0 +def test_s3_usage_metrics() -> None: + bucket, builder = round_trip_for(AwsS3Bucket, "bucket_lifecycle_policy") + builder.all_regions.update({"us-east-1": AwsRegion(id="us-east-1", name="us-east-1")}) + queries = bucket.collect_usage_metrics(builder) + lookup_map = {} + lookup_map[bucket.id] = bucket + + # simulates the `collect_usage_metrics` method found in `AwsAccountCollector`. + def collect_and_set_metrics(start_at: datetime, region: AwsRegion, queries: List[AwsCloudwatchQuery]) -> None: + with ThreadPoolExecutor(max_workers=1) as executor: + queue = ExecutorQueue(executor, tasks_per_key=lambda _: 1, name="test") + g_builder = GraphBuilder( + builder.graph, + builder.cloud, + builder.account, + region, + {region.id: region}, + builder.client, + queue, + builder.core_feedback, + last_run_started_at=builder.last_run_started_at, + ) + result = AwsCloudwatchMetricData.query_for_multiple( + g_builder, start_at, start_at + timedelta(hours=2), queries + ) + update_resource_metrics(lookup_map, result) + # compute bucket_size_bytes + for after_collect in builder.after_collect_actions: + after_collect() + + start = datetime(2020, 5, 30, 15, 45, 30) + + collect_and_set_metrics(start, AwsRegion(id="us-east-1", name="us-east-1"), queries) + + assert bucket._resource_usage["standard_storage_bucket_size_bytes"]["avg"] == 1.0 + assert bucket._resource_usage["intelligent_tiering_storage_bucket_size_bytes"]["avg"] == 2.0 + assert bucket._resource_usage["standard_ia_storage_bucket_size_bytes"]["avg"] == 3.0 + # This values is computed internally using the other values. If the number does not match, the logic is broken! + assert bucket._resource_usage["bucket_size_bytes"]["avg"] == 6.0 diff --git a/plugins/aws/test/resources/ssm_test.py b/plugins/aws/test/resources/ssm_test.py index b341722990..8e2b5a5656 100644 --- a/plugins/aws/test/resources/ssm_test.py +++ b/plugins/aws/test/resources/ssm_test.py @@ -1,3 +1,4 @@ +from fix_plugin_aws.resource.ec2 import AwsEc2Instance from fix_plugin_aws.resource.ssm import ( AwsSSMInstance, AwsSSMDocument, @@ -6,6 +7,8 @@ ) from test.resources import round_trip_for +from fixlib.baseresources import Severity + def test_instances() -> None: first, builder = round_trip_for(AwsSSMInstance) @@ -13,7 +16,10 @@ def test_instances() -> None: def test_resource_compliance() -> None: - round_trip_for(AwsSSMResourceCompliance) + collected, _ = round_trip_for(AwsEc2Instance, region_name="global", collect_also=[AwsSSMResourceCompliance]) + asseessments = collected._assessments + assert asseessments[0].findings[0].title == "State Manager Association Compliance" + assert asseessments[0].findings[0].severity == Severity.high def test_documents() -> None: diff --git a/plugins/azure/fix_plugin_azure/resource/cosmosdb.py b/plugins/azure/fix_plugin_azure/resource/cosmosdb.py index 99520734a1..12e2626cb2 100644 --- a/plugins/azure/fix_plugin_azure/resource/cosmosdb.py +++ b/plugins/azure/fix_plugin_azure/resource/cosmosdb.py @@ -2057,6 +2057,7 @@ class AzureCosmosDBLocation(CosmosDBLocationSetter, MicrosoftResource, PhantomBa query_parameters=["api-version"], access_path="value", expect_array=True, + expected_error_codes={"Internal Server Error": None}, ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("id"), diff --git a/plugins/azure/fix_plugin_azure/resource/machinelearning.py b/plugins/azure/fix_plugin_azure/resource/machinelearning.py index 949e5f00d7..3ef4599e02 100644 --- a/plugins/azure/fix_plugin_azure/resource/machinelearning.py +++ b/plugins/azure/fix_plugin_azure/resource/machinelearning.py @@ -26,9 +26,16 @@ from fix_plugin_azure.resource.network import AzureNetworkSubnet, AzureNetworkVirtualNetwork from fix_plugin_azure.resource.storage import AzureStorageAccount from fix_plugin_azure.resource.web import AzureWebApp -from fixlib.baseresources import BaseInstanceType, ModelReference, BaseAIJob, BaseAIModel, PhantomBaseResource +from fixlib.baseresources import ( + BaseInstanceType, + ModelReference, + BaseAIJob, + AIJobStatus, + BaseAIModel, + PhantomBaseResource, +) from fixlib.graph import BySearchCriteria -from fixlib.json_bender import Bender, S, ForallBend, Bend, K +from fixlib.json_bender import MapEnum, Bender, S, ForallBend, Bend, K from fixlib.types import Json log = logging.getLogger("fix.plugins.azure") @@ -56,6 +63,24 @@ def collect( return result +AZURE_ML_JOB_STATUS_MAPPING = { + "CancelRequested": AIJobStatus.STOPPING, + "Canceled": AIJobStatus.CANCELLED, + "Completed": AIJobStatus.COMPLETED, + "Failed": AIJobStatus.FAILED, + "Finalizing": AIJobStatus.STOPPING, + "NotResponding": AIJobStatus.UNKNOWN, + "NotStarted": AIJobStatus.PENDING, + "Paused": AIJobStatus.PAUSED, + "Preparing": AIJobStatus.PREPARING, + "Provisioning": AIJobStatus.PREPARING, + "Queued": AIJobStatus.PENDING, + "Running": AIJobStatus.RUNNING, + "Starting": AIJobStatus.PREPARING, + "Unknown": AIJobStatus.UNKNOWN, +} + + @define(eq=False, slots=False) class AzureEndpointAuthKeys: kind: ClassVar[str] = "azure_endpoint_auth_keys" @@ -1495,7 +1520,7 @@ class AzureMachineLearningJob(BaseAIJob, MicrosoftResource, AzureProxyResource): "job_type": S("properties", "jobType"), "notification_setting": S("properties", "notificationSetting") >> Bend(AzureNotificationSetting.mapping), "services": S("properties", "services"), - "status": S("properties", "status"), + "status": S("properties", "status") >> MapEnum(AZURE_ML_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "description": S("properties", "description"), "properties": S("properties", "properties"), } @@ -1510,7 +1535,6 @@ class AzureMachineLearningJob(BaseAIJob, MicrosoftResource, AzureProxyResource): job_type: Optional[str] = field(default=None, metadata={"description": "Enum to determine the type of job."}) notification_setting: Optional[AzureNotificationSetting] = field(default=None, metadata={'description': 'Configuration for notification.'}) # fmt: skip services: Optional[Dict[str, AzureJobService]] = field(default=None, metadata={'description': 'List of JobEndpoints. For local jobs, a job endpoint will have an endpoint value of FileStreamObject.'}) # fmt: skip - status: Optional[str] = field(default=None, metadata={"description": "The status of a job."}) def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: if compute_id := self.compute_id: @@ -1660,7 +1684,7 @@ class AzureMachineLearningLabelingJob(BaseAIJob, MicrosoftResource): "progress_metrics": S("properties", "progressMetrics") >> Bend(AzureProgressMetrics.mapping), "job_project_id": S("properties", "projectId"), "properties": S("properties", "properties"), - "status": S("properties", "status"), + "status": S("properties", "status") >> MapEnum(AZURE_ML_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "status_messages": S("properties", "statusMessages") >> ForallBend(AzureStatusMessage.mapping), "system_data": S("systemData") >> Bend(AzureSystemData.mapping), } @@ -1673,7 +1697,6 @@ class AzureMachineLearningLabelingJob(BaseAIJob, MicrosoftResource): progress_metrics: Optional[AzureProgressMetrics] = field(default=None, metadata={'description': 'Progress metrics for a labeling job.'}) # fmt: skip job_project_id: Optional[str] = field(default=None, metadata={'description': 'Internal id of the job(Previously called project).'}) # fmt: skip properties: Optional[Dict[str, Any]] = field(default=None, metadata={'description': 'The job property dictionary. Properties can be added, but not removed or altered.'}) # fmt: skip - status: Optional[str] = field(default=None, metadata={"description": "The status of a job."}) status_messages: Optional[List[AzureStatusMessage]] = field(default=None, metadata={'description': 'Status messages of the job.'}) # fmt: skip system_data: Optional[AzureSystemData] = field(default=None, metadata={'description': 'Metadata pertaining to creation and last modification of the resource.'}) # fmt: skip diff --git a/plugins/azure/fix_plugin_azure/resource/microsoft_graph.py b/plugins/azure/fix_plugin_azure/resource/microsoft_graph.py index 9b90b9e145..b77d159a20 100644 --- a/plugins/azure/fix_plugin_azure/resource/microsoft_graph.py +++ b/plugins/azure/fix_plugin_azure/resource/microsoft_graph.py @@ -826,6 +826,7 @@ class MicrosoftGraphUser(MicrosoftGraphEntity, BaseUser): "usage_location": S("usageLocation"), "user_principal_name": S("userPrincipalName"), "user_type": S("userType"), + "username": S("displayName"), } account_enabled: Optional[bool] = field(default=None, metadata={'description': 'true if the account is enabled; otherwise, false. This property is required when a user is created. Supports $filter (eq, ne, not, and in).'}) # fmt: skip age_group: Optional[str] = field(default=None, metadata={'description': 'Sets the age group of the user. Allowed values: null, Minor, NotAdult, and Adult. For more information, see legal age group property definitions. Supports $filter (eq, ne, not, and in).'}) # fmt: skip diff --git a/plugins/azure/fix_plugin_azure/resource/monitor.py b/plugins/azure/fix_plugin_azure/resource/monitor.py index 5bb83e914c..26323d906a 100644 --- a/plugins/azure/fix_plugin_azure/resource/monitor.py +++ b/plugins/azure/fix_plugin_azure/resource/monitor.py @@ -346,72 +346,6 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: ) -@define(eq=False, slots=False) -class AzureMonitorRuleDataSource: - kind: ClassVar[str] = "azure_monitor_rule_data_source" - mapping: ClassVar[Dict[str, Bender]] = { - "legacy_resource_id": S("legacyResourceId"), - "metric_namespace": S("metricNamespace"), - "type": S("odata.type"), - "resource_location": S("resourceLocation"), - "resource_uri": S("resourceUri"), - } - legacy_resource_id: Optional[str] = field(default=None, metadata={'description': 'the legacy resource identifier of the resource the rule monitors. **NOTE**: this property cannot be updated for an existing rule.'}) # fmt: skip - metric_namespace: Optional[str] = field(default=None, metadata={"description": "the namespace of the metric."}) - type: Optional[str] = field(default=None, metadata={'description': 'specifies the type of data source. There are two types of rule data sources: RuleMetricDataSource and RuleManagementEventDataSource'}) # fmt: skip - resource_location: Optional[str] = field(default=None, metadata={"description": "the location of the resource."}) - resource_uri: Optional[str] = field(default=None, metadata={'description': 'the resource identifier of the resource the rule monitors. **NOTE**: this property cannot be updated for an existing rule.'}) # fmt: skip - - -@define(eq=False, slots=False) -class AzureMonitorRuleCondition: - kind: ClassVar[str] = "azure_monitor_rule_condition" - mapping: ClassVar[Dict[str, Bender]] = { - "data_source": S("dataSource") >> Bend(AzureMonitorRuleDataSource.mapping), - "type": S("odata.type"), - } - data_source: Optional[AzureMonitorRuleDataSource] = field(default=None, metadata={'description': 'The resource from which the rule collects its data.'}) # fmt: skip - type: Optional[str] = field(default=None, metadata={'description': 'specifies the type of condition. This can be one of three types: ManagementEventRuleCondition (occurrences of management events), LocationThresholdRuleCondition (based on the number of failures of a web test), and ThresholdRuleCondition (based on the threshold of a metric).'}) # fmt: skip - - -@define(eq=False, slots=False) -class AzureMonitorAlertRule(MicrosoftResource): - kind: ClassVar[str] = "azure_monitor_alert_rule" - _kind_display: ClassVar[str] = "Azure Monitor Alert Rule" - _kind_service: ClassVar[Optional[str]] = service_name - _kind_description: ClassVar[str] = "Azure Monitor Alert Rule is a feature in Microsoft Azure that defines conditions for monitoring resources and triggers notifications when specified thresholds are met. It evaluates metrics, logs, and activity data from Azure services, then sends alerts via various channels when predefined criteria are satisfied, helping administrators respond to issues and maintain system health." # fmt: skip - _docs_url: ClassVar[str] = "https://learn.microsoft.com/en-us/azure/azure-monitor/alerts/alerts-overview" - _metadata: ClassVar[Dict[str, Any]] = {"icon": "config", "group": "management"} - _create_provider_link: ClassVar[bool] = False - api_spec: ClassVar[AzureResourceSpec] = AzureResourceSpec( - service="monitor", - version="2016-03-01", - path="/subscriptions/{subscriptionId}/providers/Microsoft.Insights/alertrules", - path_parameters=["subscriptionId"], - query_parameters=["api-version"], - access_path="value", - expect_array=True, - ) - mapping: ClassVar[Dict[str, Bender]] = { - "id": S("id"), - "tags": S("tags").or_else(K({})), - "name": S("name"), - "action": S("properties", "action", "odata.type"), - "actions": S("properties") >> S("actions", default=[]) >> ForallBend(S("odata.type")), - "rule_condition": S("properties", "condition") >> Bend(AzureMonitorRuleCondition.mapping), - "description": S("properties", "description"), - "is_enabled": S("properties", "isEnabled"), - "last_updated_time": S("properties", "lastUpdatedTime"), - "provisioning_state": S("properties", "provisioningState"), - } - action: Optional[str] = field(default=None, metadata={'description': 'The action that is performed when the alert rule becomes active, and when an alert condition is resolved.'}) # fmt: skip - actions: Optional[List[str]] = field(default=None, metadata={'description': 'the array of actions that are performed when the alert rule becomes active, and when an alert condition is resolved.'}) # fmt: skip - rule_condition: Optional[AzureMonitorRuleCondition] = field(default=None, metadata={'description': 'The condition that results in the alert rule being activated.'}) # fmt: skip - description: Optional[str] = field(default=None, metadata={'description': 'the description of the alert rule that will be included in the alert email.'}) # fmt: skip - is_enabled: Optional[bool] = field(default=None, metadata={'description': 'the flag that indicates whether the alert rule is enabled.'}) # fmt: skip - last_updated_time: Optional[datetime] = field(default=None, metadata={'description': 'Last time the rule was updated in ISO8601 format.'}) # fmt: skip - - @define(eq=False, slots=False) class AzureMonitorAccessModeSettingsExclusion: kind: ClassVar[str] = "azure_monitor_access_mode_settings_exclusion" @@ -1463,7 +1397,6 @@ def execute() -> None: resources: List[Type[MicrosoftResource]] = [ AzureMonitorActionGroup, AzureMonitorActivityLogAlert, - AzureMonitorAlertRule, AzureMonitorDataCollectionRule, AzureMonitorLogProfile, AzureMonitorMetricAlert, diff --git a/plugins/azure/fix_plugin_azure/resource/network.py b/plugins/azure/fix_plugin_azure/resource/network.py index 7b9814d51b..8c2ed7c8a8 100644 --- a/plugins/azure/fix_plugin_azure/resource/network.py +++ b/plugins/azure/fix_plugin_azure/resource/network.py @@ -40,7 +40,7 @@ EdgeType, PhantomBaseResource, ) -from fixlib.json_bender import F, Bender, S, Bend, ForallBend, AsInt, StringToUnitNumber, Upper, Lower +from fixlib.json_bender import F, MapValue, Bender, S, Bend, ForallBend, AsInt, StringToUnitNumber, Upper, Lower from fixlib.types import Json service_name = "networking" @@ -2347,6 +2347,7 @@ class AzureNetworkSubnet(MicrosoftResource, BaseSubnet): "service_endpoints": S("properties", "serviceEndpoints") >> ForallBend(AzureServiceEndpointPropertiesFormat.mapping), "type": S("type"), + "cidr_block": S("properties", "addressPrefix"), } address_prefix: Optional[str] = field(default=None, metadata={"description": "The address prefix for the subnet."}) address_prefixes: Optional[List[str]] = field(default=None, metadata={'description': 'List of address prefixes for the subnet.'}) # fmt: skip @@ -5275,6 +5276,7 @@ class AzureNetworkVirtualNetwork(MicrosoftResource, BaseNetwork): >> ForallBend(AzureVirtualNetworkPeering.mapping), "location": S("location"), "type": S("type"), + "cidr_blocks": S("properties", "addressSpace", "addressPrefixes", default=[]), } address_space: Optional[AzureAddressSpace] = field(default=None, metadata={'description': 'AddressSpace contains an array of IP address ranges that can be used by subnets of the virtual network.'}) # fmt: skip bgp_communities: Optional[AzureVirtualNetworkBgpCommunities] = field(default=None, metadata={'description': 'Bgp Communities sent over ExpressRoute with each route corresponding to a prefix in this VNET.'}) # fmt: skip @@ -6773,6 +6775,7 @@ class AzureNetworkDNSZone(MicrosoftResource, BaseDNSZone): query_parameters=["api-version"], access_path="value", expect_array=True, + expected_error_codes={"BadRequest": None}, ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("id"), @@ -6789,7 +6792,14 @@ class AzureNetworkDNSZone(MicrosoftResource, BaseDNSZone): "resolution_virtual_networks": S("properties") >> S("resolutionVirtualNetworks", default=[]) >> ForallBend(S("id")), - "zone_type": S("properties", "zoneType"), + "private_zone": S("properties", "zoneType") + >> MapValue( + { + "Public": False, + "Private": True, + }, + default=False, + ), } max_number_of_record_sets: Optional[int] = field(default=None, metadata={'description': 'The maximum number of record sets that can be created in this DNS zone. This is a read-only property and any attempt to set this value will be ignored.'}) # fmt: skip max_number_of_records_per_record_set: Optional[int] = field(default=None, metadata={'description': 'The maximum number of records per record set that can be created in this DNS zone. This is a read-only property and any attempt to set this value will be ignored.'}) # fmt: skip @@ -6797,7 +6807,6 @@ class AzureNetworkDNSZone(MicrosoftResource, BaseDNSZone): number_of_record_sets: Optional[int] = field(default=None, metadata={'description': 'The current number of record sets in this DNS zone. This is a read-only property and any attempt to set this value will be ignored.'}) # fmt: skip registration_virtual_networks: Optional[List[str]] = field(default=None, metadata={'description': 'A list of references to virtual networks that register hostnames in this DNS zone. This is a only when ZoneType is Private.'}) # fmt: skip resolution_virtual_networks: Optional[List[str]] = field(default=None, metadata={'description': 'A list of references to virtual networks that resolve records in this DNS zone. This is a only when ZoneType is Private.'}) # fmt: skip - zone_type: Optional[str] = field(default=None, metadata={'description': 'The type of this DNS zone (Public or Private).'}) # fmt: skip def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: def collect_record_sets() -> None: diff --git a/plugins/azure/fix_plugin_azure/resource/storage.py b/plugins/azure/fix_plugin_azure/resource/storage.py index d4a4e6ebcf..463c0a9f88 100644 --- a/plugins/azure/fix_plugin_azure/resource/storage.py +++ b/plugins/azure/fix_plugin_azure/resource/storage.py @@ -27,7 +27,7 @@ ModelReference, PhantomBaseResource, ) -from fixlib.json_bender import Bender, S, ForallBend, Bend +from fixlib.json_bender import K, Bender, S, ForallBend, Bend, AsBool from fixlib.types import Json log = logging.getLogger("fix.plugins.azure") @@ -165,6 +165,8 @@ class AzureStorageBlobContainer(MicrosoftResource, BaseBucket): "public_access": S("properties", "publicAccess"), "remaining_retention_days": S("properties", "remainingRetentionDays"), "version": S("properties", "version"), + "encryption_enabled": S("properties", "defaultEncryptionScope") >> AsBool(), + "versioning_enabled": S("properties", "immutableStorageWithVersioning") >> AsBool(), } type: Optional[str] = field(default=None, metadata={'description': 'The type of the resource. E.g. Microsoft.Compute/virtualMachines or Microsoft.Storage/storageAccounts '}) # fmt: skip default_encryption_scope: Optional[str] = field(default=None, metadata={'description': 'Default the container to use specified encryption scope for all writes.'}) # fmt: skip @@ -204,6 +206,7 @@ class AzureStorageAccountDeleted(MicrosoftResource, PhantomBaseResource): query_parameters=["api-version"], access_path="value", expect_array=True, + expected_error_codes={"ProviderError": None}, ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("id"), @@ -316,10 +319,11 @@ class AzureStorageQueue(MicrosoftResource, BaseQueue): "id": S("id"), "tags": S("tags", default={}), "name": S("name"), - "approximate_message_count": S("properties", "approximateMessageCount"), "queue_metadata": S("properties", "metadata"), + "queue_type": K("standard"), + "message_retention_period": K(7), + "approximate_message_count": S("properties", "approximateMessageCount"), } - approximate_message_count: Optional[int] = field(default=None, metadata={'description': 'Integer indicating an approximate number of messages in the queue. This number is not lower than the actual number of messages in the queue, but could be higher.'}) # fmt: skip queue_metadata: Optional[Dict[str, str]] = field(default=None, metadata={'description': 'A name-value pair that represents queue metadata.'}) # fmt: skip @@ -1082,7 +1086,7 @@ class AzureStorageAccountUsage(MicrosoftResource, AzureBaseUsage): query_parameters=["api-version"], access_path="value", expect_array=True, - expected_error_codes=AzureBaseUsage._expected_error_codes, + expected_error_codes=AzureBaseUsage._expected_error_codes | {"SubscriptionNotFound": None}, ) mapping: ClassVar[Dict[str, Bender]] = AzureBaseUsage.mapping | { "id": S("name", "value"), diff --git a/plugins/azure/fix_plugin_azure/resource/web.py b/plugins/azure/fix_plugin_azure/resource/web.py index c1f0296c72..7920046fcf 100644 --- a/plugins/azure/fix_plugin_azure/resource/web.py +++ b/plugins/azure/fix_plugin_azure/resource/web.py @@ -1481,6 +1481,7 @@ class AzureWebApp(MicrosoftResource, BaseServerlessFunction): "vnet_image_pull_enabled": S("properties", "vnetImagePullEnabled"), "vnet_route_all_enabled": S("properties", "vnetRouteAllEnabled"), "workload_profile_name": S("properties", "workloadProfileName"), + "memory_size": S("properties", "siteConfig", "limits", "maxMemoryInMb"), } availability_state: Optional[str] = field(default=None, metadata={'description': 'Management information availability state for the app.'}) # fmt: skip client_affinity_enabled: Optional[bool] = field(default=None, metadata={'description': 'true to enable client affinity; false to stop sending session affinity cookies, which route client requests in the same session to the same instance. Default is true.'}) # fmt: skip diff --git a/plugins/azure/test/collector_test.py b/plugins/azure/test/collector_test.py index 0540019dc8..65d9c83420 100644 --- a/plugins/azure/test/collector_test.py +++ b/plugins/azure/test/collector_test.py @@ -48,8 +48,8 @@ def test_collect( config, Cloud(id="azure"), azure_subscription, credentials, core_feedback, filter_unused_resources=False ) subscription_collector.collect() - assert len(subscription_collector.graph.nodes) == 887 - assert len(subscription_collector.graph.edges) == 1282 + assert len(subscription_collector.graph.nodes) == 885 + assert len(subscription_collector.graph.edges) == 1280 graph_collector = MicrosoftGraphOrganizationCollector( config, Cloud(id="azure"), MicrosoftGraphOrganization(id="test", name="test"), credentials, core_feedback diff --git a/plugins/azure/test/monitor_test.py b/plugins/azure/test/monitor_test.py index cf6701eb8f..67cd2dd121 100644 --- a/plugins/azure/test/monitor_test.py +++ b/plugins/azure/test/monitor_test.py @@ -3,7 +3,6 @@ from fix_plugin_azure.resource.monitor import ( AzureMonitorActionGroup, AzureMonitorActivityLogAlert, - AzureMonitorAlertRule, AzureMonitorLogProfile, AzureMonitorMetricAlert, AzureMonitorPrivateLinkScope, @@ -25,11 +24,6 @@ def test_activity_log_alert(builder: GraphBuilder) -> None: assert len(collected) == 2 -def test_alert_rule(builder: GraphBuilder) -> None: - collected = roundtrip_check(AzureMonitorAlertRule, builder) - assert len(collected) == 2 - - def test_data_collection_rule(builder: GraphBuilder) -> None: collected = roundtrip_check(AzureMonitorDataCollectionRule, builder) assert len(collected) == 2 diff --git a/plugins/gcp/fix_plugin_gcp/collector.py b/plugins/gcp/fix_plugin_gcp/collector.py index c68080294d..cdd59e4711 100644 --- a/plugins/gcp/fix_plugin_gcp/collector.py +++ b/plugins/gcp/fix_plugin_gcp/collector.py @@ -4,7 +4,17 @@ from fix_plugin_gcp.config import GcpConfig from fix_plugin_gcp.gcp_client import GcpApiSpec -from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform +from fix_plugin_gcp.resources import ( + compute, + container, + billing, + sqladmin, + storage, + aiplatform, + firestore, + filestore, + cloudfunctions, +) from fix_plugin_gcp.resources.base import GcpResource, GcpProject, ExecutorQueue, GraphBuilder, GcpRegion, GcpZone from fix_plugin_gcp.utils import Credentials from fixlib.baseresources import Cloud @@ -19,6 +29,9 @@ + sqladmin.resources + storage.resources + aiplatform.resources + + firestore.resources + + filestore.resources + + cloudfunctions.resources ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py b/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py index 886f44264d..7e21fcf38b 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py +++ b/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py @@ -12,8 +12,8 @@ GcpDeprecationStatus, GraphBuilder, ) -from fixlib.baseresources import BaseAIJob, ModelReference, BaseAIModel -from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict +from fixlib.baseresources import BaseAIJob, AIJobStatus, ModelReference, BaseAIModel +from fixlib.json_bender import Bender, S, Bend, ForallBend, MapDict, MapEnum from fixlib.types import Json service_name = "aiplatform" @@ -29,6 +29,21 @@ "Permission denied by location policies", } +GCP_AI_JOB_STATUS_MAPPING = { + "JOB_STATE_UNSPECIFIED": AIJobStatus.UNKNOWN, + "JOB_STATE_QUEUED": AIJobStatus.PENDING, + "JOB_STATE_PENDING": AIJobStatus.PENDING, + "JOB_STATE_RUNNING": AIJobStatus.RUNNING, + "JOB_STATE_SUCCEEDED": AIJobStatus.COMPLETED, + "JOB_STATE_FAILED": AIJobStatus.FAILED, + "JOB_STATE_CANCELLING": AIJobStatus.STOPPING, + "JOB_STATE_CANCELLED": AIJobStatus.CANCELLED, + "JOB_STATE_PAUSED": AIJobStatus.PAUSED, + "JOB_STATE_EXPIRED": AIJobStatus.FAILED, + "JOB_STATE_UPDATING": AIJobStatus.RUNNING, + "JOB_STATE_PARTIALLY_SUCCEEDED": AIJobStatus.COMPLETED, +} + class AIPlatformRegionFilter: @classmethod @@ -531,7 +546,7 @@ class GcpVertexAIBatchPredictionJob(AIPlatformRegionFilter, BaseAIJob, GcpResour "resources_consumed": S("resourcesConsumed", "replicaHours"), "service_account": S("serviceAccount"), "start_time": S("startTime"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "unmanaged_container_model": S("unmanagedContainerModel", default={}) >> Bend(GcpVertexAIUnmanagedContainerModel.mapping), "update_time": S("updateTime"), @@ -558,7 +573,6 @@ class GcpVertexAIBatchPredictionJob(AIPlatformRegionFilter, BaseAIJob, GcpResour resources_consumed: Optional[float] = field(default=None) service_account: Optional[str] = field(default=None) start_time: Optional[datetime] = field(default=None) - state: Optional[str] = field(default=None) unmanaged_container_model: Optional[GcpVertexAIUnmanagedContainerModel] = field(default=None) update_time: Optional[datetime] = field(default=None) @@ -725,7 +739,7 @@ class GcpVertexAICustomJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): "rpc_error": S("error", default={}) >> Bend(GcpGoogleRpcStatus.mapping), "custom_job_spec": S("jobSpec", default={}) >> Bend(GcpVertexAICustomJobSpec.mapping), "start_time": S("startTime"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "update_time": S("updateTime"), "web_access_uris": S("webAccessUris"), } @@ -736,7 +750,6 @@ class GcpVertexAICustomJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): rpc_error: Optional[GcpGoogleRpcStatus] = field(default=None) custom_job_spec: Optional[GcpVertexAICustomJobSpec] = field(default=None) start_time: Optional[datetime] = field(default=None) - state: Optional[str] = field(default=None) update_time: Optional[datetime] = field(default=None) web_access_uris: Optional[Dict[str, str]] = field(default=None) @@ -1635,7 +1648,7 @@ class GcpVertexAIHyperparameterTuningJob(AIPlatformRegionFilter, BaseAIJob, GcpR "max_trial_count": S("maxTrialCount"), "parallel_trial_count": S("parallelTrialCount"), "start_time": S("startTime"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "study_spec": S("studySpec", default={}) >> Bend(GcpVertexAIStudySpec.mapping), "trial_job_spec": S("trialJobSpec", default={}) >> Bend(GcpVertexAICustomJobSpec.mapping), "trials": S("trials", default=[]) >> ForallBend(GcpVertexAITrial.mapping), @@ -1650,7 +1663,6 @@ class GcpVertexAIHyperparameterTuningJob(AIPlatformRegionFilter, BaseAIJob, GcpR max_trial_count: Optional[int] = field(default=None) parallel_trial_count: Optional[int] = field(default=None) start_time: Optional[datetime] = field(default=None) - state: Optional[str] = field(default=None) study_spec: Optional[GcpVertexAIStudySpec] = field(default=None) trial_job_spec: Optional[GcpVertexAICustomJobSpec] = field(default=None) trials: Optional[List[GcpVertexAITrial]] = field(default=None) @@ -1886,19 +1898,6 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: @define(eq=False, slots=False) class GcpVertexAIArtifact: kind: ClassVar[str] = "gcp_vertex_ai_artifact" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "artifacts"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "region"}, - response_path="artifacts", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), @@ -2165,7 +2164,7 @@ class GcpVertexAIModelDeploymentMonitoringJob(AIPlatformRegionFilter, BaseAIJob, "predict_instance_schema_uri": S("predictInstanceSchemaUri"), "sample_predict_instance": S("samplePredictInstance"), "schedule_state": S("scheduleState"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "stats_anomalies_base_directory": S("statsAnomaliesBaseDirectory", "outputUriPrefix"), "update_time": S("updateTime"), } @@ -2193,7 +2192,6 @@ class GcpVertexAIModelDeploymentMonitoringJob(AIPlatformRegionFilter, BaseAIJob, predict_instance_schema_uri: Optional[str] = field(default=None) sample_predict_instance: Optional[Any] = field(default=None) schedule_state: Optional[str] = field(default=None) - state: Optional[str] = field(default=None) stats_anomalies_base_directory: Optional[str] = field(default=None) update_time: Optional[datetime] = field(default=None) @@ -2519,19 +2517,6 @@ class GcpVertexAIPipelineTaskDetailPipelineTaskStatus: @define(eq=False, slots=False) class GcpVertexAIExecution: kind: ClassVar[str] = "gcp_vertex_ai_execution" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "executions"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "location"}, - response_path="executions", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), @@ -2597,19 +2582,6 @@ class GcpVertexAIPipelineTaskDetail: @define(eq=False, slots=False) class GcpVertexAIContext: kind: ClassVar[str] = "gcp_vertex_ai_context" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "contexts"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "location"}, - response_path="contexts", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), @@ -2744,7 +2716,7 @@ class GcpVertexAIPipelineJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): "schedule_name": S("scheduleName"), "service_account": S("serviceAccount"), "start_time": S("startTime"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "template_metadata": S("templateMetadata", "version"), "template_uri": S("templateUri"), "update_time": S("updateTime"), @@ -2763,7 +2735,6 @@ class GcpVertexAIPipelineJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): schedule_name: Optional[str] = field(default=None) service_account: Optional[str] = field(default=None) start_time: Optional[datetime] = field(default=None) - state: Optional[str] = field(default=None) template_metadata: Optional[str] = field(default=None) template_uri: Optional[str] = field(default=None) update_time: Optional[datetime] = field(default=None) @@ -3299,7 +3270,7 @@ class GcpVertexAITuningJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): "rpc_error": S("error", default={}) >> Bend(GcpGoogleRpcStatus.mapping), "experiment": S("experiment"), "start_time": S("startTime"), - "state": S("state"), + "state": S("state") >> MapEnum(GCP_AI_JOB_STATUS_MAPPING, AIJobStatus.UNKNOWN), "supervised_tuning_spec": S("supervisedTuningSpec", default={}) >> Bend(GcpVertexAISupervisedTuningSpec.mapping), "tuned_model": S("tunedModel", default={}) >> Bend(GcpVertexAITunedModel.mapping), @@ -3314,7 +3285,6 @@ class GcpVertexAITuningJob(AIPlatformRegionFilter, BaseAIJob, GcpResource): rpc_error: Optional[GcpGoogleRpcStatus] = field(default=None) experiment: Optional[str] = field(default=None) start_time: Optional[datetime] = field(default=None) - state: Optional[str] = field(default=None) supervised_tuning_spec: Optional[GcpVertexAISupervisedTuningSpec] = field(default=None) tuned_model: Optional[GcpVertexAITunedModel] = field(default=None) tuned_model_display_name: Optional[str] = field(default=None) diff --git a/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py new file mode 100644 index 0000000000..0fd5f113e2 --- /dev/null +++ b/plugins/gcp/fix_plugin_gcp/resources/cloudfunctions.py @@ -0,0 +1,304 @@ +from datetime import datetime +from typing import ClassVar, Dict, Optional, List, Type, Any + +from attr import define, field + +from fix_plugin_gcp.gcp_client import GcpApiSpec +from fix_plugin_gcp.resources.base import GcpResource, GcpDeprecationStatus +from fixlib.baseresources import BaseServerlessFunction +from fixlib.json_bender import Bender, S, Bend, ForallBend + + +@define(eq=False, slots=False) +class GcpRepoSource: + kind: ClassVar[str] = "gcp_repo_source" + mapping: ClassVar[Dict[str, Bender]] = { + "branch_name": S("branchName"), + "commit_sha": S("commitSha"), + "dir": S("dir"), + "project_id": S("projectId"), + "repo_name": S("repoName"), + "tag_name": S("tagName"), + } + branch_name: Optional[str] = field(default=None) + commit_sha: Optional[str] = field(default=None) + dir: Optional[str] = field(default=None) + project_id: Optional[str] = field(default=None) + repo_name: Optional[str] = field(default=None) + tag_name: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpStorageSource: + kind: ClassVar[str] = "gcp_storage_source" + mapping: ClassVar[Dict[str, Bender]] = { + "bucket": S("bucket"), + "generation": S("generation"), + "object": S("object"), + "source_upload_url": S("sourceUploadUrl"), + } + bucket: Optional[str] = field(default=None) + generation: Optional[str] = field(default=None) + object: Optional[str] = field(default=None) + source_upload_url: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpSource: + kind: ClassVar[str] = "gcp_source" + mapping: ClassVar[Dict[str, Bender]] = { + "git_uri": S("gitUri"), + "repo_source": S("repoSource", default={}) >> Bend(GcpRepoSource.mapping), + "storage_source": S("storageSource", default={}) >> Bend(GcpStorageSource.mapping), + } + git_uri: Optional[str] = field(default=None) + repo_source: Optional[GcpRepoSource] = field(default=None) + storage_source: Optional[GcpStorageSource] = field(default=None) + + +@define(eq=False, slots=False) +class GcpSourceProvenance: + kind: ClassVar[str] = "gcp_source_provenance" + mapping: ClassVar[Dict[str, Bender]] = { + "git_uri": S("gitUri"), + "resolved_repo_source": S("resolvedRepoSource", default={}) >> Bend(GcpRepoSource.mapping), + "resolved_storage_source": S("resolvedStorageSource", default={}) >> Bend(GcpStorageSource.mapping), + } + git_uri: Optional[str] = field(default=None) + resolved_repo_source: Optional[GcpRepoSource] = field(default=None) + resolved_storage_source: Optional[GcpStorageSource] = field(default=None) + + +@define(eq=False, slots=False) +class GcpBuildConfig: + kind: ClassVar[str] = "gcp_build_config" + mapping: ClassVar[Dict[str, Bender]] = { + "automatic_update_policy": S("automaticUpdatePolicy", default={}), + "build": S("build"), + "docker_registry": S("dockerRegistry"), + "docker_repository": S("dockerRepository"), + "entry_point": S("entryPoint"), + "environment_variables": S("environmentVariables"), + "on_deploy_update_policy": S("onDeployUpdatePolicy", "runtimeVersion"), + "runtime": S("runtime"), + "service_account": S("serviceAccount"), + "source": S("source", default={}) >> Bend(GcpSource.mapping), + "source_provenance": S("sourceProvenance", default={}) >> Bend(GcpSourceProvenance.mapping), + "source_token": S("sourceToken"), + "worker_pool": S("workerPool"), + } + automatic_update_policy: Optional[Dict[str, Any]] = field(default=None) + build: Optional[str] = field(default=None) + docker_registry: Optional[str] = field(default=None) + docker_repository: Optional[str] = field(default=None) + entry_point: Optional[str] = field(default=None) + environment_variables: Optional[Dict[str, str]] = field(default=None) + on_deploy_update_policy: Optional[str] = field(default=None) + runtime: Optional[str] = field(default=None) + service_account: Optional[str] = field(default=None) + source: Optional[GcpSource] = field(default=None) + source_provenance: Optional[GcpSourceProvenance] = field(default=None) + source_token: Optional[str] = field(default=None) + worker_pool: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpEventFilter: + kind: ClassVar[str] = "gcp_event_filter" + mapping: ClassVar[Dict[str, Bender]] = {"attribute": S("attribute"), "operator": S("operator"), "value": S("value")} + attribute: Optional[str] = field(default=None) + operator: Optional[str] = field(default=None) + value: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpEventTrigger: + kind: ClassVar[str] = "gcp_event_trigger" + mapping: ClassVar[Dict[str, Bender]] = { + "channel": S("channel"), + "event_filters": S("eventFilters", default=[]) >> ForallBend(GcpEventFilter.mapping), + "event_type": S("eventType"), + "pubsub_topic": S("pubsubTopic"), + "retry_policy": S("retryPolicy"), + "service": S("service"), + "service_account_email": S("serviceAccountEmail"), + "trigger": S("trigger"), + "trigger_region": S("triggerRegion"), + } + channel: Optional[str] = field(default=None) + event_filters: Optional[List[GcpEventFilter]] = field(default=None) + event_type: Optional[str] = field(default=None) + pubsub_topic: Optional[str] = field(default=None) + retry_policy: Optional[str] = field(default=None) + service: Optional[str] = field(default=None) + service_account_email: Optional[str] = field(default=None) + trigger: Optional[str] = field(default=None) + trigger_region: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpSecretEnvVar: + kind: ClassVar[str] = "gcp_secret_env_var" + mapping: ClassVar[Dict[str, Bender]] = { + "key": S("key"), + "project_id": S("projectId"), + "secret": S("secret"), + "version": S("version"), + } + key: Optional[str] = field(default=None) + project_id: Optional[str] = field(default=None) + secret: Optional[str] = field(default=None) + version: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpSecretVersion: + kind: ClassVar[str] = "gcp_secret_version" + mapping: ClassVar[Dict[str, Bender]] = {"path": S("path"), "version": S("version")} + path: Optional[str] = field(default=None) + version: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpSecretVolume: + kind: ClassVar[str] = "gcp_secret_volume" + mapping: ClassVar[Dict[str, Bender]] = { + "mount_path": S("mountPath"), + "project_id": S("projectId"), + "secret": S("secret"), + "versions": S("versions", default=[]) >> ForallBend(GcpSecretVersion.mapping), + } + mount_path: Optional[str] = field(default=None) + project_id: Optional[str] = field(default=None) + secret: Optional[str] = field(default=None) + versions: Optional[List[GcpSecretVersion]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpServiceConfig: + kind: ClassVar[str] = "gcp_service_config" + mapping: ClassVar[Dict[str, Bender]] = { + "all_traffic_on_latest_revision": S("allTrafficOnLatestRevision"), + "available_cpu": S("availableCpu"), + "available_memory": S("availableMemory"), + "binary_authorization_policy": S("binaryAuthorizationPolicy"), + "environment_variables": S("environmentVariables"), + "ingress_settings": S("ingressSettings"), + "max_instance_count": S("maxInstanceCount"), + "max_instance_request_concurrency": S("maxInstanceRequestConcurrency"), + "min_instance_count": S("minInstanceCount"), + "revision": S("revision"), + "secret_environment_variables": S("secretEnvironmentVariables", default=[]) + >> ForallBend(GcpSecretEnvVar.mapping), + "secret_volumes": S("secretVolumes", default=[]) >> ForallBend(GcpSecretVolume.mapping), + "security_level": S("securityLevel"), + "service": S("service"), + "service_account_email": S("serviceAccountEmail"), + "timeout_seconds": S("timeoutSeconds"), + "uri": S("uri"), + "vpc_connector": S("vpcConnector"), + "vpc_connector_egress_settings": S("vpcConnectorEgressSettings"), + } + all_traffic_on_latest_revision: Optional[bool] = field(default=None) + available_cpu: Optional[str] = field(default=None) + available_memory: Optional[str] = field(default=None) + binary_authorization_policy: Optional[str] = field(default=None) + environment_variables: Optional[Dict[str, str]] = field(default=None) + ingress_settings: Optional[str] = field(default=None) + max_instance_count: Optional[int] = field(default=None) + max_instance_request_concurrency: Optional[int] = field(default=None) + min_instance_count: Optional[int] = field(default=None) + revision: Optional[str] = field(default=None) + secret_environment_variables: Optional[List[GcpSecretEnvVar]] = field(default=None) + secret_volumes: Optional[List[GcpSecretVolume]] = field(default=None) + security_level: Optional[str] = field(default=None) + service: Optional[str] = field(default=None) + service_account_email: Optional[str] = field(default=None) + timeout_seconds: Optional[int] = field(default=None) + uri: Optional[str] = field(default=None) + vpc_connector: Optional[str] = field(default=None) + vpc_connector_egress_settings: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpCloudFunctionsStateMessage: + kind: ClassVar[str] = "gcp_cloud_functions_state_message" + mapping: ClassVar[Dict[str, Bender]] = {"message": S("message"), "severity": S("severity"), "type": S("type")} + message: Optional[str] = field(default=None) + severity: Optional[str] = field(default=None) + type: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpUpgradeInfo: + kind: ClassVar[str] = "gcp_upgrade_info" + mapping: ClassVar[Dict[str, Bender]] = { + "build_config": S("buildConfig", default={}) >> Bend(GcpBuildConfig.mapping), + "event_trigger": S("eventTrigger", default={}) >> Bend(GcpEventTrigger.mapping), + "service_config": S("serviceConfig", default={}) >> Bend(GcpServiceConfig.mapping), + "upgrade_state": S("upgradeState"), + } + build_config: Optional[GcpBuildConfig] = field(default=None) + event_trigger: Optional[GcpEventTrigger] = field(default=None) + service_config: Optional[GcpServiceConfig] = field(default=None) + upgrade_state: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpCloudFunction(GcpResource, BaseServerlessFunction): + kind: ClassVar[str] = "gcp_cloud_function" + _kind_display: ClassVar[str] = "GCP Cloud Function" + _kind_description: ClassVar[str] = ( + "GCP Cloud Function is a serverless execution environment for building and connecting cloud services." + " It allows you to run your code in response to events without provisioning or managing servers." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/functions/docs" + _kind_service: ClassVar[Optional[str]] = "cloudfunctions" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "function", "group": "compute"} + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="cloudfunctions", + version="v2", + accessors=["projects", "locations", "functions"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="functions", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "build_config": S("buildConfig", default={}) >> Bend(GcpBuildConfig.mapping), + "create_time": S("createTime"), + "environment": S("environment"), + "event_trigger": S("eventTrigger", default={}) >> Bend(GcpEventTrigger.mapping), + "kms_key_name": S("kmsKeyName"), + "satisfies_pzs": S("satisfiesPzs"), + "service_config": S("serviceConfig", default={}) >> Bend(GcpServiceConfig.mapping), + "state": S("state"), + "state_messages": S("stateMessages", default=[]) >> ForallBend(GcpCloudFunctionsStateMessage.mapping), + "update_time": S("updateTime"), + "upgrade_info": S("upgradeInfo", default={}) >> Bend(GcpUpgradeInfo.mapping), + "url": S("url"), + } + build_config: Optional[GcpBuildConfig] = field(default=None) + create_time: Optional[datetime] = field(default=None) + environment: Optional[str] = field(default=None) + event_trigger: Optional[GcpEventTrigger] = field(default=None) + kms_key_name: Optional[str] = field(default=None) + satisfies_pzs: Optional[bool] = field(default=None) + service_config: Optional[GcpServiceConfig] = field(default=None) + state: Optional[str] = field(default=None) + state_messages: Optional[List[GcpCloudFunctionsStateMessage]] = field(default=None) + update_time: Optional[datetime] = field(default=None) + upgrade_info: Optional[GcpUpgradeInfo] = field(default=None) + url: Optional[str] = field(default=None) + + +resources: List[Type[GcpResource]] = [GcpCloudFunction] diff --git a/plugins/gcp/fix_plugin_gcp/resources/compute.py b/plugins/gcp/fix_plugin_gcp/resources/compute.py index 5f5cd4ae3b..f8aedcd9a3 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/compute.py +++ b/plugins/gcp/fix_plugin_gcp/resources/compute.py @@ -7196,6 +7196,7 @@ class GcpSubnetwork(GcpResource, BaseSubnet): "secondary_ip_ranges": S("secondaryIpRanges", default=[]) >> ForallBend(GcpSubnetworkSecondaryRange.mapping), "stack_type": S("stackType"), "subnetwork_state": S("state"), + "cidr_block": S("ipCidrRange"), } enable_flow_logs: Optional[bool] = field(default=None) external_ipv6_prefix: Optional[str] = field(default=None) diff --git a/plugins/gcp/fix_plugin_gcp/resources/filestore.py b/plugins/gcp/fix_plugin_gcp/resources/filestore.py new file mode 100644 index 0000000000..7f7714fa15 --- /dev/null +++ b/plugins/gcp/fix_plugin_gcp/resources/filestore.py @@ -0,0 +1,327 @@ +from datetime import datetime +import logging +from typing import ClassVar, Dict, Optional, List, Type, Any + +from attr import define, field + +from fix_plugin_gcp.gcp_client import GcpApiSpec +from fix_plugin_gcp.resources.base import GraphBuilder, GcpErrorHandler, GcpResource, GcpDeprecationStatus +from fixlib.baseresources import BaseNetworkShare, ModelReference +from fixlib.json_bender import Bender, S, Bend, ForallBend +from fixlib.types import Json + +log = logging.getLogger("fix.plugins.gcp") + + +service_name = "filestore" + + +@define(eq=False, slots=False) +class GcpFilestoreBackup(GcpResource): + kind: ClassVar[str] = "gcp_filestore_backup" + _kind_display: ClassVar[str] = "GCP Filestore Backup" + _kind_description: ClassVar[str] = ( + "GCP Filestore Backup is a service that allows you to create backups of your Filestore instances." + " It provides a way to protect your data and restore it in case of data loss." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/backups" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "backup", "group": "storage"} + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "backups"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="backups", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "capacity_gb": S("capacityGb"), + "create_time": S("createTime"), + "download_bytes": S("downloadBytes"), + "file_system_protocol": S("fileSystemProtocol"), + "kms_key": S("kmsKey"), + "satisfies_pzi": S("satisfiesPzi"), + "satisfies_pzs": S("satisfiesPzs"), + "source_file_share": S("sourceFileShare"), + "source_instance": S("sourceInstance"), + "source_instance_tier": S("sourceInstanceTier"), + "state": S("state"), + "storage_bytes": S("storageBytes"), + "tags": S("tags", default={}), + } + capacity_gb: Optional[str] = field(default=None) + create_time: Optional[datetime] = field(default=None) + download_bytes: Optional[str] = field(default=None) + file_system_protocol: Optional[str] = field(default=None) + kms_key: Optional[str] = field(default=None) + satisfies_pzi: Optional[bool] = field(default=None) + satisfies_pzs: Optional[bool] = field(default=None) + source_file_share: Optional[str] = field(default=None) + source_instance: Optional[str] = field(default=None) + source_instance_tier: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + storage_bytes: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpNfsExportOptions: + kind: ClassVar[str] = "gcp_nfs_export_options" + mapping: ClassVar[Dict[str, Bender]] = { + "access_mode": S("accessMode"), + "anon_gid": S("anonGid"), + "anon_uid": S("anonUid"), + "ip_ranges": S("ipRanges", default=[]), + "squash_mode": S("squashMode"), + } + access_mode: Optional[str] = field(default=None) + anon_gid: Optional[str] = field(default=None) + anon_uid: Optional[str] = field(default=None) + ip_ranges: Optional[List[str]] = field(default=None) + squash_mode: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFileShareConfig: + kind: ClassVar[str] = "gcp_file_share_config" + mapping: ClassVar[Dict[str, Bender]] = { + "capacity_gb": S("capacityGb"), + "name": S("name"), + "nfs_export_options": S("nfsExportOptions", default=[]) >> ForallBend(GcpNfsExportOptions.mapping), + "source_backup": S("sourceBackup"), + } + capacity_gb: Optional[str] = field(default=None) + name: Optional[str] = field(default=None) + nfs_export_options: Optional[List[GcpNfsExportOptions]] = field(default=None) + source_backup: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpNetworkConfig: + kind: ClassVar[str] = "gcp_network_config" + mapping: ClassVar[Dict[str, Bender]] = { + "connect_mode": S("connectMode"), + "ip_addresses": S("ipAddresses", default=[]), + "modes": S("modes", default=[]), + "network": S("network"), + "reserved_ip_range": S("reservedIpRange"), + } + connect_mode: Optional[str] = field(default=None) + ip_addresses: Optional[List[str]] = field(default=None) + modes: Optional[List[str]] = field(default=None) + network: Optional[str] = field(default=None) + reserved_ip_range: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpPerformanceConfig: + kind: ClassVar[str] = "gcp_performance_config" + mapping: ClassVar[Dict[str, Bender]] = { + "fixed_iops": S("fixedIops", "maxReadIops"), + "iops_per_tb": S("iopsPerTb", "maxReadIopsPerTb"), + } + fixed_iops: Optional[str] = field(default=None) + iops_per_tb: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpPerformanceLimits: + kind: ClassVar[str] = "gcp_performance_limits" + mapping: ClassVar[Dict[str, Bender]] = { + "max_read_iops": S("maxReadIops"), + "max_read_throughput_bps": S("maxReadThroughputBps"), + "max_write_iops": S("maxWriteIops"), + "max_write_throughput_bps": S("maxWriteThroughputBps"), + } + max_read_iops: Optional[str] = field(default=None) + max_read_throughput_bps: Optional[str] = field(default=None) + max_write_iops: Optional[str] = field(default=None) + max_write_throughput_bps: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpReplicaConfig: + kind: ClassVar[str] = "gcp_replica_config" + mapping: ClassVar[Dict[str, Bender]] = { + "last_active_sync_time": S("lastActiveSyncTime"), + "peer_instance": S("peerInstance"), + "state": S("state"), + "state_reasons": S("stateReasons", default=[]), + } + last_active_sync_time: Optional[datetime] = field(default=None) + peer_instance: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + state_reasons: Optional[List[str]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpReplication: + kind: ClassVar[str] = "gcp_replication" + mapping: ClassVar[Dict[str, Bender]] = { + "replicas": S("replicas", default=[]) >> ForallBend(GcpReplicaConfig.mapping), + "role": S("role"), + } + replicas: Optional[List[GcpReplicaConfig]] = field(default=None) + role: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFilestoreInstance(GcpResource, BaseNetworkShare): + kind: ClassVar[str] = "gcp_filestore_instance" + _kind_display: ClassVar[str] = "GCP Filestore Instance" + _kind_description: ClassVar[str] = ( + "GCP Filestore Instance is a fully managed file storage service that provides scalable and high-performance" + " file systems for applications running on Google Cloud." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/instances" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "network_share", "group": "storage"} + _reference_kinds: ClassVar[ModelReference] = { + "successors": { + "default": [ + "gcp_filestore_instance_snapshot", + ], + }, + } + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="instances", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "configurable_performance_enabled": S("configurablePerformanceEnabled"), + "create_time": S("createTime"), + "deletion_protection_enabled": S("deletionProtectionEnabled"), + "deletion_protection_reason": S("deletionProtectionReason"), + "etag": S("etag"), + "file_shares": S("fileShares", default=[]) >> ForallBend(GcpFileShareConfig.mapping), + "kms_key_name": S("kmsKeyName"), + "networks": S("networks", default=[]) >> ForallBend(GcpNetworkConfig.mapping), + "performance_config": S("performanceConfig", default={}) >> Bend(GcpPerformanceConfig.mapping), + "performance_limits": S("performanceLimits", default={}) >> Bend(GcpPerformanceLimits.mapping), + "protocol": S("protocol"), + "replication": S("replication", default={}) >> Bend(GcpReplication.mapping), + "satisfies_pzi": S("satisfiesPzi"), + "satisfies_pzs": S("satisfiesPzs"), + "state": S("state"), + "status_message": S("statusMessage"), + "suspension_reasons": S("suspensionReasons", default=[]), + "tags": S("tags", default={}), + "tier": S("tier"), + } + configurable_performance_enabled: Optional[bool] = field(default=None) + create_time: Optional[datetime] = field(default=None) + deletion_protection_enabled: Optional[bool] = field(default=None) + deletion_protection_reason: Optional[str] = field(default=None) + etag: Optional[str] = field(default=None) + file_shares: Optional[List[GcpFileShareConfig]] = field(default=None) + kms_key_name: Optional[str] = field(default=None) + networks: Optional[List[GcpNetworkConfig]] = field(default=None) + performance_config: Optional[GcpPerformanceConfig] = field(default=None) + performance_limits: Optional[GcpPerformanceLimits] = field(default=None) + protocol: Optional[str] = field(default=None) + replication: Optional[GcpReplication] = field(default=None) + satisfies_pzi: Optional[bool] = field(default=None) + satisfies_pzs: Optional[bool] = field(default=None) + state: Optional[str] = field(default=None) + status_message: Optional[str] = field(default=None) + suspension_reasons: Optional[List[str]] = field(default=None) + tier: Optional[str] = field(default=None) + + @classmethod + def called_collect_apis(cls) -> List[GcpApiSpec]: + return [ + cls.api_spec, + GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances", "snapshots"], + action="list", + request_parameter={"parent": "projects/{project}/locations/{location}/instances/{instanceId}"}, + request_parameter_in={"project", "location", "instanceId"}, + response_path="snapshots", + response_regional_sub_path=None, + ), + ] + + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: + def collect_snapshots() -> None: + spec = GcpApiSpec( + service="file", + version="v1", + accessors=["projects", "locations", "instances", "snapshots"], + action="list", + request_parameter={"parent": f"{self.id}"}, + request_parameter_in=set(), + response_path="snapshots", + response_regional_sub_path=None, + ) + with GcpErrorHandler( + spec.action, + graph_builder.error_accumulator, + spec.service, + graph_builder.region.safe_name if graph_builder.region else None, + set(), + f" in {graph_builder.project.id} kind {GcpFilestoreInstanceSnapshot.kind}", + ): + items = graph_builder.client.list(spec) + snapshots = GcpFilestoreInstanceSnapshot.collect(items, graph_builder) + for snapshot in snapshots: + graph_builder.add_edge(self, node=snapshot) + log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFilestoreInstanceSnapshot.kind}") + + graph_builder.submit_work(collect_snapshots) + + +@define(eq=False, slots=False) +class GcpFilestoreInstanceSnapshot(GcpResource): + # collected via GcpFilestoreInstance() + kind: ClassVar[str] = "gcp_filestore_instance_snapshot" + _kind_display: ClassVar[str] = "GCP Filestore Snapshot" + _kind_description: ClassVar[str] = ( + "GCP Filestore Snapshot is a point-in-time copy of a Filestore instance, allowing you to restore" + " data to a previous state or create new instances from the snapshot." + ) + _docs_url: ClassVar[str] = "https://cloud.google.com/filestore/docs/snapshots" + _kind_service: ClassVar[Optional[str]] = "filestore" + _metadata: ClassVar[Dict[str, Any]] = {"icon": "snapshot", "group": "storage"} + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "create_time": S("createTime"), + "filesystem_used_bytes": S("filesystemUsedBytes"), + "state": S("state"), + "tags": S("tags", default={}), + } + create_time: Optional[datetime] = field(default=None) + filesystem_used_bytes: Optional[str] = field(default=None) + state: Optional[str] = field(default=None) + + +resources: List[Type[GcpResource]] = [GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreInstanceSnapshot] diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py new file mode 100644 index 0000000000..5909408ce2 --- /dev/null +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -0,0 +1,291 @@ +from datetime import datetime +import logging +from typing import ClassVar, Dict, Optional, List, Any, Type + +from attr import define, field + +from fix_plugin_gcp.gcp_client import GcpApiSpec +from fix_plugin_gcp.resources.base import GcpErrorHandler, GcpResource, GcpDeprecationStatus, GraphBuilder +from fixlib.baseresources import BaseDatabase, ModelReference +from fixlib.json_bender import Bender, S, Bend, MapDict +from fixlib.types import Json + +log = logging.getLogger("fix.plugins.gcp") + + +# https://cloud.google.com/firestore/docs + +service_name = "firestore" + + +@define(eq=False, slots=False) +class GcpFirestoreCmekConfig: + kind: ClassVar[str] = "gcp_firestore_cmek_config" + mapping: ClassVar[Dict[str, Bender]] = { + "active_key_version": S("activeKeyVersion", default=[]), + "kms_key_name": S("kmsKeyName"), + } + active_key_version: Optional[List[str]] = field(default=None) + kms_key_name: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreSourceInfo: + kind: ClassVar[str] = "gcp_firestore_source_info" + mapping: ClassVar[Dict[str, Bender]] = {"backup": S("backup", "backup"), "operation": S("operation")} + backup: Optional[str] = field(default=None) + operation: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreDatabase(GcpResource, BaseDatabase): + kind: ClassVar[str] = "gcp_firestore_database" + _kind_display: ClassVar[str] = "GCP Firestore Database" + _kind_description: ClassVar[str] = ( + "A Firestore Database in GCP, which is a scalable NoSQL cloud database to store and sync data for client- and server-side development." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "database", "group": "storage"} + _reference_kinds: ClassVar[ModelReference] = { + "successors": { + "default": [ + "gcp_firestore_document", + ], + }, + } + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases"], + action="list", + request_parameter={"parent": "projects/{project}"}, + request_parameter_in={"project"}, + response_path="databases", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "app_engine_integration_mode": S("appEngineIntegrationMode"), + "cmek_config": S("cmekConfig", default={}) >> Bend(GcpFirestoreCmekConfig.mapping), + "concurrency_mode": S("concurrencyMode"), + "create_time": S("createTime"), + "delete_protection_state": S("deleteProtectionState"), + "delete_time": S("deleteTime"), + "earliest_version_time": S("earliestVersionTime"), + "etag": S("etag"), + "key_prefix": S("keyPrefix"), + "location_id": S("locationId"), + "point_in_time_recovery_enablement": S("pointInTimeRecoveryEnablement"), + "previous_id": S("previousId"), + "source_info": S("sourceInfo", default={}) >> Bend(GcpFirestoreSourceInfo.mapping), + "type": S("type"), + "uid": S("uid"), + "update_time": S("updateTime"), + "version_retention_period": S("versionRetentionPeriod"), + } + app_engine_integration_mode: Optional[str] = field(default=None) + cmek_config: Optional[GcpFirestoreCmekConfig] = field(default=None) + concurrency_mode: Optional[str] = field(default=None) + create_time: Optional[datetime] = field(default=None) + delete_protection_state: Optional[str] = field(default=None) + delete_time: Optional[datetime] = field(default=None) + earliest_version_time: Optional[datetime] = field(default=None) + etag: Optional[str] = field(default=None) + key_prefix: Optional[str] = field(default=None) + location_id: Optional[str] = field(default=None) + point_in_time_recovery_enablement: Optional[str] = field(default=None) + previous_id: Optional[str] = field(default=None) + source_info: Optional[GcpFirestoreSourceInfo] = field(default=None) + type: Optional[str] = field(default=None) + uid: Optional[str] = field(default=None) + update_time: Optional[datetime] = field(default=None) + version_retention_period: Optional[str] = field(default=None) + + @classmethod + def called_collect_apis(cls) -> List[GcpApiSpec]: + return [ + cls.api_spec, + GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases", "documents"], + action="list", + request_parameter={"parent": "projects/{project}/databases/{databaseId}/documents"}, + request_parameter_in={"project", "databaseId"}, + response_path="documents", + response_regional_sub_path=None, + ), + ] + + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: + def collect_documents() -> None: + spec = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases", "documents"], + action="list", + request_parameter={"parent": f"{self.id}/documents"}, + request_parameter_in=set(), + response_path="documents", + response_regional_sub_path=None, + ) + with GcpErrorHandler( + spec.action, + graph_builder.error_accumulator, + spec.service, + graph_builder.region.safe_name if graph_builder.region else None, + set(), + f" in {graph_builder.project.id} kind {GcpFirestoreDocument.kind}", + ): + items = graph_builder.client.list(spec) + documents = GcpFirestoreDocument.collect(items, graph_builder) + for document in documents: + graph_builder.add_edge(self, node=document) + log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFirestoreDocument.kind}") + + graph_builder.submit_work(collect_documents) + + +@define(eq=False, slots=False) +class GcpArrayValue: + kind: ClassVar[str] = "gcp_array_value" + mapping: ClassVar[Dict[str, Bender]] = {"values": S("values", default=[])} + values: Optional[List[Any]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpLatLng: + kind: ClassVar[str] = "gcp_lat_lng" + mapping: ClassVar[Dict[str, Bender]] = {"latitude": S("latitude"), "longitude": S("longitude")} + latitude: Optional[float] = field(default=None) + longitude: Optional[float] = field(default=None) + + +@define(eq=False, slots=False) +class GcpMapValue: + kind: ClassVar[str] = "gcp_map_value" + mapping: ClassVar[Dict[str, Bender]] = {"fields": S("fields", default={})} + fields: Optional[Dict[str, Any]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpValue: + kind: ClassVar[str] = "gcp_value" + mapping: ClassVar[Dict[str, Bender]] = { + "array_value": S("arrayValue", default={}) >> Bend(GcpArrayValue.mapping), + "boolean_value": S("booleanValue"), + "bytes_value": S("bytesValue"), + "double_value": S("doubleValue"), + "geo_point_value": S("geoPointValue", default={}) >> Bend(GcpLatLng.mapping), + "integer_value": S("integerValue"), + "map_value": S("mapValue", default={}) >> Bend(GcpMapValue.mapping), + "null_value": S("nullValue"), + "reference_value": S("referenceValue"), + "string_value": S("stringValue"), + "timestamp_value": S("timestampValue"), + } + array_value: Optional[GcpArrayValue] = field(default=None) + boolean_value: Optional[bool] = field(default=None) + bytes_value: Optional[str] = field(default=None) + double_value: Optional[float] = field(default=None) + geo_point_value: Optional[GcpLatLng] = field(default=None) + integer_value: Optional[str] = field(default=None) + map_value: Optional[GcpMapValue] = field(default=None) + null_value: Optional[str] = field(default=None) + reference_value: Optional[str] = field(default=None) + string_value: Optional[str] = field(default=None) + timestamp_value: Optional[datetime] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreDocument(GcpResource): + kind: ClassVar[str] = "gcp_firestore_document" + _kind_display: ClassVar[str] = "GCP Firestore Document" + _kind_description: ClassVar[str] = ( + "A Firestore Document in GCP, representing a single document in a Firestore database, which can contain fields and subcollections." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "database", "group": "storage"} + # collected via GcpFirestoreDatabase() + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "create_time": S("createTime"), + "fields": S("fields", default={}) >> MapDict(value_bender=Bend(GcpValue.mapping)), + "update_time": S("updateTime"), + } + create_time: Optional[datetime] = field(default=None) + fields: Optional[Dict[str, GcpValue]] = field(default=None) + update_time: Optional[datetime] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreStats: + kind: ClassVar[str] = "gcp_firestore_stats" + mapping: ClassVar[Dict[str, Bender]] = { + "document_count": S("documentCount"), + "index_count": S("indexCount"), + "size_bytes": S("sizeBytes"), + } + document_count: Optional[str] = field(default=None) + index_count: Optional[str] = field(default=None) + size_bytes: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreBackup(GcpResource): + kind: ClassVar[str] = "gcp_firestore_backup" + _kind_display: ClassVar[str] = "GCP Firestore Backup" + _kind_description: ClassVar[str] = ( + "A Firestore Backup in GCP, which provides a way to back up and restore Firestore databases to protect against data loss." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "backup", "group": "storage"} + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "locations", "backups"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="backups", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "database_name": S("database"), + "database_uid": S("databaseUid"), + "expire_time": S("expireTime"), + "snapshot_time": S("snapshotTime"), + "state": S("state"), + "backup_stats": S("stats", default={}) >> Bend(GcpFirestoreStats.mapping), + } + database_name: Optional[str] = field(default=None) + database_uid: Optional[str] = field(default=None) + expire_time: Optional[datetime] = field(default=None) + snapshot_time: Optional[datetime] = field(default=None) + state: Optional[str] = field(default=None) + backup_stats: Optional[GcpFirestoreStats] = field(default=None) + + +resources: List[Type[GcpResource]] = [GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup] diff --git a/plugins/gcp/fix_plugin_gcp/resources/storage.py b/plugins/gcp/fix_plugin_gcp/resources/storage.py index f907077642..e453bba658 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/storage.py +++ b/plugins/gcp/fix_plugin_gcp/resources/storage.py @@ -7,7 +7,7 @@ from fix_plugin_gcp.resources.base import GcpResource, GcpDeprecationStatus, get_client from fixlib.baseresources import BaseBucket from fixlib.graph import Graph -from fixlib.json_bender import Bender, S, Bend, ForallBend +from fixlib.json_bender import Bender, S, Bend, ForallBend, AsBool service_name = "storage" @@ -391,6 +391,7 @@ class GcpBucket(GcpResource, BaseBucket): "updated": S("updated"), "versioning_enabled": S("versioning", "enabled"), "bucket_website": S("website", default={}) >> Bend(GcpWebsite.mapping), + "encryption_enabled": S("encryption", "defaultKmsKeyName") >> AsBool(), } acl: Optional[List[GcpBucketAccessControl]] = field(default=None) autoclass: Optional[GcpAutoclass] = field(default=None) @@ -415,7 +416,6 @@ class GcpBucket(GcpResource, BaseBucket): updated: Optional[datetime] = field(default=None) bucket_website: Optional[GcpWebsite] = field(default=None) requester_pays: Optional[bool] = field(default=None) - versioning_enabled: Optional[bool] = field(default=None) lifecycle_rule: List[GcpRule] = field(factory=list) def pre_delete(self, graph: Graph) -> bool: diff --git a/plugins/gcp/test/aiplatform_test.py b/plugins/gcp/test/aiplatform_test.py new file mode 100644 index 0000000000..fee761a960 --- /dev/null +++ b/plugins/gcp/test/aiplatform_test.py @@ -0,0 +1,16 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.aiplatform import resources + + +def test_gcp_aiplatform_resources(random_builder: GraphBuilder) -> None: + file_path = os.path.join(os.path.dirname(__file__), "files", "aiplatform_resources.json") + with open(file_path, "r") as f: + data = json.load(f) + + for resource, res_class in zip(data["resources"], resources): + res_class.collect(raw=[resource], builder=random_builder) + collected = random_builder.nodes(clazz=res_class) + assert len(collected) == 1 diff --git a/plugins/gcp/test/files/aiplatform_resources.json b/plugins/gcp/test/files/aiplatform_resources.json new file mode 100644 index 0000000000..2d0435af52 --- /dev/null +++ b/plugins/gcp/test/files/aiplatform_resources.json @@ -0,0 +1,740 @@ +{ + "resources": [ + { + "id": "projects/1234567890/locations/us-central1/models/my-model", + "tags": { + "env": "prod" + }, + "name": "projects/1234567890/locations/us-central1/models/my-model", + "ctime": "2024-07-26T12:00:00Z", + "description": "My trained model", + "link": "https://console.cloud.google.com/vertex-ai/locations/us-central1/models/my-model", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE", + "end_time": null + }, + "artifact_uri": "gs://my-bucket/model-artifact", + "base_model_source": { + "model_garden_source": null, + "genie_source": "projects/my-project/locations/us-central1/models/my-base-model@1" + }, + "container_spec": { + "image_uri": "us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest", + "command": [], + "args": [], + "env": [], + "ports": [], + "predict_route": "", + "health_route": "", + "grpc_ports": [], + "startup_probe": null, + "health_probe": null, + "deployment_timeout": null, + "shared_memory_size_mb": null + }, + "create_time": "2024-07-26T12:00:00Z", + "data_stats": { + "training_data_items_count": "1000", + "validation_data_items_count": "200", + "test_data_items_count": "100", + "training_annotations_count": null, + "validation_annotations_count": null, + "test_annotations_count": null + }, + "endpoint_deployed_model_refs": [ + { + "deployed_model_id": "12345", + "endpoint": "projects/.../locations/.../endpoints/my-endpoint" + } + ], + "display_name": "My Model", + "encryption_spec": "projects/1234567890/locations/global/keyRings/my-keyring/cryptoKeys/my-key", + "etag": "abcdef123456", + "explanation_spec": { + "_metadata": { + "inputs": { + "input_1": { + "input_tensor_name": "input_tensor" + } + } + }, + "parameters": { + "sampled_shapley_attribution": null, + "integrated_gradients_attribution": null, + "xrai_attribution": null, + "examples": null, + "top_k": 10 + } + } + }, + { + "id": "projects/12345/locations/us-central1/datasets/my-dataset", + "tags": { + "purpose": "training" + }, + "name": "projects/12345/locations/us-central1/datasets/my-dataset", + "ctime": "2024-08-07T15:00:00Z", + "description": "Dataset for image classification", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-07T15:00:00Z", + "data_item_count": "10000", + "display_name": "Image Classification Dataset", + "encryption_spec": null, + "etag": "abcdef123456", + "_metadata": {}, + "metadata_artifact": null, + "metadata_schema_uri": "gs://my-bucket/dataset_schema.json", + "model_reference": null, + "saved_queries": [], + "update_time": "2024-08-07T16:00:00Z" + }, + { + "id": "projects/12345/locations/us-central1/datasets/my-dataset/versions/v1", + "tags": {}, + "name": "projects/12345/locations/us-central1/datasets/my-dataset/versions/v1", + "ctime": "2024-08-07T16:30:00Z", + "description": "Version 1 of the dataset", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "big_query_dataset_name": null, + "create_time": "2024-08-07T16:30:00Z", + "display_name": "v1", + "etag": "abcdef123456", + "_metadata": {}, + "model_reference": null, + "update_time": "2024-08-07T16:30:00Z" + }, + { + "id": "projects/12345/locations/us-central1/endpoints/my-endpoint", + "tags": { + "team": "ml-team" + }, + "name": "projects/12345/locations/us-central1/endpoints/my-endpoint", + "ctime": "2024-08-08T10:00:00Z", + "description": "Endpoint for image classification model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-08T10:00:00Z", + "deployed_models": [ + { + "automatic_resources": null, + "create_time": "2024-08-08T10:15:00Z", + "dedicated_resources": { + "machine_spec": { + "machine_type": "n1-standard-2", + "accelerator_type": null, + "accelerator_count": null + }, + "min_replica_count": 1, + "max_replica_count": 3, + "autoscaling_metric_specs": [] + }, + "deployed_index_auth_config": null, + "deployment_group": null, + "disable_container_logging": false, + "disable_explanations": false, + "display_name": "Deployed Model 1", + "enable_access_logging": true, + "explanation_spec": null, + "id": "67890", + "model": "projects/12345/locations/us-central1/models/my-model", + "model_version_id": "1", + "private_endpoints": null, + "service_account": "vertex-ai@my-project.iam.gserviceaccount.com", + "shared_resources": null + } + ], + "display_name": "Image Classifier Endpoint", + "enable_private_service_connect": false, + "encryption_spec": null, + "etag": "abcdef123456", + "model_deployment_monitoring_job": null, + "network": null, + "predict_request_response_logging_config": null, + "private_service_connect_config": null, + "traffic_split": {}, + "update_time": "2024-08-08T11:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/schedules/my-schedule", + "tags": { + "owner": "john.doe" + }, + "name": "projects/my-project/locations/us-central1/schedules/my-schedule", + "ctime": "2024-08-09T14:00:00Z", + "description": "Schedule for daily batch prediction", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "allow_queueing": false, + "catch_up": false, + "create_pipeline_job_request": { + "parent": "projects/my-project/locations/us-central1", + "pipeline_job": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline", + "pipeline_job_id": "my-pipeline-job-id" + }, + "create_time": "2024-08-09T14:00:00Z", + "cron": "0 0 * * *", + "display_name": "Daily Prediction", + "end_time": null, + "last_pause_time": null, + "last_resume_time": null, + "last_scheduled_run_response": { + "run_response": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "scheduled_run_time": "2024-08-09T00:00:00Z" + }, + "max_concurrent_run_count": 1, + "max_run_count": null, + "next_run_time": "2024-08-10T00:00:00Z", + "start_time": "2024-08-09T14:00:00Z", + "started_run_count": 10, + "state": "ACTIVE", + "update_time": "2024-08-09T14:30:00Z" + }, + { + "id": "projects/12345/locations/us-central1/featureGroups/my-feature-group", + "tags": { + "department": "sales" + }, + "name": "projects/12345/locations/us-central1/featureGroups/my-feature-group", + "ctime": "2024-08-10T09:00:00Z", + "description": "Feature group for customer data", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "big_query": { + "big_query_source": "bq://my-project.my_dataset.my_table", + "entity_id_columns": [ + "customer_id" + ] + }, + "create_time": "2024-08-10T09:00:00Z", + "etag": "abcdef123456", + "update_time": "2024-08-10T09:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/featureGroups/my-feature-group/features/customer_age", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featureGroups/my-feature-group/features/customer_age", + "ctime": "2024-08-10T10:00:00Z", + "description": "Age of the customer", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-10T10:00:00Z", + "disable_monitoring": false, + "etag": "abcdef123456", + "monitoring_stats_anomalies": [], + "point_of_contact": "data-scientist@example.com", + "update_time": "2024-08-10T10:30:00Z", + "value_type": "INT64", + "version_column_name": null + }, + { + "id": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "ctime": "2024-08-11T08:00:00Z", + "description": "My Feature Store", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-11T08:00:00Z", + "encryption_spec": null, + "etag": "abcdef123456", + "online_serving_config": { + "fixed_node_count": 2, + "scaling": { + "min_node_count": 1, + "max_node_count": 5, + "cpu_utilization_target": 70 + } + }, + "online_storage_ttl_days": 7, + "state": "ACTIVE", + "update_time": "2024-08-11T08:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/trainingPipelines/my-training-pipeline", + "tags": { + "model_type": "image_classifier" + }, + "name": "projects/my-project/locations/us-central1/trainingPipelines/my-training-pipeline", + "ctime": "2024-08-12T12:00:00Z", + "description": "Training pipeline for image classification", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-12T12:00:00Z", + "display_name": "Image Classifier Training", + "encryption_spec": null, + "end_time": "2024-08-12T14:00:00Z", + "error": null, + "input_data_config": { + "dataset_id": "my-dataset", + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1 + }, + "gcs_destination": "gs://my-bucket/training_data", + "annotation_schema_uri": null, + "annotations_filter": null, + "bigquery_destination": null, + "filter_split": null, + "persist_ml_use_assignment": null, + "predefined_split": null, + "saved_query_id": null, + "stratified_split": null, + "timestamp_split": null + }, + "model_id": null, + "model_to_upload": "projects/my-project/locations/us-central1/models/my-trained-model", + "parent_model": null, + "start_time": "2024-08-12T12:30:00Z", + "state": "SUCCEEDED", + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_classification_1.0.0.yaml", + "training_task_inputs": {}, + "training_task_metadata": {}, + "update_time": "2024-08-12T14:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "ctime": "2024-08-13T09:00:00Z", + "description": "Batch prediction job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "completion_stats": { + "successful_count": "1000", + "failed_count": "10", + "incomplete_count": "0", + "successful_forecast_point_count": null + }, + "create_time": "2024-08-13T09:00:00Z", + "dedicated_resources": null, + "disable_container_logging": false, + "display_name": "My Batch Prediction", + "encryption_spec": null, + "end_time": "2024-08-13T10:00:00Z", + "error": null, + "explanation_spec": null, + "generate_explanation": false, + "input_config": { + "gcs_source": { + "uris": [ + "gs://my-bucket/input-data.jsonl" + ] + }, + "instances_format": "jsonl", + "bigquery_source": null + }, + "instance_config": null, + "manual_batch_tuning_parameters": null, + "model": "projects/my-project/locations/us-central1/models/my-model", + "model_parameters": {}, + "model_version_id": "1", + "output_config": { + "gcs_destination": "gs://my-bucket/output-predictions", + "predictions_format": "jsonl", + "bigquery_destination": null + }, + "output_info": { + "bigquery_output_dataset": null, + "bigquery_output_table": null, + "gcs_output_directory": "gs://my-bucket/output-predictions" + }, + "partial_failures": [], + "resources_consumed": 1.5, + "service_account": null, + "start_time": "2024-08-13T09:15:00Z", + "state": "SUCCEEDED", + "unmanaged_container_model": null, + "update_time": "2024-08-13T10:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/models/my-model/evaluations/eval123", + "tags": {}, + "name": "projects/my-project/locations/us-central1/models/my-model/evaluations/eval123", + "ctime": "2024-08-14T11:00:00Z", + "description": "Evaluation of model performance", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "annotation_schema_uri": null, + "create_time": "2024-08-14T11:00:00Z", + "data_item_schema_uri": null, + "display_name": "Model Eval 1", + "explanation_specs": [], + "_metadata": {}, + "metrics": { + "accuracy": 0.95, + "precision": 0.92 + }, + "metrics_schema_uri": null, + "model_explanation": null, + "slice_dimensions": [] + }, + { + "id": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "ctime": "2024-08-11T08:00:00Z", + "description": "My Feature Store", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-11T08:00:00Z", + "encryption_spec": null, + "etag": "abcdef123456", + "online_serving_config": { + "fixed_node_count": 2 + }, + "online_storage_ttl_days": 7, + "state": "ACTIVE", + "update_time": "2024-08-11T08:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/hyperparameterTuningJobs/my-tuning-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/hyperparameterTuningJobs/my-tuning-job", + "ctime": "2024-08-15T15:00:00Z", + "description": "Hyperparameter tuning job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-15T15:00:00Z", + "display_name": "My Tuning Job", + "encryption_spec": null, + "end_time": "2024-08-15T17:00:00Z", + "error": null, + "max_failed_trial_count": 3, + "max_trial_count": 10, + "parallel_trial_count": 2, + "start_time": "2024-08-15T15:15:00Z", + "state": "SUCCEEDED", + "study_spec": { + "algorithm": "ALGORITHM_UNSPECIFIED", + "convex_automated_stopping_spec": null, + "decay_curve_stopping_spec": null, + "measurement_selection_type": "BEST_MEASUREMENT", + "median_automated_stopping_spec": null, + "metrics": [ + { + "goal": "MAXIMIZE", + "metric_id": "accuracy", + "safety_config": null + } + ], + "observation_noise": "OBSERVATION_NOISE_UNSPECIFIED", + "parameters": [], + "study_stopping_config": null + }, + "trial_job_spec": null, + "trials": [], + "update_time": "2024-08-15T17:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/customJobs/my-custom-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/customJobs/my-custom-job", + "ctime": "2024-08-16T10:00:00Z", + "description": "Custom training job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-16T10:00:00Z", + "display_name": "My Custom Training", + "encryption_spec": null, + "end_time": "2024-08-16T12:00:00Z", + "error": null, + "job_spec": { + "base_output_directory": "gs://my-bucket/custom-job-output", + "enable_dashboard_access": true, + "enable_web_access": null, + "experiment": null, + "experiment_run": null, + "models": [], + "network": null, + "persistent_resource_id": null, + "protected_artifact_location_id": null, + "reserved_ip_ranges": [], + "scheduling": { + "disable_retries": null, + "restart_job_on_worker_restart": null, + "timeout": null + }, + "service_account": "custom-job-sa@my-project.iam.gserviceaccount.com", + "tensorboard": null, + "worker_pool_specs": [ + { + "container_spec": { + "image_uri": "gcr.io/my-project/my-custom-training-image", + "args": [], + "command": [], + "env": [] + }, + "python_package_spec": null, + "machine_spec": null, + "disk_spec": null, + "nfs_mounts": [], + "replica_count": "1" + } + ] + }, + "start_time": "2024-08-16T10:15:00Z", + "state": "SUCCEEDED", + "update_time": "2024-08-16T12:00:00Z", + "web_access_uris": {} + }, + { + "id": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline-job", + "ctime": "2024-08-17T14:00:00Z", + "description": "My pipeline job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-17T14:00:00Z", + "display_name": "My Pipeline", + "encryption_spec": null, + "end_time": "2024-08-17T16:00:00Z", + "error": null, + "job_detail": { + "pipeline_context": {}, + "pipeline_run_context": {}, + "task_details": [] + }, + "network": null, + "pipeline_spec": {}, + "preflight_validations": null, + "reserved_ip_ranges": [], + "runtime_config": { + "failure_policy": null, + "gcs_output_directory": null, + "input_artifacts": {}, + "parameter_values": {}, + "parameters": {} + }, + "schedule_name": null, + "service_account": "pipeline-sa@my-project.iam.gserviceaccount.com", + "start_time": "2024-08-17T14:30:00Z", + "state": "SUCCEEDED", + "template_metadata": "1", + "template_uri": "gs://my-bucket/pipeline.json", + "update_time": "2024-08-17T16:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/tensorboards/my-tensorboard", + "tags": {}, + "name": "projects/my-project/locations/us-central1/tensorboards/my-tensorboard", + "ctime": "2024-08-18T11:30:00Z", + "description": "My Tensorboard instance", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "blob_storage_path_prefix": "gs://my-bucket/tensorboard-logs", + "create_time": "2024-08-18T11:30:00Z", + "display_name": "My Tensorboard", + "encryption_spec": null, + "etag": "abcdef123456", + "isDefault": false, + "run_count": 5, + "satisfiesPzi": false, + "satisfiesPzs": false, + "update_time": "2024-08-18T12:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/indexes/my-index", + "tags": {}, + "name": "projects/my-project/locations/us-central1/indexes/my-index", + "ctime": "2024-08-19T16:00:00Z", + "description": "Index for vector similarity search", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-19T16:00:00Z", + "deployed_indexes": [], + "display_name": "My Vector Index", + "encryption_spec": null, + "etag": "abcdef123456", + "index_stats": { + "vectors_count": "10000", + "shards_count": 4, + "sparse_vectors_count": null + }, + "index_update_method": "BATCH_UPDATE", + "_metadata": {}, + "metadata_schema_uri": "gs://my-bucket/index_metadata_schema.json", + "update_time": "2024-08-19T16:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/indexEndpoints/my-index-endpoint", + "tags": {}, + "name": "projects/my-project/locations/us-central1/indexEndpoints/my-index-endpoint", + "ctime": "2024-08-20T09:00:00Z", + "description": "Endpoint for vector similarity search", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-20T09:00:00Z", + "deployed_indexes": [ + { + "automatic_resources": null, + "deployed_index_auth_config": null, + "deployment_group": null, + "create_time": "2024-08-20T09:15:00Z", + "dedicated_resources": { + "machine_spec": { + "machine_type": "n1-standard-4", + "accelerator_type": null, + "accelerator_count": null, + "tpu_topology": null + }, + "min_replica_count": 2, + "max_replica_count": 4, + "autoscaling_metric_specs": [] + }, + "display_name": "My Deployed Index", + "enable_access_logging": null, + "id": "deployed-index-123", + "index": "projects/my-project/locations/us-central1/indexes/my-index", + "index_sync_time": "2024-08-20T09:20:00Z", + "private_endpoints": null, + "reserved_ip_ranges": [] + } + ], + "display_name": "My Index Endpoint", + "enable_private_service_connect": false, + "encryption_spec": null, + "etag": "abcdef123456", + "network": null, + "private_service_connect_config": null, + "public_endpoint_domain_name": "my-index-endpoint.aiplatform.googleusercontent.com", + "public_endpoint_enabled": true, + "update_time": "2024-08-20T09:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/modelDeploymentMonitoringJobs/my-monitoring-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/modelDeploymentMonitoringJobs/my-monitoring-job", + "ctime": "2024-08-21T11:00:00Z", + "description": "Monitoring job for deployed model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "analysis_instance_schema_uri": null, + "bigquery_tables": [], + "create_time": "2024-08-21T11:00:00Z", + "display_name": "My Monitoring Job", + "enable_monitoring_pipeline_logs": false, + "encryption_spec": null, + "endpoint": "projects/my-project/locations/us-central1/endpoints/my-endpoint", + "error": null, + "latest_monitoring_pipeline_metadata": null, + "log_ttl": null, + "logging_sampling_strategy": null, + "model_deployment_monitoring_objective_configs": [], + "model_deployment_monitoring_schedule_config": null, + "model_monitoring_alert_config": null, + "next_schedule_time": null, + "predict_instance_schema_uri": null, + "sample_predict_instance": {}, + "schedule_state": null, + "state": "JOB_STATE_UNSPECIFIED", + "stats_anomalies_base_directory": null, + "update_time": "2024-08-21T11:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/tuningJobs/my-tuning-job-2", + "tags": { + "experiment_id": "exp123" + }, + "name": "projects/my-project/locations/us-central1/tuningJobs/my-tuning-job-2", + "ctime": "2024-08-22T13:00:00Z", + "description": "Tuning job for a language model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "base_model": "projects/my-project/locations/us-central1/models/my-base-model", + "create_time": "2024-08-22T13:00:00Z", + "encryption_spec": null, + "end_time": "2024-08-22T15:00:00Z", + "error": null, + "experiment": "projects/my-project/locations/us-central1/experiments/exp123", + "start_time": "2024-08-22T13:15:00Z", + "state": "SUCCEEDED", + "supervised_tuning_spec": null, + "tuned_model": null, + "tuned_model_display_name": null, + "tuning_data_stats": null, + "update_time": "2024-08-22T15:00:00Z" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/cloudfunctions.json b/plugins/gcp/test/files/cloudfunctions.json new file mode 100644 index 0000000000..072332ebd4 --- /dev/null +++ b/plugins/gcp/test/files/cloudfunctions.json @@ -0,0 +1,159 @@ +{ + "functions": [ + { + "id": "projects/sample-project/locations/us-central1/functions/my-function", + "tags": { + "env": "production", + "version": "v1.0" + }, + "name": "my-function", + "ctime": "2023-07-15T10:35:00Z", + "description": "This is a sample GCP cloud function for processing data.", + "link": "https://cloudfunctions.googleapis.com/v1/projects/sample-project/locations/us-central1/functions/my-function", + "label_fingerprint": "abc123fingerprint", + "deprecation_status": { + "state": "DEPRECATED", + "deleted": "2024-01-01T00:00:00Z" + }, + "build_config": { + "automatic_update_policy": { + "policyName": "autoUpdateEnabled" + }, + "build": "gcr.io/sample-project/build-image:latest", + "docker_registry": "gcr.io", + "docker_repository": "sample-project-repo", + "entry_point": "main", + "environment_variables": { + "VAR1": "value1", + "VAR2": "value2" + }, + "on_deploy_update_policy": "runtimeVersion", + "runtime": "nodejs16", + "service_account": "service-account@sample-project.iam.gserviceaccount.com", + "source": { + "git_uri": "https://github.com/example/repo.git", + "repo_source": { + "branch_name": "main", + "commit_sha": "abc123commitsha", + "dir": "/src", + "project_id": "sample-project", + "repo_name": "example-repo", + "tag_name": null + }, + "storage_source": null + }, + "source_provenance": { + "git_uri": "https://github.com/example/repo.git", + "resolved_repo_source": { + "branch_name": "main", + "commit_sha": "abc123commitsha", + "dir": "/src", + "project_id": "sample-project", + "repo_name": "example-repo", + "tag_name": null + }, + "resolved_storage_source": null + }, + "source_token": "token123", + "worker_pool": "projects/sample-project/workerPools/pool1" + }, + "create_time": "2023-07-15T10:35:00Z", + "environment": "GEN_2", + "event_trigger": { + "channel": "projects/sample-project/topics/my-topic", + "event_filters": [ + { + "attribute": "type", + "operator": "=", + "value": "google.cloud.pubsub.topic.publish" + } + ], + "event_type": "google.pubsub.topic.publish", + "pubsub_topic": "projects/sample-project/topics/my-topic", + "retry_policy": "RETRY_POLICY_DEFAULT", + "service": "pubsub.googleapis.com", + "service_account_email": "service-account@sample-project.iam.gserviceaccount.com", + "trigger": "projects/sample-project/locations/us-central1/triggers/my-trigger", + "trigger_region": "us-central1" + }, + "kms_key_name": "projects/sample-project/locations/us-central1/keyRings/my-key-ring/cryptoKeys/my-key", + "satisfies_pzs": true, + "service_config": { + "all_traffic_on_latest_revision": true, + "available_cpu": "2", + "available_memory": "512Mi", + "binary_authorization_policy": "ALLOW", + "environment_variables": { + "SERVICE_ENV": "production" + }, + "ingress_settings": "ALLOW_ALL", + "max_instance_count": 5, + "max_instance_request_concurrency": 1, + "min_instance_count": 1, + "revision": "my-function-rev1", + "secret_environment_variables": [ + { + "key": "DB_PASSWORD", + "project_id": "sample-project", + "secret": "db-password", + "version": "latest" + } + ], + "secret_volumes": [ + { + "mount_path": "/etc/secrets", + "project_id": "sample-project", + "secret": "config-secrets", + "versions": [ + { + "path": "config.json", + "version": "1" + } + ] + } + ], + "security_level": "SECURE", + "service": "my-function-service", + "service_account_email": "service-account@sample-project.iam.gserviceaccount.com", + "timeout_seconds": 300, + "uri": "https://us-central1-my-function.cloudfunctions.net/my-function", + "vpc_connector": "projects/sample-project/locations/us-central1/connectors/my-connector", + "vpc_connector_egress_settings": "ALL_TRAFFIC" + }, + "state": "ACTIVE", + "state_messages": [ + { + "message": "Function deployed successfully.", + "severity": "INFO", + "type": "DEPLOYMENT_STATUS" + } + ], + "update_time": "2024-01-10T15:20:00Z", + "upgrade_info": { + "build_config": { + "automatic_update_policy": { + "policyName": "autoUpdateEnabled" + }, + "build": "gcr.io/sample-project/build-image:latest", + "docker_registry": "gcr.io", + "docker_repository": "sample-project-repo", + "entry_point": "main", + "environment_variables": { + "UPGRADE_VAR": "newValue" + }, + "on_deploy_update_policy": "runtimeVersion", + "runtime": "nodejs18", + "service_account": "upgrade-service-account@sample-project.iam.gserviceaccount.com", + "source": null, + "source_provenance": null, + "source_token": null, + "worker_pool": null + }, + "event_trigger": null, + "service_config": null, + "upgrade_state": "UPGRADE_AVAILABLE" + }, + "url": "https://us-central1-my-function.cloudfunctions.net/my-function" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/filestore_backup.json b/plugins/gcp/test/files/filestore_backup.json new file mode 100644 index 0000000000..2af88e45dd --- /dev/null +++ b/plugins/gcp/test/files/filestore_backup.json @@ -0,0 +1,31 @@ +{ + "backups": [ + { + "id": "projects/sample-project/locations/us-central1/backups/backup-1", + "name": "backup-1", + "ctime": "2024-11-11T08:30:00Z", + "description": "Daily backup for critical data", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/backups/backup-1", + "label_fingerprint": "abc123", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "capacity_gb": "1024", + "create_time": "2024-11-11T08:30:00Z", + "download_bytes": "5368709120", + "file_system_protocol": "NFSv3", + "kms_key": "projects/sample-project/locations/us-central1/keyRings/sample-ring/cryptoKeys/sample-key", + "satisfies_pzi": true, + "satisfies_pzs": false, + "source_file_share": "share-1", + "source_instance": "projects/sample-project/locations/us-central1/instances/instance-1", + "source_instance_tier": "STANDARD", + "state": "READY", + "storage_bytes": "5368709120" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/filestore_instance.json b/plugins/gcp/test/files/filestore_instance.json new file mode 100644 index 0000000000..28ceeb251c --- /dev/null +++ b/plugins/gcp/test/files/filestore_instance.json @@ -0,0 +1,86 @@ +{ + "instances": [ + { + "id": "projects/sample-project/locations/us-central1/instances/instance-1", + "name": "instance-1", + "ctime": "2024-10-01T14:00:00Z", + "description": "Primary file storage instance", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/instances/instance-1", + "label_fingerprint": "def456", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "configurable_performance_enabled": true, + "create_time": "2024-10-01T14:00:00Z", + "deletion_protection_enabled": false, + "deletion_protection_reason": null, + "etag": "etag-789", + "file_shares": [ + { + "capacity_gb": "2048", + "name": "file-share-1", + "nfs_export_options": [ + { + "access_mode": "READ_WRITE", + "anon_gid": "65534", + "anon_uid": "65534", + "ip_ranges": [ + "192.168.1.0/24" + ], + "squash_mode": "NO_ROOT_SQUASH" + } + ], + "source_backup": "projects/sample-project/locations/us-central1/backups/backup-1" + } + ], + "kms_key_name": "projects/sample-project/locations/us-central1/keyRings/sample-ring/cryptoKeys/sample-key", + "networks": [ + { + "connect_mode": "PRIVATE_SERVICE_ACCESS", + "ip_addresses": [ + "10.1.2.3" + ], + "modes": [ + "MODE_IPV4" + ], + "network": "projects/sample-project/global/networks/default", + "reserved_ip_range": "10.1.0.0/24" + } + ], + "performance_config": { + "fixed_iops": "1000", + "iops_per_tb": "300" + }, + "performance_limits": { + "max_read_iops": "3000", + "max_read_throughput_bps": "104857600", + "max_write_iops": "2000", + "max_write_throughput_bps": "52428800" + }, + "protocol": "NFSv3", + "replication": { + "replicas": [ + { + "last_active_sync_time": "2024-11-01T12:00:00Z", + "peer_instance": "projects/sample-project/locations/us-central1/instances/replica-1", + "state": "ACTIVE", + "state_reasons": [ + "Synced successfully" + ] + } + ], + "role": "PRIMARY" + }, + "satisfies_pzi": true, + "satisfies_pzs": true, + "state": "READY", + "status_message": "Instance is operating normally", + "suspension_reasons": [], + "tier": "STANDARD" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/filestore_instance_snapshot.json b/plugins/gcp/test/files/filestore_instance_snapshot.json new file mode 100644 index 0000000000..eb9cfdb9d1 --- /dev/null +++ b/plugins/gcp/test/files/filestore_instance_snapshot.json @@ -0,0 +1,22 @@ +{ + "snapshots": [ + { + "id": "projects/sample-project/locations/us-central1/snapshots/snapshot-1", + "name": "snapshot-1", + "ctime": "2024-10-15T09:00:00Z", + "description": "Snapshot before maintenance", + "link": "https://www.googleapis.com/filestore/v1/projects/sample-project/locations/us-central1/snapshots/snapshot-1", + "label_fingerprint": "ghi789", + "deprecation_status": { + "state": "ACTIVE", + "deleted": null, + "deprecated": null, + "obsolete": null, + "replacement": null + }, + "create_time": "2024-10-15T09:00:00Z", + "filesystem_used_bytes": "1073741824", + "state": "READY" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/firestore_backup.json b/plugins/gcp/test/files/firestore_backup.json new file mode 100644 index 0000000000..b0a7a9e640 --- /dev/null +++ b/plugins/gcp/test/files/firestore_backup.json @@ -0,0 +1,28 @@ +{ + "backups": [ + { + "id": "projects/my-project/locations/us-central1/backups/backup-1", + "tags": { + "backupType": "daily" + }, + "name": "backup-1", + "ctime": "2023-07-13T00:00:00.000Z", + "description": "Daily backup for disaster recovery", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/locations/us-central1/backups/backup-1", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "database_name": "projects/my-project/databases/(default)", + "database_uid": "db-uid-1234", + "expire_time": "2023-07-20T00:00:00.000Z", + "snapshot_time": "2023-07-13T00:00:00.000Z", + "state": "COMPLETED", + "backup_stats": { + "document_count": "5000", + "index_count": "15", + "size_bytes": "104857600" + } + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/firestore_database.json b/plugins/gcp/test/files/firestore_database.json new file mode 100644 index 0000000000..ffb9b72abb --- /dev/null +++ b/plugins/gcp/test/files/firestore_database.json @@ -0,0 +1,44 @@ +{ + "databases": [ + { + "id": "projects/my-project/databases/(default)", + "tags": { + "env": "production", + "team": "backend" + }, + "name": "projects/my-project/databases/(default)", + "ctime": "2023-07-14T10:22:33.123Z", + "description": "Main production database", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/databases/(default)", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "app_engine_integration_mode": "DEFAULT", + "cmek_config": { + "active_key_version": [ + "projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key/cryptoKeyVersions/1" + ], + "kms_key_name": "projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key" + }, + "concurrency_mode": "PESSIMISTIC", + "create_time": "2023-07-14T10:22:33.123Z", + "delete_protection_state": "ENABLED", + "delete_time": null, + "earliest_version_time": "2023-07-14T10:22:33.123Z", + "etag": "etag12345", + "key_prefix": "db-prefix", + "location_id": "us-central1", + "point_in_time_recovery_enablement": "ENABLED", + "previous_id": "prev-db-id", + "source_info": { + "backup": "projects/my-project/locations/us-central1/backups/backup-1", + "operation": "projects/my-project/operations/operation-123" + }, + "type": "FIRESTORE_NATIVE", + "uid": "db-uid-1234", + "update_time": "2023-07-15T12:34:56.789Z", + "version_retention_period": "7d" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/firestore_document.json b/plugins/gcp/test/files/firestore_document.json new file mode 100644 index 0000000000..218f5074ab --- /dev/null +++ b/plugins/gcp/test/files/firestore_document.json @@ -0,0 +1,37 @@ +{ + "documents": [ + { + "id": "projects/my-project/databases/(default)/documents/collection/doc1", + "tags": { + "documentType": "user-data" + }, + "name": "projects/my-project/databases/(default)/documents/collection/doc1", + "ctime": "2023-07-15T14:30:00.000Z", + "description": "User profile document", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/databases/(default)/documents/collection/doc1", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "create_time": "2023-07-15T14:30:00.000Z", + "fields": { + "name": { + "string_value": "John Doe" + }, + "age": { + "integer_value": "30" + }, + "isActive": { + "boolean_value": true + }, + "location": { + "geo_point_value": { + "latitude": 37.7749, + "longitude": -122.4194 + } + } + }, + "update_time": "2023-07-16T08:45:00.000Z" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/firestore_test.py b/plugins/gcp/test/firestore_test.py new file mode 100644 index 0000000000..636b7316d2 --- /dev/null +++ b/plugins/gcp/test/firestore_test.py @@ -0,0 +1,29 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.firestore import GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup + + +def test_gcp_firestore_database(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_database.json") as f: + GcpFirestoreDatabase.collect(raw=json.load(f)["databases"], builder=random_builder) + + databases = random_builder.nodes(clazz=GcpFirestoreDatabase) + assert len(databases) == 1 + + +def test_gcp_firestore_document(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_document.json") as f: + GcpFirestoreDocument.collect(raw=json.load(f)["documents"], builder=random_builder) + + documents = random_builder.nodes(clazz=GcpFirestoreDocument) + assert len(documents) == 1 + + +def test_gcp_firestore_backup(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_backup.json") as f: + GcpFirestoreBackup.collect(raw=json.load(f)["backups"], builder=random_builder) + + backups = random_builder.nodes(clazz=GcpFirestoreBackup) + assert len(backups) == 1 diff --git a/plugins/gcp/test/test_cloudfunctions.py b/plugins/gcp/test/test_cloudfunctions.py new file mode 100644 index 0000000000..e5c85724fa --- /dev/null +++ b/plugins/gcp/test/test_cloudfunctions.py @@ -0,0 +1,13 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.cloudfunctions import GcpCloudFunction + + +def test_gcp_cloudfunctions(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/cloudfunctions.json") as f: + GcpCloudFunction.collect(raw=json.load(f)["functions"], builder=random_builder) + + functions = random_builder.nodes(clazz=GcpCloudFunction) + assert len(functions) == 1 diff --git a/plugins/gcp/test/test_filestore.py b/plugins/gcp/test/test_filestore.py new file mode 100644 index 0000000000..394cab86f8 --- /dev/null +++ b/plugins/gcp/test/test_filestore.py @@ -0,0 +1,29 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.filestore import GcpFilestoreBackup, GcpFilestoreInstance, GcpFilestoreInstanceSnapshot + + +def test_gcp_filestore_backup(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_backup.json") as f: + GcpFilestoreBackup.collect(raw=json.load(f)["backups"], builder=random_builder) + + backups = random_builder.nodes(clazz=GcpFilestoreBackup) + assert len(backups) == 1 + + +def test_gcp_filestore_instance(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_instance.json") as f: + GcpFilestoreInstance.collect(raw=json.load(f)["instances"], builder=random_builder) + + instances = random_builder.nodes(clazz=GcpFilestoreInstance) + assert len(instances) == 1 + + +def test_gcp_filestore_instance_snapshot(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/filestore_instance_snapshot.json") as f: + GcpFilestoreInstanceSnapshot.collect(raw=json.load(f)["snapshots"], builder=random_builder) + + snapshots = random_builder.nodes(clazz=GcpFilestoreInstanceSnapshot) + assert len(snapshots) == 1 diff --git a/plugins/gcp/tools/model_gen.py b/plugins/gcp/tools/model_gen.py index 0a0d1e5453..0404d244f6 100644 --- a/plugins/gcp/tools/model_gen.py +++ b/plugins/gcp/tools/model_gen.py @@ -510,6 +510,12 @@ def generate_test_classes() -> None: "name": "", "parent": "projects/{project}/locations/{region}", }, + "cloudfunctions": { + "name": "", + "parent": "projects/{project}/locations/-", + }, + "firestore": {"parent": "projects/{project_id}/databases/{database_id}/documents", "collectionId": "", "name": ""}, + "file": {"name": "", "parent": "projects/{projectId}/locations/-"}, } # See https://googleapis.github.io/google-api-python-client/docs/dyn/ for the list of available resources @@ -519,8 +525,11 @@ def generate_test_classes() -> None: # ("container", "v1", "Container", ["UsableSubnetwork"]), # ("sqladmin", "v1", "Sql", ["Tier"]), # ("cloudbilling", "v1", "", []), - # ("storage", "v1", "", []) - ("aiplatform", "v1", "", []) + # ("storage", "v1", "", []), + # ("aiplatform", "v1", "", []), + # ("firestore", "v1", "", []), + # ("cloudfunctions", "v2", "", []), + ("file", "v1", "", []) ] diff --git a/requirements-all.txt b/requirements-all.txt index 50791c998d..ffd5e59667 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,27 +1,25 @@ aiodns==3.2.0 aiofiles==24.1.0 aiohappyeyeballs==2.4.3 -aiohttp[speedups]==3.10.10 +aiohttp[speedups]==3.10.11 aiohttp-jinja2==1.6 aiohttp-swagger3==0.9.0 aiosignal==1.3.1 -aiostream==0.6.3 -appdirs==1.4.4 apscheduler==3.10.4 asn1crypto==1.5.1 astroid==3.3.5 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.31.0 +azure-core==1.32.0 azure-identity==1.19.0 -azure-mgmt-core==1.4.0 +azure-mgmt-core==1.5.0 azure-mgmt-resource==23.2.0 backoff==2.2.1 beautifulsoup4==4.12.3 black==24.10.0 -boto3==1.35.50 -botocore==1.35.50 +boto3==1.35.65 +botocore==1.35.65 brotli==1.1.0 build==1.2.2.post1 cached-property==2.0.1 @@ -38,11 +36,11 @@ click==8.1.7 click-option-group==0.5.6 cloudsplaining==0.7.0 colorama==0.4.6 -coverage[toml]==7.6.4 +coverage[toml]==7.6.7 cryptography==43.0.3 deepdiff==8.0.1 defusedxml==0.7.1 -deprecated==1.2.14 +deprecated==1.2.15 detect-secrets==1.5.0 dill==0.3.9 distlib==0.3.9 @@ -55,21 +53,21 @@ fixinventoryclient==2.0.1 fixinventorydata==0.2.6 flake8==7.1.1 flexcache==0.3 -flexparser==0.3.1 +flexparser==0.4 frozendict==2.4.6 frozenlist==1.5.0 -google-api-core==2.22.0 -google-api-python-client==2.149.0 -google-auth==2.35.0 +google-api-core==2.23.0 +google-api-python-client==2.153.0 +google-auth==2.36.0 google-auth-httplib2==0.2.0 google-cloud-core==2.4.1 google-cloud-storage==2.18.2 google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 hcloud==2.2.0 httplib2==0.22.0 -hypothesis==6.115.5 +hypothesis==6.119.3 idna==3.10 importlib-metadata==8.5.0 iniconfig==2.0.0 @@ -91,7 +89,7 @@ mccabe==0.7.0 mdurl==0.1.2 monotonic==1.6 more-itertools==10.5.0 -msal==1.31.0 +msal==1.31.1 msal-extensions==1.2.0 mstache==0.2.0 multidict==6.1.0 @@ -102,12 +100,12 @@ oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 -orjson==3.10.10 -packaging==24.1 +orjson==3.10.11 +packaging==24.2 parsy==2.1 pathspec==0.12.1 pep8-naming==0.14.1 -pint==0.24.3 +pint==0.24.4 pip==24.3.1 pip-tools==7.4.1 plantuml==0.3.0 @@ -116,7 +114,7 @@ pluggy==1.5.0 policy-sentry==0.13.1 portalocker==2.10.1 portend==3.2.0 -posthog==3.7.0 +posthog==3.7.2 prometheus-client==0.21.0 prompt-toolkit==3.0.48 propcache==0.2.0 @@ -131,9 +129,9 @@ pycares==4.4.0 pycodestyle==2.12.1 pycparser==2.22 pyflakes==3.2.0 -pygithub==2.4.0 +pygithub==2.5.0 pygments==2.18.0 -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.0 pylint==3.3.1 pymysql==1.1.1 pynacl==1.5.0 @@ -143,7 +141,7 @@ pyproject-api==1.8.0 pyproject-hooks==1.2.0 pytest==8.3.3 pytest-asyncio==0.24.0 -pytest-cov==5.0.0 +pytest-cov==6.0.0 pytest-runner==6.0.1 python-arango==8.1.2 python-dateutil==2.9.0.post0 @@ -154,13 +152,13 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.9.3 +rich==13.9.4 rsa==4.9 s3transfer==0.10.3 schema==0.7.7 -setuptools==75.3.0 +setuptools==75.5.0 six==1.16.0 -slack-sdk==3.33.2 +slack-sdk==3.33.4 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 @@ -173,14 +171,14 @@ tomlkit==0.13.2 toolz==1.0.0 tox==4.23.2 transitions==0.9.2 -typeguard==4.4.0 +typeguard==4.4.1 types-aiofiles==24.1.0.20240626 types-python-dateutil==2.9.0.20241003 types-pytz==2024.2.0.20241003 types-pyyaml==6.0.12.20240917 types-requests==2.31.0.6 -types-setuptools==75.2.0.20241025 -types-six==1.16.21.20241009 +types-setuptools==75.5.0.20241122 +types-six==1.16.21.20241105 types-tzlocal==5.1.0.1 types-urllib3==1.26.25.14 typing-extensions==4.12.2 @@ -193,8 +191,8 @@ ustache==0.1.6 virtualenv==20.27.1 wcwidth==0.2.13 websocket-client==1.8.0 -wheel==0.44.0 +wheel==0.45.0 wrapt==1.16.0 -yarl==1.17.0 +yarl==1.17.2 zc-lockfile==3.0.post1 -zipp==3.20.2 +zipp==3.21.0 diff --git a/requirements-extra.txt b/requirements-extra.txt index 98cebdde10..03ce8ad2f9 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -1,25 +1,23 @@ aiodns==3.2.0 aiofiles==24.1.0 aiohappyeyeballs==2.4.3 -aiohttp[speedups]==3.10.10 +aiohttp[speedups]==3.10.11 aiohttp-jinja2==1.6 aiohttp-swagger3==0.9.0 aiosignal==1.3.1 -aiostream==0.6.3 -appdirs==1.4.4 apscheduler==3.10.4 asn1crypto==1.5.1 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.31.0 +azure-core==1.32.0 azure-identity==1.19.0 -azure-mgmt-core==1.4.0 +azure-mgmt-core==1.5.0 azure-mgmt-resource==23.2.0 backoff==2.2.1 beautifulsoup4==4.12.3 -boto3==1.35.50 -botocore==1.35.50 +boto3==1.35.65 +botocore==1.35.65 brotli==1.1.0 cached-property==2.0.1 cachetools==5.5.0 @@ -36,7 +34,7 @@ cloudsplaining==0.7.0 cryptography==43.0.3 deepdiff==8.0.1 defusedxml==0.7.1 -deprecated==1.2.14 +deprecated==1.2.15 detect-secrets==1.5.0 durationpy==0.9 fastjsonschema==2.19.1 @@ -46,18 +44,18 @@ fixdatalink[extra]==2.0.2 fixinventoryclient==2.0.1 fixinventorydata==0.2.6 flexcache==0.3 -flexparser==0.3.1 +flexparser==0.4 frozendict==2.4.6 frozenlist==1.5.0 -google-api-core==2.22.0 -google-api-python-client==2.149.0 -google-auth==2.35.0 +google-api-core==2.23.0 +google-api-python-client==2.153.0 +google-auth==2.36.0 google-auth-httplib2==0.2.0 google-cloud-core==2.4.1 google-cloud-storage==2.18.2 google-crc32c==1.6.0 google-resumable-media==2.7.2 -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 hcloud==2.2.0 httplib2==0.22.0 idna==3.10 @@ -78,7 +76,7 @@ markupsafe==3.0.2 mdurl==0.1.2 monotonic==1.6 more-itertools==10.5.0 -msal==1.31.0 +msal==1.31.1 msal-extensions==1.2.0 mstache==0.2.0 multidict==6.1.0 @@ -87,16 +85,16 @@ oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 -orjson==3.10.10 -packaging==24.1 +orjson==3.10.11 +packaging==24.2 parsy==2.1 -pint==0.24.3 +pint==0.24.4 plantuml==0.3.0 platformdirs==4.3.6 policy-sentry==0.13.1 portalocker==2.10.1 portend==3.2.0 -posthog==3.7.0 +posthog==3.7.2 prometheus-client==0.21.0 prompt-toolkit==3.0.48 propcache==0.2.0 @@ -109,9 +107,9 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 pycares==4.4.0 pycparser==2.22 -pygithub==2.4.0 +pygithub==2.5.0 pygments==2.18.0 -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.0 pymysql==1.1.1 pynacl==1.5.0 pyopenssl==24.2.1 @@ -125,13 +123,13 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.9.3 +rich==13.9.4 rsa==4.9 s3transfer==0.10.3 schema==0.7.7 -setuptools==75.3.0 +setuptools==75.5.0 six==1.16.0 -slack-sdk==3.33.2 +slack-sdk==3.33.4 snowflake-connector-python==3.12.3 snowflake-sqlalchemy==1.6.1 sortedcontainers==2.4.0 @@ -142,7 +140,7 @@ tenacity==9.0.0 tomlkit==0.13.2 toolz==1.0.0 transitions==0.9.2 -typeguard==4.4.0 +typeguard==4.4.1 typing-extensions==4.12.2 typish==1.9.3 tzdata==2024.2 @@ -153,6 +151,6 @@ ustache==0.1.6 wcwidth==0.2.13 websocket-client==1.8.0 wrapt==1.16.0 -yarl==1.17.0 +yarl==1.17.2 zc-lockfile==3.0.post1 -zipp==3.20.2 +zipp==3.21.0 diff --git a/requirements.txt b/requirements.txt index 4b37f2983f..727b5ce149 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,22 @@ aiodns==3.2.0 aiofiles==24.1.0 aiohappyeyeballs==2.4.3 -aiohttp[speedups]==3.10.10 +aiohttp[speedups]==3.10.11 aiohttp-jinja2==1.6 aiohttp-swagger3==0.9.0 aiosignal==1.3.1 -aiostream==0.6.3 -appdirs==1.4.4 apscheduler==3.10.4 attrs==24.2.0 autocommand==2.2.2 azure-common==1.1.28 -azure-core==1.31.0 +azure-core==1.32.0 azure-identity==1.19.0 -azure-mgmt-core==1.4.0 +azure-mgmt-core==1.5.0 azure-mgmt-resource==23.2.0 backoff==2.2.1 beautifulsoup4==4.12.3 -boto3==1.35.50 -botocore==1.35.50 +boto3==1.35.65 +botocore==1.35.65 brotli==1.1.0 cached-property==2.0.1 cachetools==5.5.0 @@ -35,7 +33,7 @@ cloudsplaining==0.7.0 cryptography==43.0.3 deepdiff==8.0.1 defusedxml==0.7.1 -deprecated==1.2.14 +deprecated==1.2.15 detect-secrets==1.5.0 durationpy==0.9 fastjsonschema==2.19.1 @@ -44,14 +42,14 @@ fixdatalink==2.0.2 fixinventoryclient==2.0.1 fixinventorydata==0.2.6 flexcache==0.3 -flexparser==0.3.1 +flexparser==0.4 frozendict==2.4.6 frozenlist==1.5.0 -google-api-core==2.22.0 -google-api-python-client==2.149.0 -google-auth==2.35.0 +google-api-core==2.23.0 +google-api-python-client==2.153.0 +google-auth==2.36.0 google-auth-httplib2==0.2.0 -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 hcloud==2.2.0 httplib2==0.22.0 idna==3.10 @@ -72,7 +70,7 @@ markupsafe==3.0.2 mdurl==0.1.2 monotonic==1.6 more-itertools==10.5.0 -msal==1.31.0 +msal==1.31.1 msal-extensions==1.2.0 mstache==0.2.0 multidict==6.1.0 @@ -81,15 +79,16 @@ oauth2client==4.1.3 oauthlib==3.2.2 onelogin==2.0.4 orderly-set==5.2.2 -orjson==3.10.10 -packaging==24.1 +orjson==3.10.11 +packaging==24.2 parsy==2.1 -pint==0.24.3 +pint==0.24.4 plantuml==0.3.0 +platformdirs==4.3.6 policy-sentry==0.13.1 portalocker==2.10.1 portend==3.2.0 -posthog==3.7.0 +posthog==3.7.2 prometheus-client==0.21.0 prompt-toolkit==3.0.48 propcache==0.2.0 @@ -100,9 +99,9 @@ pyasn1==0.6.1 pyasn1-modules==0.4.1 pycares==4.4.0 pycparser==2.22 -pygithub==2.4.0 +pygithub==2.5.0 pygments==2.18.0 -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.0 pynacl==1.5.0 pyparsing==3.2.0 python-arango==8.1.2 @@ -114,20 +113,20 @@ requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 retrying==1.3.4 rfc3339-validator==0.1.4 -rich==13.9.3 +rich==13.9.4 rsa==4.9 s3transfer==0.10.3 schema==0.7.7 -setuptools==75.3.0 +setuptools==75.5.0 six==1.16.0 -slack-sdk==3.33.2 +slack-sdk==3.33.4 soupsieve==2.6 sqlalchemy==1.4.54 tempora==5.7.0 tenacity==9.0.0 toolz==1.0.0 transitions==0.9.2 -typeguard==4.4.0 +typeguard==4.4.1 typing-extensions==4.12.2 typish==1.9.3 tzdata==2024.2 @@ -138,6 +137,6 @@ ustache==0.1.6 wcwidth==0.2.13 websocket-client==1.8.0 wrapt==1.16.0 -yarl==1.17.0 +yarl==1.17.2 zc-lockfile==3.0.post1 -zipp==3.20.2 +zipp==3.21.0