From 2b722bb1df5a2cae23a48a461b97e51f5dd3e708 Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 10:33:43 +0000 Subject: [PATCH 1/7] feat: init commit --- .../gcp/fix_plugin_gcp/resources/firestore.py | 248 ++++++++++++++++++ plugins/gcp/tools/model_gen.py | 4 +- 2 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 plugins/gcp/fix_plugin_gcp/resources/firestore.py diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py new file mode 100644 index 0000000000..76320ef5c7 --- /dev/null +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -0,0 +1,248 @@ +from datetime import datetime +import logging +from typing import ClassVar, Dict, Optional, List, Any + +from attr import define, field + +from fix_plugin_gcp.gcp_client import GcpApiSpec +from fix_plugin_gcp.resources.base import GcpErrorHandler, GcpResource, GcpDeprecationStatus, GraphBuilder +from fixlib.baseresources import BaseDatabase +from fixlib.json_bender import Bender, S, Bend, MapDict +from fixlib.types import Json + +log = logging.getLogger("fix.plugins.gcp") + + +# https://cloud.google.com/firestore/docs + +service_name = "firestore" + + +@define(eq=False, slots=False) +class GcpGoogleFirestoreAdminV1CmekConfig: + kind: ClassVar[str] = "gcp_google_firestore_admin_v1_cmek_config" + mapping: ClassVar[Dict[str, Bender]] = { + "active_key_version": S("activeKeyVersion", default=[]), + "kms_key_name": S("kmsKeyName"), + } + active_key_version: Optional[List[str]] = field(default=None) + kms_key_name: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpGoogleFirestoreAdminV1SourceInfo: + kind: ClassVar[str] = "gcp_google_firestore_admin_v1_source_info" + mapping: ClassVar[Dict[str, Bender]] = {"backup": S("backup", "backup"), "operation": S("operation")} + backup: Optional[str] = field(default=None) + operation: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreDatabase(GcpResource, BaseDatabase): + kind: ClassVar[str] = "gcp_firestore_database" + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases"], + action="list", + request_parameter={"parent": "projects/{project}"}, + request_parameter_in={"project"}, + response_path="databases", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "app_engine_integration_mode": S("appEngineIntegrationMode"), + "cmek_config": S("cmekConfig", default={}) >> Bend(GcpGoogleFirestoreAdminV1CmekConfig.mapping), + "concurrency_mode": S("concurrencyMode"), + "create_time": S("createTime"), + "delete_protection_state": S("deleteProtectionState"), + "delete_time": S("deleteTime"), + "earliest_version_time": S("earliestVersionTime"), + "etag": S("etag"), + "key_prefix": S("keyPrefix"), + "location_id": S("locationId"), + "point_in_time_recovery_enablement": S("pointInTimeRecoveryEnablement"), + "previous_id": S("previousId"), + "source_info": S("sourceInfo", default={}) >> Bend(GcpGoogleFirestoreAdminV1SourceInfo.mapping), + "type": S("type"), + "uid": S("uid"), + "update_time": S("updateTime"), + "version_retention_period": S("versionRetentionPeriod"), + } + app_engine_integration_mode: Optional[str] = field(default=None) + cmek_config: Optional[GcpGoogleFirestoreAdminV1CmekConfig] = field(default=None) + concurrency_mode: Optional[str] = field(default=None) + create_time: Optional[datetime] = field(default=None) + delete_protection_state: Optional[str] = field(default=None) + delete_time: Optional[datetime] = field(default=None) + earliest_version_time: Optional[datetime] = field(default=None) + etag: Optional[str] = field(default=None) + key_prefix: Optional[str] = field(default=None) + location_id: Optional[str] = field(default=None) + point_in_time_recovery_enablement: Optional[str] = field(default=None) + previous_id: Optional[str] = field(default=None) + source_info: Optional[GcpGoogleFirestoreAdminV1SourceInfo] = field(default=None) + type: Optional[str] = field(default=None) + uid: Optional[str] = field(default=None) + update_time: Optional[datetime] = field(default=None) + version_retention_period: Optional[str] = field(default=None) + + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: + def collect_documents() -> None: + spec = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases", "documents"], + action="list", + request_parameter={"parent": f"{self.id}/documents"}, + request_parameter_in={"project"}, + response_path="documents", + response_regional_sub_path=None, + ) + with GcpErrorHandler( + spec.action, + graph_builder.error_accumulator, + spec.service, + graph_builder.region.safe_name if graph_builder.region else None, + set(), + f" in {graph_builder.project.id} kind {GcpFirestoreDocument.kind}", + ): + items = graph_builder.client.list(spec) + GcpFirestoreDocument.collect(items, graph_builder) + log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFirestoreDocument.kind}") + + graph_builder.submit_work(collect_documents) + + +@define(eq=False, slots=False) +class GcpArrayValue: + kind: ClassVar[str] = "gcp_array_value" + mapping: ClassVar[Dict[str, Bender]] = {"values": S("values", default=[])} + values: Optional[List[Any]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpLatLng: + kind: ClassVar[str] = "gcp_lat_lng" + mapping: ClassVar[Dict[str, Bender]] = {"latitude": S("latitude"), "longitude": S("longitude")} + latitude: Optional[float] = field(default=None) + longitude: Optional[float] = field(default=None) + + +@define(eq=False, slots=False) +class GcpMapValue: + kind: ClassVar[str] = "gcp_map_value" + mapping: ClassVar[Dict[str, Bender]] = {"fields": S("fields", default={})} + fields: Optional[Dict[str, Any]] = field(default=None) + + +@define(eq=False, slots=False) +class GcpValue: + kind: ClassVar[str] = "gcp_value" + mapping: ClassVar[Dict[str, Bender]] = { + "array_value": S("arrayValue", default={}) >> Bend(GcpArrayValue.mapping), + "boolean_value": S("booleanValue"), + "bytes_value": S("bytesValue"), + "double_value": S("doubleValue"), + "geo_point_value": S("geoPointValue", default={}) >> Bend(GcpLatLng.mapping), + "integer_value": S("integerValue"), + "map_value": S("mapValue", default={}) >> Bend(GcpMapValue.mapping), + "null_value": S("nullValue"), + "reference_value": S("referenceValue"), + "string_value": S("stringValue"), + "timestamp_value": S("timestampValue"), + } + array_value: Optional[GcpArrayValue] = field(default=None) + boolean_value: Optional[bool] = field(default=None) + bytes_value: Optional[str] = field(default=None) + double_value: Optional[float] = field(default=None) + geo_point_value: Optional[GcpLatLng] = field(default=None) + integer_value: Optional[str] = field(default=None) + map_value: Optional[GcpMapValue] = field(default=None) + null_value: Optional[str] = field(default=None) + reference_value: Optional[str] = field(default=None) + string_value: Optional[str] = field(default=None) + timestamp_value: Optional[datetime] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreDocument(GcpResource): + kind: ClassVar[str] = "gcp_firestore_document" + # collected via GcpFirestoreDatabase() + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "create_time": S("createTime"), + "fields": S("fields", default={}) >> MapDict(value_bender=Bend(GcpValue.mapping)), + "update_time": S("updateTime"), + } + create_time: Optional[datetime] = field(default=None) + fields: Optional[Dict[str, GcpValue]] = field(default=None) + update_time: Optional[datetime] = field(default=None) + + +@define(eq=False, slots=False) +class GcpGoogleFirestoreAdminV1Stats: + kind: ClassVar[str] = "gcp_google_firestore_admin_v1_stats" + mapping: ClassVar[Dict[str, Bender]] = { + "document_count": S("documentCount"), + "index_count": S("indexCount"), + "size_bytes": S("sizeBytes"), + } + document_count: Optional[str] = field(default=None) + index_count: Optional[str] = field(default=None) + size_bytes: Optional[str] = field(default=None) + + +@define(eq=False, slots=False) +class GcpFirestoreBackup(GcpResource): + kind: ClassVar[str] = "gcp_firestore_backup" + api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "locations", "backups"], + action="list", + request_parameter={"parent": "projects/{project}/locations/-"}, + request_parameter_in={"project"}, + response_path="backups", + response_regional_sub_path=None, + ) + mapping: ClassVar[Dict[str, Bender]] = { + "id": S("name").or_else(S("id")).or_else(S("selfLink")), + "tags": S("labels", default={}), + "name": S("name"), + "ctime": S("creationTimestamp"), + "description": S("description"), + "link": S("selfLink"), + "label_fingerprint": S("labelFingerprint"), + "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), + "database": S("database"), + "database_uid": S("databaseUid"), + "expire_time": S("expireTime"), + "snapshot_time": S("snapshotTime"), + "state": S("state"), + "stats": S("stats", default={}) >> Bend(GcpGoogleFirestoreAdminV1Stats.mapping), + } + database: Optional[str] = field(default=None) + database_uid: Optional[str] = field(default=None) + expire_time: Optional[datetime] = field(default=None) + snapshot_time: Optional[datetime] = field(default=None) + state: Optional[str] = field(default=None) + stats: Optional[GcpGoogleFirestoreAdminV1Stats] = field(default=None) + + +resources = [GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup] diff --git a/plugins/gcp/tools/model_gen.py b/plugins/gcp/tools/model_gen.py index 0a0d1e5453..db5704cf53 100644 --- a/plugins/gcp/tools/model_gen.py +++ b/plugins/gcp/tools/model_gen.py @@ -510,6 +510,7 @@ def generate_test_classes() -> None: "name": "", "parent": "projects/{project}/locations/{region}", }, + "firestore": {"parent": "projects/{project_id}/databases/{database_id}/documents", "collectionId": "", "name": ""}, } # See https://googleapis.github.io/google-api-python-client/docs/dyn/ for the list of available resources @@ -520,7 +521,8 @@ def generate_test_classes() -> None: # ("sqladmin", "v1", "Sql", ["Tier"]), # ("cloudbilling", "v1", "", []), # ("storage", "v1", "", []) - ("aiplatform", "v1", "", []) + # ("aiplatform", "v1", "", []) + ("firestore", "v1", "", []), ] From a92238ad36dbd295f5f872b0b972465b3fff948f Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 11:02:49 +0000 Subject: [PATCH 2/7] feat: adjusted collection --- plugins/gcp/fix_plugin_gcp/collector.py | 3 +- .../gcp/fix_plugin_gcp/resources/firestore.py | 32 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/collector.py b/plugins/gcp/fix_plugin_gcp/collector.py index c68080294d..edd81cfec7 100644 --- a/plugins/gcp/fix_plugin_gcp/collector.py +++ b/plugins/gcp/fix_plugin_gcp/collector.py @@ -4,7 +4,7 @@ from fix_plugin_gcp.config import GcpConfig from fix_plugin_gcp.gcp_client import GcpApiSpec -from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform +from fix_plugin_gcp.resources import compute, container, billing, sqladmin, storage, aiplatform, firestore from fix_plugin_gcp.resources.base import GcpResource, GcpProject, ExecutorQueue, GraphBuilder, GcpRegion, GcpZone from fix_plugin_gcp.utils import Credentials from fixlib.baseresources import Cloud @@ -19,6 +19,7 @@ + sqladmin.resources + storage.resources + aiplatform.resources + + firestore.resources ) diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py index 76320ef5c7..704e741572 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/firestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -1,6 +1,6 @@ from datetime import datetime import logging -from typing import ClassVar, Dict, Optional, List, Any +from typing import ClassVar, Dict, Optional, List, Any, Type from attr import define, field @@ -19,8 +19,8 @@ @define(eq=False, slots=False) -class GcpGoogleFirestoreAdminV1CmekConfig: - kind: ClassVar[str] = "gcp_google_firestore_admin_v1_cmek_config" +class GcpFirestoreCmekConfig: + kind: ClassVar[str] = "gcp_firestore_cmek_config" mapping: ClassVar[Dict[str, Bender]] = { "active_key_version": S("activeKeyVersion", default=[]), "kms_key_name": S("kmsKeyName"), @@ -30,8 +30,8 @@ class GcpGoogleFirestoreAdminV1CmekConfig: @define(eq=False, slots=False) -class GcpGoogleFirestoreAdminV1SourceInfo: - kind: ClassVar[str] = "gcp_google_firestore_admin_v1_source_info" +class GcpFirestoreSourceInfo: + kind: ClassVar[str] = "gcp_firestore_source_info" mapping: ClassVar[Dict[str, Bender]] = {"backup": S("backup", "backup"), "operation": S("operation")} backup: Optional[str] = field(default=None) operation: Optional[str] = field(default=None) @@ -60,7 +60,7 @@ class GcpFirestoreDatabase(GcpResource, BaseDatabase): "label_fingerprint": S("labelFingerprint"), "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), "app_engine_integration_mode": S("appEngineIntegrationMode"), - "cmek_config": S("cmekConfig", default={}) >> Bend(GcpGoogleFirestoreAdminV1CmekConfig.mapping), + "cmek_config": S("cmekConfig", default={}) >> Bend(GcpFirestoreCmekConfig.mapping), "concurrency_mode": S("concurrencyMode"), "create_time": S("createTime"), "delete_protection_state": S("deleteProtectionState"), @@ -71,14 +71,14 @@ class GcpFirestoreDatabase(GcpResource, BaseDatabase): "location_id": S("locationId"), "point_in_time_recovery_enablement": S("pointInTimeRecoveryEnablement"), "previous_id": S("previousId"), - "source_info": S("sourceInfo", default={}) >> Bend(GcpGoogleFirestoreAdminV1SourceInfo.mapping), + "source_info": S("sourceInfo", default={}) >> Bend(GcpFirestoreSourceInfo.mapping), "type": S("type"), "uid": S("uid"), "update_time": S("updateTime"), "version_retention_period": S("versionRetentionPeriod"), } app_engine_integration_mode: Optional[str] = field(default=None) - cmek_config: Optional[GcpGoogleFirestoreAdminV1CmekConfig] = field(default=None) + cmek_config: Optional[GcpFirestoreCmekConfig] = field(default=None) concurrency_mode: Optional[str] = field(default=None) create_time: Optional[datetime] = field(default=None) delete_protection_state: Optional[str] = field(default=None) @@ -89,7 +89,7 @@ class GcpFirestoreDatabase(GcpResource, BaseDatabase): location_id: Optional[str] = field(default=None) point_in_time_recovery_enablement: Optional[str] = field(default=None) previous_id: Optional[str] = field(default=None) - source_info: Optional[GcpGoogleFirestoreAdminV1SourceInfo] = field(default=None) + source_info: Optional[GcpFirestoreSourceInfo] = field(default=None) type: Optional[str] = field(default=None) uid: Optional[str] = field(default=None) update_time: Optional[datetime] = field(default=None) @@ -196,8 +196,8 @@ class GcpFirestoreDocument(GcpResource): @define(eq=False, slots=False) -class GcpGoogleFirestoreAdminV1Stats: - kind: ClassVar[str] = "gcp_google_firestore_admin_v1_stats" +class GcpFirestoreStats: + kind: ClassVar[str] = "gcp_firestore_stats" mapping: ClassVar[Dict[str, Bender]] = { "document_count": S("documentCount"), "index_count": S("indexCount"), @@ -230,19 +230,19 @@ class GcpFirestoreBackup(GcpResource): "link": S("selfLink"), "label_fingerprint": S("labelFingerprint"), "deprecation_status": S("deprecated", default={}) >> Bend(GcpDeprecationStatus.mapping), - "database": S("database"), + "database_name": S("database"), "database_uid": S("databaseUid"), "expire_time": S("expireTime"), "snapshot_time": S("snapshotTime"), "state": S("state"), - "stats": S("stats", default={}) >> Bend(GcpGoogleFirestoreAdminV1Stats.mapping), + "backup_stats": S("stats", default={}) >> Bend(GcpFirestoreStats.mapping), } - database: Optional[str] = field(default=None) + database_name: Optional[str] = field(default=None) database_uid: Optional[str] = field(default=None) expire_time: Optional[datetime] = field(default=None) snapshot_time: Optional[datetime] = field(default=None) state: Optional[str] = field(default=None) - stats: Optional[GcpGoogleFirestoreAdminV1Stats] = field(default=None) + backup_stats: Optional[GcpFirestoreStats] = field(default=None) -resources = [GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup] +resources: List[Type[GcpResource]] = [GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup] From 045b94a14defd63e6f2d5dcaccf1452cdefd4df2 Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 11:23:53 +0000 Subject: [PATCH 3/7] feat: added metadata --- .../fix_plugin_gcp/resources/aiplatform.py | 39 ------------------- .../gcp/fix_plugin_gcp/resources/firestore.py | 18 +++++++++ 2 files changed, 18 insertions(+), 39 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py b/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py index 886f44264d..e9577c3425 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py +++ b/plugins/gcp/fix_plugin_gcp/resources/aiplatform.py @@ -1886,19 +1886,6 @@ def connect_in_graph(self, builder: GraphBuilder, source: Json) -> None: @define(eq=False, slots=False) class GcpVertexAIArtifact: kind: ClassVar[str] = "gcp_vertex_ai_artifact" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "artifacts"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "region"}, - response_path="artifacts", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), @@ -2519,19 +2506,6 @@ class GcpVertexAIPipelineTaskDetailPipelineTaskStatus: @define(eq=False, slots=False) class GcpVertexAIExecution: kind: ClassVar[str] = "gcp_vertex_ai_execution" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "executions"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "location"}, - response_path="executions", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), @@ -2597,19 +2571,6 @@ class GcpVertexAIPipelineTaskDetail: @define(eq=False, slots=False) class GcpVertexAIContext: kind: ClassVar[str] = "gcp_vertex_ai_context" - _kind_display = "" - _kind_service = "" - api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( - service="aiplatform", - version="v1", - service_with_region_prefix=True, - accessors=["projects", "locations", "metadataStores", "contexts"], - action="list", - request_parameter={"parent": "projects/{project}/locations/{region}"}, - request_parameter_in={"project", "location"}, - response_path="contexts", - response_regional_sub_path=None, - ) mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), "tags": S("labels", default={}), diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py index 704e741572..e2cc2d3701 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/firestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -40,6 +40,12 @@ class GcpFirestoreSourceInfo: @define(eq=False, slots=False) class GcpFirestoreDatabase(GcpResource, BaseDatabase): kind: ClassVar[str] = "gcp_firestore_database" + _kind_display: ClassVar[str] = "GCP Firestore Database" + _kind_description: ClassVar[str] = ( + "A Firestore Database in GCP, which is a scalable NoSQL cloud database to store and sync data for client- and server-side development." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "database", "group": "storage"} api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( service="firestore", version="v1", @@ -176,6 +182,12 @@ class GcpValue: @define(eq=False, slots=False) class GcpFirestoreDocument(GcpResource): kind: ClassVar[str] = "gcp_firestore_document" + _kind_display: ClassVar[str] = "GCP Firestore Document" + _kind_description: ClassVar[str] = ( + "A Firestore Document in GCP, representing a single document in a Firestore database, which can contain fields and subcollections." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "database", "group": "storage"} # collected via GcpFirestoreDatabase() mapping: ClassVar[Dict[str, Bender]] = { "id": S("name").or_else(S("id")).or_else(S("selfLink")), @@ -211,6 +223,12 @@ class GcpFirestoreStats: @define(eq=False, slots=False) class GcpFirestoreBackup(GcpResource): kind: ClassVar[str] = "gcp_firestore_backup" + _kind_display: ClassVar[str] = "GCP Firestore Backup" + _kind_description: ClassVar[str] = ( + "A Firestore Backup in GCP, which provides a way to back up and restore Firestore databases to protect against data loss." + ) + _kind_service: ClassVar[Optional[str]] = service_name + _metadata: ClassVar[Dict[str, Any]] = {"icon": "backup", "group": "storage"} api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( service="firestore", version="v1", From b1c7a63528b91a3c6fe7400be987da6b241a7de5 Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 13:33:44 +0000 Subject: [PATCH 4/7] added called collect apis --- .../gcp/fix_plugin_gcp/resources/firestore.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py index e2cc2d3701..f1a3996b98 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/firestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -101,6 +101,22 @@ class GcpFirestoreDatabase(GcpResource, BaseDatabase): update_time: Optional[datetime] = field(default=None) version_retention_period: Optional[str] = field(default=None) + @classmethod + def called_collect_apis(cls) -> List[GcpApiSpec]: + return [ + cls.api_spec, + GcpApiSpec( + service="firestore", + version="v1", + accessors=["projects", "databases", "documents"], + action="list", + request_parameter={"parent": "projects/{project}/databases/{databaseId}/documents"}, + request_parameter_in={"project", "databaseId"}, + response_path="documents", + response_regional_sub_path=None, + ), + ] + def post_process(self, graph_builder: GraphBuilder, source: Json) -> None: def collect_documents() -> None: spec = GcpApiSpec( @@ -109,7 +125,7 @@ def collect_documents() -> None: accessors=["projects", "databases", "documents"], action="list", request_parameter={"parent": f"{self.id}/documents"}, - request_parameter_in={"project"}, + request_parameter_in=set(), response_path="documents", response_regional_sub_path=None, ) From eb7d763d09ebe1c009db630b23845013e17905bf Mon Sep 17 00:00:00 2001 From: Kirill Date: Sat, 9 Nov 2024 13:49:56 +0000 Subject: [PATCH 5/7] feat: added referece kinds --- plugins/gcp/fix_plugin_gcp/resources/firestore.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/gcp/fix_plugin_gcp/resources/firestore.py b/plugins/gcp/fix_plugin_gcp/resources/firestore.py index f1a3996b98..5909408ce2 100644 --- a/plugins/gcp/fix_plugin_gcp/resources/firestore.py +++ b/plugins/gcp/fix_plugin_gcp/resources/firestore.py @@ -6,7 +6,7 @@ from fix_plugin_gcp.gcp_client import GcpApiSpec from fix_plugin_gcp.resources.base import GcpErrorHandler, GcpResource, GcpDeprecationStatus, GraphBuilder -from fixlib.baseresources import BaseDatabase +from fixlib.baseresources import BaseDatabase, ModelReference from fixlib.json_bender import Bender, S, Bend, MapDict from fixlib.types import Json @@ -46,6 +46,13 @@ class GcpFirestoreDatabase(GcpResource, BaseDatabase): ) _kind_service: ClassVar[Optional[str]] = service_name _metadata: ClassVar[Dict[str, Any]] = {"icon": "database", "group": "storage"} + _reference_kinds: ClassVar[ModelReference] = { + "successors": { + "default": [ + "gcp_firestore_document", + ], + }, + } api_spec: ClassVar[GcpApiSpec] = GcpApiSpec( service="firestore", version="v1", @@ -138,7 +145,9 @@ def collect_documents() -> None: f" in {graph_builder.project.id} kind {GcpFirestoreDocument.kind}", ): items = graph_builder.client.list(spec) - GcpFirestoreDocument.collect(items, graph_builder) + documents = GcpFirestoreDocument.collect(items, graph_builder) + for document in documents: + graph_builder.add_edge(self, node=document) log.info(f"[GCP:{graph_builder.project.id}] finished collecting: {GcpFirestoreDocument.kind}") graph_builder.submit_work(collect_documents) From ad845f7935e0eb2b57f808f767be6be37e398d8d Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 11 Nov 2024 12:40:18 +0000 Subject: [PATCH 6/7] feat: added tests --- plugins/gcp/test/files/firestore_backup.json | 28 ++++++++++++ .../gcp/test/files/firestore_database.json | 44 +++++++++++++++++++ .../gcp/test/files/firestore_document.json | 37 ++++++++++++++++ plugins/gcp/test/firestore_test.py | 29 ++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 plugins/gcp/test/files/firestore_backup.json create mode 100644 plugins/gcp/test/files/firestore_database.json create mode 100644 plugins/gcp/test/files/firestore_document.json create mode 100644 plugins/gcp/test/firestore_test.py diff --git a/plugins/gcp/test/files/firestore_backup.json b/plugins/gcp/test/files/firestore_backup.json new file mode 100644 index 0000000000..b0a7a9e640 --- /dev/null +++ b/plugins/gcp/test/files/firestore_backup.json @@ -0,0 +1,28 @@ +{ + "backups": [ + { + "id": "projects/my-project/locations/us-central1/backups/backup-1", + "tags": { + "backupType": "daily" + }, + "name": "backup-1", + "ctime": "2023-07-13T00:00:00.000Z", + "description": "Daily backup for disaster recovery", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/locations/us-central1/backups/backup-1", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "database_name": "projects/my-project/databases/(default)", + "database_uid": "db-uid-1234", + "expire_time": "2023-07-20T00:00:00.000Z", + "snapshot_time": "2023-07-13T00:00:00.000Z", + "state": "COMPLETED", + "backup_stats": { + "document_count": "5000", + "index_count": "15", + "size_bytes": "104857600" + } + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/firestore_database.json b/plugins/gcp/test/files/firestore_database.json new file mode 100644 index 0000000000..ffb9b72abb --- /dev/null +++ b/plugins/gcp/test/files/firestore_database.json @@ -0,0 +1,44 @@ +{ + "databases": [ + { + "id": "projects/my-project/databases/(default)", + "tags": { + "env": "production", + "team": "backend" + }, + "name": "projects/my-project/databases/(default)", + "ctime": "2023-07-14T10:22:33.123Z", + "description": "Main production database", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/databases/(default)", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "app_engine_integration_mode": "DEFAULT", + "cmek_config": { + "active_key_version": [ + "projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key/cryptoKeyVersions/1" + ], + "kms_key_name": "projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key" + }, + "concurrency_mode": "PESSIMISTIC", + "create_time": "2023-07-14T10:22:33.123Z", + "delete_protection_state": "ENABLED", + "delete_time": null, + "earliest_version_time": "2023-07-14T10:22:33.123Z", + "etag": "etag12345", + "key_prefix": "db-prefix", + "location_id": "us-central1", + "point_in_time_recovery_enablement": "ENABLED", + "previous_id": "prev-db-id", + "source_info": { + "backup": "projects/my-project/locations/us-central1/backups/backup-1", + "operation": "projects/my-project/operations/operation-123" + }, + "type": "FIRESTORE_NATIVE", + "uid": "db-uid-1234", + "update_time": "2023-07-15T12:34:56.789Z", + "version_retention_period": "7d" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/files/firestore_document.json b/plugins/gcp/test/files/firestore_document.json new file mode 100644 index 0000000000..218f5074ab --- /dev/null +++ b/plugins/gcp/test/files/firestore_document.json @@ -0,0 +1,37 @@ +{ + "documents": [ + { + "id": "projects/my-project/databases/(default)/documents/collection/doc1", + "tags": { + "documentType": "user-data" + }, + "name": "projects/my-project/databases/(default)/documents/collection/doc1", + "ctime": "2023-07-15T14:30:00.000Z", + "description": "User profile document", + "link": "https://www.googleapis.com/firestore/v1/projects/my-project/databases/(default)/documents/collection/doc1", + "label_fingerprint": "abcd1234efgh5678", + "deprecation_status": { + "state": "ACTIVE" + }, + "create_time": "2023-07-15T14:30:00.000Z", + "fields": { + "name": { + "string_value": "John Doe" + }, + "age": { + "integer_value": "30" + }, + "isActive": { + "boolean_value": true + }, + "location": { + "geo_point_value": { + "latitude": 37.7749, + "longitude": -122.4194 + } + } + }, + "update_time": "2023-07-16T08:45:00.000Z" + } + ] +} \ No newline at end of file diff --git a/plugins/gcp/test/firestore_test.py b/plugins/gcp/test/firestore_test.py new file mode 100644 index 0000000000..636b7316d2 --- /dev/null +++ b/plugins/gcp/test/firestore_test.py @@ -0,0 +1,29 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.firestore import GcpFirestoreDatabase, GcpFirestoreDocument, GcpFirestoreBackup + + +def test_gcp_firestore_database(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_database.json") as f: + GcpFirestoreDatabase.collect(raw=json.load(f)["databases"], builder=random_builder) + + databases = random_builder.nodes(clazz=GcpFirestoreDatabase) + assert len(databases) == 1 + + +def test_gcp_firestore_document(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_document.json") as f: + GcpFirestoreDocument.collect(raw=json.load(f)["documents"], builder=random_builder) + + documents = random_builder.nodes(clazz=GcpFirestoreDocument) + assert len(documents) == 1 + + +def test_gcp_firestore_backup(random_builder: GraphBuilder) -> None: + with open(os.path.dirname(__file__) + "/files/firestore_backup.json") as f: + GcpFirestoreBackup.collect(raw=json.load(f)["backups"], builder=random_builder) + + backups = random_builder.nodes(clazz=GcpFirestoreBackup) + assert len(backups) == 1 From d34172ba6e13d0aefc81e82455fbbab46f44a5ee Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 11 Nov 2024 13:42:24 +0000 Subject: [PATCH 7/7] feat: added tests for aiplatform resources --- plugins/gcp/test/aiplatform_test.py | 16 + .../gcp/test/files/aiplatform_resources.json | 740 ++++++++++++++++++ 2 files changed, 756 insertions(+) create mode 100644 plugins/gcp/test/aiplatform_test.py create mode 100644 plugins/gcp/test/files/aiplatform_resources.json diff --git a/plugins/gcp/test/aiplatform_test.py b/plugins/gcp/test/aiplatform_test.py new file mode 100644 index 0000000000..fee761a960 --- /dev/null +++ b/plugins/gcp/test/aiplatform_test.py @@ -0,0 +1,16 @@ +import json +import os + +from fix_plugin_gcp.resources.base import GraphBuilder +from fix_plugin_gcp.resources.aiplatform import resources + + +def test_gcp_aiplatform_resources(random_builder: GraphBuilder) -> None: + file_path = os.path.join(os.path.dirname(__file__), "files", "aiplatform_resources.json") + with open(file_path, "r") as f: + data = json.load(f) + + for resource, res_class in zip(data["resources"], resources): + res_class.collect(raw=[resource], builder=random_builder) + collected = random_builder.nodes(clazz=res_class) + assert len(collected) == 1 diff --git a/plugins/gcp/test/files/aiplatform_resources.json b/plugins/gcp/test/files/aiplatform_resources.json new file mode 100644 index 0000000000..2d0435af52 --- /dev/null +++ b/plugins/gcp/test/files/aiplatform_resources.json @@ -0,0 +1,740 @@ +{ + "resources": [ + { + "id": "projects/1234567890/locations/us-central1/models/my-model", + "tags": { + "env": "prod" + }, + "name": "projects/1234567890/locations/us-central1/models/my-model", + "ctime": "2024-07-26T12:00:00Z", + "description": "My trained model", + "link": "https://console.cloud.google.com/vertex-ai/locations/us-central1/models/my-model", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE", + "end_time": null + }, + "artifact_uri": "gs://my-bucket/model-artifact", + "base_model_source": { + "model_garden_source": null, + "genie_source": "projects/my-project/locations/us-central1/models/my-base-model@1" + }, + "container_spec": { + "image_uri": "us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest", + "command": [], + "args": [], + "env": [], + "ports": [], + "predict_route": "", + "health_route": "", + "grpc_ports": [], + "startup_probe": null, + "health_probe": null, + "deployment_timeout": null, + "shared_memory_size_mb": null + }, + "create_time": "2024-07-26T12:00:00Z", + "data_stats": { + "training_data_items_count": "1000", + "validation_data_items_count": "200", + "test_data_items_count": "100", + "training_annotations_count": null, + "validation_annotations_count": null, + "test_annotations_count": null + }, + "endpoint_deployed_model_refs": [ + { + "deployed_model_id": "12345", + "endpoint": "projects/.../locations/.../endpoints/my-endpoint" + } + ], + "display_name": "My Model", + "encryption_spec": "projects/1234567890/locations/global/keyRings/my-keyring/cryptoKeys/my-key", + "etag": "abcdef123456", + "explanation_spec": { + "_metadata": { + "inputs": { + "input_1": { + "input_tensor_name": "input_tensor" + } + } + }, + "parameters": { + "sampled_shapley_attribution": null, + "integrated_gradients_attribution": null, + "xrai_attribution": null, + "examples": null, + "top_k": 10 + } + } + }, + { + "id": "projects/12345/locations/us-central1/datasets/my-dataset", + "tags": { + "purpose": "training" + }, + "name": "projects/12345/locations/us-central1/datasets/my-dataset", + "ctime": "2024-08-07T15:00:00Z", + "description": "Dataset for image classification", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-07T15:00:00Z", + "data_item_count": "10000", + "display_name": "Image Classification Dataset", + "encryption_spec": null, + "etag": "abcdef123456", + "_metadata": {}, + "metadata_artifact": null, + "metadata_schema_uri": "gs://my-bucket/dataset_schema.json", + "model_reference": null, + "saved_queries": [], + "update_time": "2024-08-07T16:00:00Z" + }, + { + "id": "projects/12345/locations/us-central1/datasets/my-dataset/versions/v1", + "tags": {}, + "name": "projects/12345/locations/us-central1/datasets/my-dataset/versions/v1", + "ctime": "2024-08-07T16:30:00Z", + "description": "Version 1 of the dataset", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "big_query_dataset_name": null, + "create_time": "2024-08-07T16:30:00Z", + "display_name": "v1", + "etag": "abcdef123456", + "_metadata": {}, + "model_reference": null, + "update_time": "2024-08-07T16:30:00Z" + }, + { + "id": "projects/12345/locations/us-central1/endpoints/my-endpoint", + "tags": { + "team": "ml-team" + }, + "name": "projects/12345/locations/us-central1/endpoints/my-endpoint", + "ctime": "2024-08-08T10:00:00Z", + "description": "Endpoint for image classification model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-08T10:00:00Z", + "deployed_models": [ + { + "automatic_resources": null, + "create_time": "2024-08-08T10:15:00Z", + "dedicated_resources": { + "machine_spec": { + "machine_type": "n1-standard-2", + "accelerator_type": null, + "accelerator_count": null + }, + "min_replica_count": 1, + "max_replica_count": 3, + "autoscaling_metric_specs": [] + }, + "deployed_index_auth_config": null, + "deployment_group": null, + "disable_container_logging": false, + "disable_explanations": false, + "display_name": "Deployed Model 1", + "enable_access_logging": true, + "explanation_spec": null, + "id": "67890", + "model": "projects/12345/locations/us-central1/models/my-model", + "model_version_id": "1", + "private_endpoints": null, + "service_account": "vertex-ai@my-project.iam.gserviceaccount.com", + "shared_resources": null + } + ], + "display_name": "Image Classifier Endpoint", + "enable_private_service_connect": false, + "encryption_spec": null, + "etag": "abcdef123456", + "model_deployment_monitoring_job": null, + "network": null, + "predict_request_response_logging_config": null, + "private_service_connect_config": null, + "traffic_split": {}, + "update_time": "2024-08-08T11:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/schedules/my-schedule", + "tags": { + "owner": "john.doe" + }, + "name": "projects/my-project/locations/us-central1/schedules/my-schedule", + "ctime": "2024-08-09T14:00:00Z", + "description": "Schedule for daily batch prediction", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "allow_queueing": false, + "catch_up": false, + "create_pipeline_job_request": { + "parent": "projects/my-project/locations/us-central1", + "pipeline_job": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline", + "pipeline_job_id": "my-pipeline-job-id" + }, + "create_time": "2024-08-09T14:00:00Z", + "cron": "0 0 * * *", + "display_name": "Daily Prediction", + "end_time": null, + "last_pause_time": null, + "last_resume_time": null, + "last_scheduled_run_response": { + "run_response": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "scheduled_run_time": "2024-08-09T00:00:00Z" + }, + "max_concurrent_run_count": 1, + "max_run_count": null, + "next_run_time": "2024-08-10T00:00:00Z", + "start_time": "2024-08-09T14:00:00Z", + "started_run_count": 10, + "state": "ACTIVE", + "update_time": "2024-08-09T14:30:00Z" + }, + { + "id": "projects/12345/locations/us-central1/featureGroups/my-feature-group", + "tags": { + "department": "sales" + }, + "name": "projects/12345/locations/us-central1/featureGroups/my-feature-group", + "ctime": "2024-08-10T09:00:00Z", + "description": "Feature group for customer data", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "big_query": { + "big_query_source": "bq://my-project.my_dataset.my_table", + "entity_id_columns": [ + "customer_id" + ] + }, + "create_time": "2024-08-10T09:00:00Z", + "etag": "abcdef123456", + "update_time": "2024-08-10T09:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/featureGroups/my-feature-group/features/customer_age", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featureGroups/my-feature-group/features/customer_age", + "ctime": "2024-08-10T10:00:00Z", + "description": "Age of the customer", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-10T10:00:00Z", + "disable_monitoring": false, + "etag": "abcdef123456", + "monitoring_stats_anomalies": [], + "point_of_contact": "data-scientist@example.com", + "update_time": "2024-08-10T10:30:00Z", + "value_type": "INT64", + "version_column_name": null + }, + { + "id": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "ctime": "2024-08-11T08:00:00Z", + "description": "My Feature Store", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-11T08:00:00Z", + "encryption_spec": null, + "etag": "abcdef123456", + "online_serving_config": { + "fixed_node_count": 2, + "scaling": { + "min_node_count": 1, + "max_node_count": 5, + "cpu_utilization_target": 70 + } + }, + "online_storage_ttl_days": 7, + "state": "ACTIVE", + "update_time": "2024-08-11T08:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/trainingPipelines/my-training-pipeline", + "tags": { + "model_type": "image_classifier" + }, + "name": "projects/my-project/locations/us-central1/trainingPipelines/my-training-pipeline", + "ctime": "2024-08-12T12:00:00Z", + "description": "Training pipeline for image classification", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-12T12:00:00Z", + "display_name": "Image Classifier Training", + "encryption_spec": null, + "end_time": "2024-08-12T14:00:00Z", + "error": null, + "input_data_config": { + "dataset_id": "my-dataset", + "fraction_split": { + "training_fraction": 0.8, + "validation_fraction": 0.1, + "test_fraction": 0.1 + }, + "gcs_destination": "gs://my-bucket/training_data", + "annotation_schema_uri": null, + "annotations_filter": null, + "bigquery_destination": null, + "filter_split": null, + "persist_ml_use_assignment": null, + "predefined_split": null, + "saved_query_id": null, + "stratified_split": null, + "timestamp_split": null + }, + "model_id": null, + "model_to_upload": "projects/my-project/locations/us-central1/models/my-trained-model", + "parent_model": null, + "start_time": "2024-08-12T12:30:00Z", + "state": "SUCCEEDED", + "training_task_definition": "gs://google-cloud-aiplatform/schema/trainingjob/definition/automl_image_classification_1.0.0.yaml", + "training_task_inputs": {}, + "training_task_metadata": {}, + "update_time": "2024-08-12T14:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/batchPredictionJobs/my-batch-job", + "ctime": "2024-08-13T09:00:00Z", + "description": "Batch prediction job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "completion_stats": { + "successful_count": "1000", + "failed_count": "10", + "incomplete_count": "0", + "successful_forecast_point_count": null + }, + "create_time": "2024-08-13T09:00:00Z", + "dedicated_resources": null, + "disable_container_logging": false, + "display_name": "My Batch Prediction", + "encryption_spec": null, + "end_time": "2024-08-13T10:00:00Z", + "error": null, + "explanation_spec": null, + "generate_explanation": false, + "input_config": { + "gcs_source": { + "uris": [ + "gs://my-bucket/input-data.jsonl" + ] + }, + "instances_format": "jsonl", + "bigquery_source": null + }, + "instance_config": null, + "manual_batch_tuning_parameters": null, + "model": "projects/my-project/locations/us-central1/models/my-model", + "model_parameters": {}, + "model_version_id": "1", + "output_config": { + "gcs_destination": "gs://my-bucket/output-predictions", + "predictions_format": "jsonl", + "bigquery_destination": null + }, + "output_info": { + "bigquery_output_dataset": null, + "bigquery_output_table": null, + "gcs_output_directory": "gs://my-bucket/output-predictions" + }, + "partial_failures": [], + "resources_consumed": 1.5, + "service_account": null, + "start_time": "2024-08-13T09:15:00Z", + "state": "SUCCEEDED", + "unmanaged_container_model": null, + "update_time": "2024-08-13T10:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/models/my-model/evaluations/eval123", + "tags": {}, + "name": "projects/my-project/locations/us-central1/models/my-model/evaluations/eval123", + "ctime": "2024-08-14T11:00:00Z", + "description": "Evaluation of model performance", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "annotation_schema_uri": null, + "create_time": "2024-08-14T11:00:00Z", + "data_item_schema_uri": null, + "display_name": "Model Eval 1", + "explanation_specs": [], + "_metadata": {}, + "metrics": { + "accuracy": 0.95, + "precision": 0.92 + }, + "metrics_schema_uri": null, + "model_explanation": null, + "slice_dimensions": [] + }, + { + "id": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "tags": {}, + "name": "projects/my-project/locations/us-central1/featurestores/my-featurestore", + "ctime": "2024-08-11T08:00:00Z", + "description": "My Feature Store", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-11T08:00:00Z", + "encryption_spec": null, + "etag": "abcdef123456", + "online_serving_config": { + "fixed_node_count": 2 + }, + "online_storage_ttl_days": 7, + "state": "ACTIVE", + "update_time": "2024-08-11T08:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/hyperparameterTuningJobs/my-tuning-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/hyperparameterTuningJobs/my-tuning-job", + "ctime": "2024-08-15T15:00:00Z", + "description": "Hyperparameter tuning job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-15T15:00:00Z", + "display_name": "My Tuning Job", + "encryption_spec": null, + "end_time": "2024-08-15T17:00:00Z", + "error": null, + "max_failed_trial_count": 3, + "max_trial_count": 10, + "parallel_trial_count": 2, + "start_time": "2024-08-15T15:15:00Z", + "state": "SUCCEEDED", + "study_spec": { + "algorithm": "ALGORITHM_UNSPECIFIED", + "convex_automated_stopping_spec": null, + "decay_curve_stopping_spec": null, + "measurement_selection_type": "BEST_MEASUREMENT", + "median_automated_stopping_spec": null, + "metrics": [ + { + "goal": "MAXIMIZE", + "metric_id": "accuracy", + "safety_config": null + } + ], + "observation_noise": "OBSERVATION_NOISE_UNSPECIFIED", + "parameters": [], + "study_stopping_config": null + }, + "trial_job_spec": null, + "trials": [], + "update_time": "2024-08-15T17:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/customJobs/my-custom-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/customJobs/my-custom-job", + "ctime": "2024-08-16T10:00:00Z", + "description": "Custom training job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-16T10:00:00Z", + "display_name": "My Custom Training", + "encryption_spec": null, + "end_time": "2024-08-16T12:00:00Z", + "error": null, + "job_spec": { + "base_output_directory": "gs://my-bucket/custom-job-output", + "enable_dashboard_access": true, + "enable_web_access": null, + "experiment": null, + "experiment_run": null, + "models": [], + "network": null, + "persistent_resource_id": null, + "protected_artifact_location_id": null, + "reserved_ip_ranges": [], + "scheduling": { + "disable_retries": null, + "restart_job_on_worker_restart": null, + "timeout": null + }, + "service_account": "custom-job-sa@my-project.iam.gserviceaccount.com", + "tensorboard": null, + "worker_pool_specs": [ + { + "container_spec": { + "image_uri": "gcr.io/my-project/my-custom-training-image", + "args": [], + "command": [], + "env": [] + }, + "python_package_spec": null, + "machine_spec": null, + "disk_spec": null, + "nfs_mounts": [], + "replica_count": "1" + } + ] + }, + "start_time": "2024-08-16T10:15:00Z", + "state": "SUCCEEDED", + "update_time": "2024-08-16T12:00:00Z", + "web_access_uris": {} + }, + { + "id": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/pipelineJobs/my-pipeline-job", + "ctime": "2024-08-17T14:00:00Z", + "description": "My pipeline job", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-17T14:00:00Z", + "display_name": "My Pipeline", + "encryption_spec": null, + "end_time": "2024-08-17T16:00:00Z", + "error": null, + "job_detail": { + "pipeline_context": {}, + "pipeline_run_context": {}, + "task_details": [] + }, + "network": null, + "pipeline_spec": {}, + "preflight_validations": null, + "reserved_ip_ranges": [], + "runtime_config": { + "failure_policy": null, + "gcs_output_directory": null, + "input_artifacts": {}, + "parameter_values": {}, + "parameters": {} + }, + "schedule_name": null, + "service_account": "pipeline-sa@my-project.iam.gserviceaccount.com", + "start_time": "2024-08-17T14:30:00Z", + "state": "SUCCEEDED", + "template_metadata": "1", + "template_uri": "gs://my-bucket/pipeline.json", + "update_time": "2024-08-17T16:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/tensorboards/my-tensorboard", + "tags": {}, + "name": "projects/my-project/locations/us-central1/tensorboards/my-tensorboard", + "ctime": "2024-08-18T11:30:00Z", + "description": "My Tensorboard instance", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "blob_storage_path_prefix": "gs://my-bucket/tensorboard-logs", + "create_time": "2024-08-18T11:30:00Z", + "display_name": "My Tensorboard", + "encryption_spec": null, + "etag": "abcdef123456", + "isDefault": false, + "run_count": 5, + "satisfiesPzi": false, + "satisfiesPzs": false, + "update_time": "2024-08-18T12:00:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/indexes/my-index", + "tags": {}, + "name": "projects/my-project/locations/us-central1/indexes/my-index", + "ctime": "2024-08-19T16:00:00Z", + "description": "Index for vector similarity search", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-19T16:00:00Z", + "deployed_indexes": [], + "display_name": "My Vector Index", + "encryption_spec": null, + "etag": "abcdef123456", + "index_stats": { + "vectors_count": "10000", + "shards_count": 4, + "sparse_vectors_count": null + }, + "index_update_method": "BATCH_UPDATE", + "_metadata": {}, + "metadata_schema_uri": "gs://my-bucket/index_metadata_schema.json", + "update_time": "2024-08-19T16:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/indexEndpoints/my-index-endpoint", + "tags": {}, + "name": "projects/my-project/locations/us-central1/indexEndpoints/my-index-endpoint", + "ctime": "2024-08-20T09:00:00Z", + "description": "Endpoint for vector similarity search", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "create_time": "2024-08-20T09:00:00Z", + "deployed_indexes": [ + { + "automatic_resources": null, + "deployed_index_auth_config": null, + "deployment_group": null, + "create_time": "2024-08-20T09:15:00Z", + "dedicated_resources": { + "machine_spec": { + "machine_type": "n1-standard-4", + "accelerator_type": null, + "accelerator_count": null, + "tpu_topology": null + }, + "min_replica_count": 2, + "max_replica_count": 4, + "autoscaling_metric_specs": [] + }, + "display_name": "My Deployed Index", + "enable_access_logging": null, + "id": "deployed-index-123", + "index": "projects/my-project/locations/us-central1/indexes/my-index", + "index_sync_time": "2024-08-20T09:20:00Z", + "private_endpoints": null, + "reserved_ip_ranges": [] + } + ], + "display_name": "My Index Endpoint", + "enable_private_service_connect": false, + "encryption_spec": null, + "etag": "abcdef123456", + "network": null, + "private_service_connect_config": null, + "public_endpoint_domain_name": "my-index-endpoint.aiplatform.googleusercontent.com", + "public_endpoint_enabled": true, + "update_time": "2024-08-20T09:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/modelDeploymentMonitoringJobs/my-monitoring-job", + "tags": {}, + "name": "projects/my-project/locations/us-central1/modelDeploymentMonitoringJobs/my-monitoring-job", + "ctime": "2024-08-21T11:00:00Z", + "description": "Monitoring job for deployed model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "analysis_instance_schema_uri": null, + "bigquery_tables": [], + "create_time": "2024-08-21T11:00:00Z", + "display_name": "My Monitoring Job", + "enable_monitoring_pipeline_logs": false, + "encryption_spec": null, + "endpoint": "projects/my-project/locations/us-central1/endpoints/my-endpoint", + "error": null, + "latest_monitoring_pipeline_metadata": null, + "log_ttl": null, + "logging_sampling_strategy": null, + "model_deployment_monitoring_objective_configs": [], + "model_deployment_monitoring_schedule_config": null, + "model_monitoring_alert_config": null, + "next_schedule_time": null, + "predict_instance_schema_uri": null, + "sample_predict_instance": {}, + "schedule_state": null, + "state": "JOB_STATE_UNSPECIFIED", + "stats_anomalies_base_directory": null, + "update_time": "2024-08-21T11:30:00Z" + }, + { + "id": "projects/my-project/locations/us-central1/tuningJobs/my-tuning-job-2", + "tags": { + "experiment_id": "exp123" + }, + "name": "projects/my-project/locations/us-central1/tuningJobs/my-tuning-job-2", + "ctime": "2024-08-22T13:00:00Z", + "description": "Tuning job for a language model", + "link": "https://...", + "label_fingerprint": "abcdef123456", + "deprecation_status": { + "deprecated": false, + "state": "LIVE" + }, + "base_model": "projects/my-project/locations/us-central1/models/my-base-model", + "create_time": "2024-08-22T13:00:00Z", + "encryption_spec": null, + "end_time": "2024-08-22T15:00:00Z", + "error": null, + "experiment": "projects/my-project/locations/us-central1/experiments/exp123", + "start_time": "2024-08-22T13:15:00Z", + "state": "SUCCEEDED", + "supervised_tuning_spec": null, + "tuned_model": null, + "tuned_model_display_name": null, + "tuning_data_stats": null, + "update_time": "2024-08-22T15:00:00Z" + } + ] +} \ No newline at end of file