From e237d134127c5703bbeeac2486d43316c2f7add4 Mon Sep 17 00:00:00 2001 From: Emil Christensen Date: Tue, 8 Oct 2024 17:56:22 -0400 Subject: [PATCH] Adds support for cloud-run-v2, modal work pool metadata (#279) * Adds cloud run v2 to worker_metadata * Fixes test in TestAccDatasource_worker_metadata * Use the same endpoints as the Prefect client The endpoint to list the base job configs differs between OSS and Cloud. This change calculates the correct endpoint, and updates the test to refer to the specific evergreen workspace used for testing. * Generate Terraform Docs * Support Modal push workers * Generate Terraform Docs --------- Co-authored-by: Mitchell Nielsen Co-authored-by: github-actions[bot] --- docs/data-sources/worker_metadata.md | 8 ++++ internal/api/client.go | 2 +- internal/client/collections.go | 27 +++++++++++-- .../provider/datasources/worker_metadata.go | 38 ++++++++++++++++++- .../datasources/worker_metadata_test.go | 22 +++++++++-- 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/docs/data-sources/worker_metadata.md b/docs/data-sources/worker_metadata.md index a0dc4fe2..3c19c5bc 100644 --- a/docs/data-sources/worker_metadata.md +++ b/docs/data-sources/worker_metadata.md @@ -50,6 +50,11 @@ resource "prefect_work_pool" "process" { ## Schema +### Optional + +- `account_id` (String) Account ID (UUID), defaults to the account set in the provider +- `workspace_id` (String) Workspace ID (UUID), defaults to the workspace set in the provider + ### Read-Only - `base_job_configs` (Attributes) A map of default base job configurations (JSON) for each of the primary worker types (see [below for nested schema](#nestedatt--base_job_configs)) @@ -63,10 +68,13 @@ Read-Only: - `azure_container_instances_push` (String) Default base job configuration for Azure Container Instances Push workers - `cloud_run` (String) Default base job configuration for Cloud Run workers - `cloud_run_push` (String) Default base job configuration for Cloud Run Push workers +- `cloud_run_v2` (String) Default base job configuration for Cloud Run V2 workers +- `cloud_run_v2_push` (String) Default base job configuration for Cloud Run V2 Push workers - `docker` (String) Default base job configuration for Docker workers - `ecs` (String) Default base job configuration for ECS workers - `ecs_push` (String) Default base job configuration for ECS Push workers - `kubernetes` (String) Default base job configuration for Kubernetes workers +- `modal_push` (String) Default base job configuration for Modal Push workers - `prefect_agent` (String) Default base job configuration for Prefect Agent workers - `process` (String) Default base job configuration for Process workers - `vertex_ai` (String) Default base job configuration for Vertex AI workers diff --git a/internal/api/client.go b/internal/api/client.go index 3e998da4..049fc65f 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -12,7 +12,7 @@ type PrefectClient interface { BlockDocuments(accountID uuid.UUID, workspaceID uuid.UUID) (BlockDocumentClient, error) BlockSchemas(accountID uuid.UUID, workspaceID uuid.UUID) (BlockSchemaClient, error) BlockTypes(accountID uuid.UUID, workspaceID uuid.UUID) (BlockTypeClient, error) - Collections() (CollectionsClient, error) + Collections(accountID uuid.UUID, workspaceID uuid.UUID) (CollectionsClient, error) Deployments(accountID uuid.UUID, workspaceID uuid.UUID) (DeploymentsClient, error) Teams(accountID uuid.UUID) (TeamsClient, error) Flows(accountID uuid.UUID, workspaceID uuid.UUID) (FlowsClient, error) diff --git a/internal/client/collections.go b/internal/client/collections.go index 4886a0fa..f6ebe6de 100644 --- a/internal/client/collections.go +++ b/internal/client/collections.go @@ -7,7 +7,9 @@ import ( "io" "net/http" + "github.com/google/uuid" "github.com/prefecthq/terraform-provider-prefect/internal/api" + "github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers" ) var _ = api.CollectionsClient(&CollectionsClient{}) @@ -21,18 +23,37 @@ type CollectionsClient struct { // Collections returns an CollectionsClient. // //nolint:ireturn // required to support PrefectClient mocking -func (c *Client) Collections() (api.CollectionsClient, error) { +func (c *Client) Collections(accountID, workspaceID uuid.UUID) (api.CollectionsClient, error) { + if accountID == uuid.Nil { + accountID = c.defaultAccountID + } + + if workspaceID == uuid.Nil { + workspaceID = c.defaultWorkspaceID + } + + if helpers.IsCloudEndpoint(c.endpoint) && (accountID == uuid.Nil || workspaceID == uuid.Nil) { + return nil, fmt.Errorf("prefect Cloud endpoints require an account_id and workspace_id to be set on either the provider or the resource") + } + return &CollectionsClient{ hc: c.hc, apiKey: c.apiKey, - routePrefix: fmt.Sprintf("%s/collections", c.endpoint), + routePrefix: getWorkspaceScopedURL(c.endpoint, accountID, workspaceID, "collections"), }, nil } // GetWorkerMetadataViews returns a map of worker metadata views by prefect package name. // This endpoint serves base job configurations for the primary worker types. func (c *CollectionsClient) GetWorkerMetadataViews(ctx context.Context) (api.WorkerTypeByPackage, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/views/aggregate-worker-metadata", c.routePrefix), http.NoBody) + routeSuffix := "views/aggregate-worker-metadata" + if helpers.IsCloudEndpoint(c.routePrefix) { + routeSuffix = "work_pool_types" + } + + url := fmt.Sprintf("%s/%s", c.routePrefix, routeSuffix) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) } diff --git a/internal/provider/datasources/worker_metadata.go b/internal/provider/datasources/worker_metadata.go index 76d2c115..31ef7a0a 100644 --- a/internal/provider/datasources/worker_metadata.go +++ b/internal/provider/datasources/worker_metadata.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/datasource/schema" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/prefecthq/terraform-provider-prefect/internal/api" + "github.com/prefecthq/terraform-provider-prefect/internal/provider/customtypes" "github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers" ) @@ -18,6 +19,9 @@ type WorkerMetadataDataSource struct { } type WorkerMetadataDataSourceModel struct { + AccountID customtypes.UUIDValue `tfsdk:"account_id"` + WorkspaceID customtypes.UUIDValue `tfsdk:"workspace_id"` + BaseJobConfigs types.Object `tfsdk:"base_job_configs"` } @@ -58,6 +62,16 @@ Get metadata information about the common Worker types, such as Kubernetes, ECS, Use this data source to get the default base job configurations for those common Worker types. `, Attributes: map[string]schema.Attribute{ + "account_id": schema.StringAttribute{ + CustomType: customtypes.UUIDType{}, + Description: "Account ID (UUID), defaults to the account set in the provider", + Optional: true, + }, + "workspace_id": schema.StringAttribute{ + CustomType: customtypes.UUIDType{}, + Description: "Workspace ID (UUID), defaults to the workspace set in the provider", + Optional: true, + }, "base_job_configs": schema.SingleNestedAttribute{ Computed: true, Description: "A map of default base job configurations (JSON) for each of the primary worker types", @@ -87,6 +101,11 @@ Use this data source to get the default base job configurations for those common Description: "Default base job configuration for Cloud Run workers", CustomType: jsontypes.NormalizedType{}, }, + "cloud_run_v2": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Cloud Run V2 workers", + CustomType: jsontypes.NormalizedType{}, + }, "vertex_ai": schema.StringAttribute{ Computed: true, Description: "Default base job configuration for Vertex AI workers", @@ -112,11 +131,21 @@ Use this data source to get the default base job configurations for those common Description: "Default base job configuration for Cloud Run Push workers", CustomType: jsontypes.NormalizedType{}, }, + "cloud_run_v2_push": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Cloud Run V2 Push workers", + CustomType: jsontypes.NormalizedType{}, + }, "ecs_push": schema.StringAttribute{ Computed: true, Description: "Default base job configuration for ECS Push workers", CustomType: jsontypes.NormalizedType{}, }, + "modal_push": schema.StringAttribute{ + Computed: true, + Description: "Default base job configuration for Modal Push workers", + CustomType: jsontypes.NormalizedType{}, + }, }, }, }, @@ -133,7 +162,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read return } - client, err := d.client.Collections() + client, err := d.client.Collections(model.AccountID.ValueUUID(), model.WorkspaceID.ValueUUID()) if err != nil { resp.Diagnostics.Append(helpers.CreateClientErrorDiagnostic("Collections", err)) @@ -157,18 +186,22 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read } // https://developer.hashicorp.com/terraform/plugin/framework/handling-data/types/object#setting-values + // https://docs.prefect.io/latest/deploy/infrastructure-concepts/work-pools#work-pool-types attributeTypes := map[string]attr.Type{ "kubernetes": jsontypes.NormalizedType{}, "ecs": jsontypes.NormalizedType{}, "azure_container_instances": jsontypes.NormalizedType{}, "docker": jsontypes.NormalizedType{}, "cloud_run": jsontypes.NormalizedType{}, + "cloud_run_v2": jsontypes.NormalizedType{}, "vertex_ai": jsontypes.NormalizedType{}, "prefect_agent": jsontypes.NormalizedType{}, "process": jsontypes.NormalizedType{}, "azure_container_instances_push": jsontypes.NormalizedType{}, "cloud_run_push": jsontypes.NormalizedType{}, + "cloud_run_v2_push": jsontypes.NormalizedType{}, "ecs_push": jsontypes.NormalizedType{}, + "modal_push": jsontypes.NormalizedType{}, } attributeValues := map[string]attr.Value{ "kubernetes": jsontypes.NewNormalizedValue(string(remap["kubernetes"])), @@ -176,12 +209,15 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read "azure_container_instances": jsontypes.NewNormalizedValue(string(remap["azure-container-instance"])), "docker": jsontypes.NewNormalizedValue(string(remap["docker"])), "cloud_run": jsontypes.NewNormalizedValue(string(remap["cloud-run"])), + "cloud_run_v2": jsontypes.NewNormalizedValue(string(remap["cloud-run-v2"])), "vertex_ai": jsontypes.NewNormalizedValue(string(remap["vertex-ai"])), "prefect_agent": jsontypes.NewNormalizedValue(string(remap["prefect-agent"])), "process": jsontypes.NewNormalizedValue(string(remap["process"])), "azure_container_instances_push": jsontypes.NewNormalizedValue(string(remap["azure-container-instance:push"])), "cloud_run_push": jsontypes.NewNormalizedValue(string(remap["cloud-run:push"])), + "cloud_run_v2_push": jsontypes.NewNormalizedValue(string(remap["cloud-run-v2:push"])), "ecs_push": jsontypes.NewNormalizedValue(string(remap["ecs:push"])), + "modal_push": jsontypes.NewNormalizedValue(string(remap["modal:push"])), } obj, diag := types.ObjectValue(attributeTypes, attributeValues) diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go index 1be035f5..e63b850c 100644 --- a/internal/provider/datasources/worker_metadata_test.go +++ b/internal/provider/datasources/worker_metadata_test.go @@ -1,6 +1,8 @@ package datasources_test import ( + "fmt" + "os" "testing" "github.com/hashicorp/terraform-plugin-testing/helper/resource" @@ -8,9 +10,18 @@ import ( ) func fixtureAccWorkerMetadtata() string { - return ` -data "prefect_worker_metadata" "default" {} -` + aID := os.Getenv("PREFECT_CLOUD_ACCOUNT_ID") + + return fmt.Sprintf(` +data "prefect_workspace" "evergreen" { + handle = "github-ci-tests" +} + +data "prefect_worker_metadata" "default" { + account_id = "%s" + workspace_id = data.prefect_workspace.evergreen.id +} +`, aID) } //nolint:paralleltest // we use the resource.ParallelTest helper instead @@ -24,17 +35,20 @@ func TestAccDatasource_worker_metadata(t *testing.T) { { Config: fixtureAccWorkerMetadtata(), Check: resource.ComposeAggregateTestCheckFunc( - resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "11"), + resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "14"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.kubernetes"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.docker"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.vertex_ai"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.prefect_agent"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.process"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_push"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2_push"), + resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.modal_push"), resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs_push"), ), },