diff --git a/.codegen.json b/.codegen.json index a1f1732c6..3a5a817d3 100644 --- a/.codegen.json +++ b/.codegen.json @@ -1,9 +1,6 @@ { "formatter": "make fmt", "mode": "tf_v1", - "packages": { - ".codegen/model.go.tmpl": "internal/service/{{.Name}}_tf/model.go" - }, "changelog_config": ".codegen/changelog_config.yml", "version": { "common/version.go": "version = \"$VERSION\"" diff --git a/.codegen/model.go.tmpl b/.codegen/model.go.tmpl deleted file mode 100644 index 55145cbc5..000000000 --- a/.codegen/model.go.tmpl +++ /dev/null @@ -1,291 +0,0 @@ -// Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. -/* -These generated types are for terraform plugin framework to interact with the terraform state conveniently. - -These types follow the same structure as the types in go-sdk. -The only difference is that the primitive types are no longer using the go-native types, but with tfsdk types. -Plus the json tags get converted into tfsdk tags. -We use go-native types for lists and maps intentionally for the ease for converting these types into the go-sdk types. -*/ - -package {{.Name}}_tf - -import ( - {{range .ImportedPackages}} - "github.com/databricks/databricks-sdk-go/service/{{.}}"{{end}} - "github.com/databricks/databricks-sdk-go/service/{{.Name}}" - "io" - "github.com/databricks/databricks-sdk-go/marshal" - pluginfwcommon "github.com/databricks/terraform-provider-databricks/internal/providers/pluginfw/common" - "github.com/hashicorp/terraform-plugin-framework/types" - "github.com/hashicorp/terraform-plugin-framework/types/basetypes" -) -{{range .Types}} -{{- if or .Fields .IsEmpty}} -{{.Comment "// " 80}} -type {{.PascalName}} struct { - {{- range .Fields}} - {{.Comment " // " 80}} - {{- $data := dict "field" . }} - {{template "field" $data}} - {{- if and .Entity.Terraform .Entity.Terraform.IsServiceProposedIfEmpty -}} - {{- $data := dict "field" . "effective" true }} - {{template "field" $data}} - {{- end -}} - {{end}} -} - -func (newState *{{.PascalName}}) SyncEffectiveFieldsDuringCreateOrUpdate(plan {{.PascalName}}) { - {{- range .Fields -}} - {{- if and (and (ne .Entity.Terraform nil) .Entity.Terraform.IsServiceProposedIfEmpty) (or .Entity.IsString .Entity.IsBool .Entity.IsInt64 .Entity.IsFloat64 .Entity.IsInt .Entity.Enum)}} - newState.Effective{{.PascalName}} = newState.{{.PascalName}} - newState.{{.PascalName}} = plan.{{.PascalName}} - {{- end}} - {{- end}} -} - -func (newState *{{.PascalName}}) SyncEffectiveFieldsDuringRead(existingState {{.PascalName}}) { - {{- range .Fields -}} - {{- if and (and (ne .Entity.Terraform nil) .Entity.Terraform.IsServiceProposedIfEmpty) (or .Entity.IsString .Entity.IsBool .Entity.IsInt64 .Entity.IsFloat64 .Entity.IsInt .Entity.Enum)}} - {{- $type := "" -}} - {{- if .Entity.IsString}}{{$type = "String"}}{{end}} - {{- if .Entity.IsBool}}{{$type = "Bool"}}{{end}} - {{- if .Entity.IsInt64}}{{$type = "Int64"}}{{end}} - {{- if .Entity.IsFloat64}}{{$type = "Float64"}}{{end}} - {{- if .Entity.IsInt}}{{$type = "Int64"}}{{end}} - {{- if .Entity.Enum}}{{$type = "String"}}{{end}} - newState.Effective{{.PascalName}} = existingState.Effective{{.PascalName}} - if existingState.Effective{{.PascalName}}.Value{{$type}}() == newState.{{.PascalName}}.Value{{$type}}() { - newState.{{.PascalName}} = existingState.{{.PascalName}} - } - {{- end}} - {{- end}} -} - -// GetComplexFieldTypes returns a map of the types of elements in complex fields in {{.PascalName}}. -// Container types (types.Map, types.List, types.Set) and object types (types.Object) do not carry -// the type information of their elements in the Go type system. This function provides a way to -// retrieve the type information of the elements in complex fields at runtime. The values of the map -// are the reflected types of the contained elements. They must be either primitive values from the -// plugin framework type system (types.String{}, types.Bool{}, types.Int64{}, types.Float64{}) or TF -// SDK values. -func (a {{.PascalName}}) GetComplexFieldTypes(ctx context.Context) map[string]reflect.Type { - return map[string]reflect.Type{ - {{- range .Fields -}} - {{- if or .Entity.IsObject .Entity.ArrayValue .Entity.MapValue}} - {{ $data := dict "field" . -}} - "{{template "tfsdk-name" $data}}": reflect.TypeOf({{ template "complex-field-value" .Entity }}), - {{- end}} - {{- end}} - } -} - -// TFSDK types cannot implement the ObjectValuable interface directly, as it would otherwise -// interfere with how the plugin framework retrieves and sets values in state. Thus, {{.PascalName}} -// only implements ToObjectValue() and Type(). -func (o {{.PascalName}}) ToObjectValue(ctx context.Context) basetypes.ObjectValue { - return types.ObjectValueMust( - o.Type(ctx).(basetypes.ObjectType).AttrTypes, - map[string]attr.Value{ - {{ range .Fields -}} - {{- $data := dict "field" . -}} - "{{template "tfsdk-name" $data}}": o.{{template "field-name" $data}}, - {{if and .Entity.Terraform .Entity.Terraform.IsServiceProposedIfEmpty -}} - {{- $data := dict "field" . "effective" true }} - "{{template "tfsdk-name" $data}}": o.{{template "field-name" $data}}, - {{- end -}} - {{- end}} - }) -} - -// Type implements basetypes.ObjectValuable. -func (o {{.PascalName}}) Type(ctx context.Context) attr.Type { - return types.ObjectType{ - AttrTypes: map[string]attr.Type{ - {{ range .Fields -}} - {{ $data := dict "field" . -}} - "{{template "tfsdk-name" $data}}": {{ template "attr-type" .Entity }}, - {{ end}} - }, - } -} - -{{/* Getters and setters for types.List and types.Map fields. */}} -{{ $parent := . }} -{{ range .Fields }} -{{ if or .Entity.IsObject .Entity.ArrayValue .Entity.MapValue }} -// Get{{.PascalName}} returns the value of the {{.PascalName}} field in {{ $parent.PascalName}} as -// {{ if .Entity.IsObject }}a {{template "complex-field-type" .Entity}} value{{ else if .Entity.ArrayValue }}a slice of {{template "complex-field-type" .Entity}} values{{ else }}a map of string to {{template "complex-field-type" .Entity}} values{{ end }}. -// If the field is unknown or null, the boolean return value is false. -func (o *{{ $parent.PascalName}}) Get{{.PascalName}}(ctx context.Context) {{template "getter-type" .Entity}} { - {{if .Entity.IsObject -}} - var e {{template "complex-field-type" .Entity}} - {{ end -}} - if o.{{.PascalName}}.IsNull() || o.{{.PascalName}}.IsUnknown() { - {{if .Entity.IsObject -}} - return e, false - {{- else -}} - return nil, false - {{- end}} - } - var v {{if .Entity.IsObject}}[]{{end}}{{template "getter-setter-type" .Entity}} - d := o.{{.PascalName}}.ElementsAs(ctx, &v, true) - if d.HasError() { - panic(pluginfwcommon.DiagToString(d)) - } - {{if .Entity.IsObject -}} - if len(v) == 0 { - return e, false - } - return v[0], true - {{- else -}} - return v, true - {{- end}} -} - -// Set{{.PascalName}} sets the value of the {{.PascalName}} field in {{ $parent.PascalName}}. -func (o *{{ $parent.PascalName}}) Set{{.PascalName}}(ctx context.Context, v {{template "getter-setter-type" .Entity}}) { - {{ if .Entity.IsObject -}} - vs := []attr.Value{v.ToObjectValue(ctx)} - {{- else if .Entity.ArrayValue -}} - vs := make([]attr.Value, 0, len(v)) - for _, e := range v { - vs = append(vs, e{{if or (and .Entity.ArrayValue .Entity.ArrayValue.IsObject) (and .Entity.MapValue .Entity.MapValue.IsObject)}}.ToObjectValue(ctx){{end}}) - } - {{- else -}} - vs := make(map[string]attr.Value, len(v)) - for k, e := range v { - vs[k] = e{{if or (and .Entity.ArrayValue .Entity.ArrayValue.IsObject) (and .Entity.MapValue .Entity.MapValue.IsObject)}}.ToObjectValue(ctx){{end}} - } - {{- end}} - t := o.Type(ctx).(basetypes.ObjectType).AttrTypes["{{template "tfsdk-name" (dict "field" .)}}"] - {{- if or .Entity.ArrayValue .Entity.MapValue }} - t = t.(attr.TypeWithElementType).ElementType() - {{- end }} - o.{{.PascalName}}{{if eq .PascalName "Type"}}_{{end}} = types.{{if .Entity.MapValue}}Map{{else}}List{{end}}ValueMust(t, vs) -} -{{end}} - -{{end}} -{{end}} -{{end}} - -{{- define "getter-type" -}} -({{ template "getter-setter-type" . }}, bool) -{{- end -}} - -{{- define "getter-setter-type" -}} -{{- if .ArrayValue -}} - []{{ template "complex-field-type" . }} -{{- else if .IsObject -}} - {{ template "complex-field-type" . }} -{{- else -}} - map[string]{{ template "complex-field-type" . }} -{{- end -}} -{{- end -}} - -{{- define "complex-field-value" -}} -{{ template "complex-field-type" . }}{} -{{- end -}} - -{{- define "complex-field-type" -}} - {{- if .ArrayValue }}{{ template "complex-field-type" .ArrayValue }} - {{- else if .MapValue }}{{ template "complex-field-type" .MapValue }} - {{- else -}} - {{- if or .IsString .Enum -}}types.String - {{- else if .IsBool -}}types.Bool - {{- else if .IsInt64 -}}types.Int64 - {{- else if .IsFloat64 -}}types.Float64 - {{- else if .IsInt -}}types.Int64 - {{- else if .IsAny -}}types.Object - {{- else if or .IsEmpty .IsObject -}}{{if .IsExternal}}{{.Package.Name}}_tf.{{end}}{{.PascalName}} - {{- end -}} - {{- end -}} -{{- end -}} - -{{/* -Jobs has a recursive structure: Tasks contain ForEachTasks, which contain Tasks. -Because of this, GetComplexFieldTypes and ToObjectType will never terminate. -TODO: capture visited types in the context to ensure these methods terminate, -even when they are called recursively. -*/}} -{{- define "attr-type" -}} - {{- if .ArrayValue -}} - {{- if .ArrayValue.IsObject -}}{{/* Objects are wraped in lists automatically. */}} - {{ template "attr-type" .ArrayValue }} - {{- else -}} - basetypes.ListType{ - ElemType: {{ template "attr-type" .ArrayValue }}, - } - {{- end -}} - {{- else if .MapValue }}basetypes.MapType{ - ElemType: {{ template "attr-type" .MapValue }}, - } - {{- else -}} - {{- if or .IsString .Enum -}}types.StringType - {{- else if .IsBool -}}types.BoolType - {{- else if .IsInt64 -}}types.Int64Type - {{- else if .IsFloat64 -}}types.Float64Type - {{- else if .IsInt -}}types.Int64Type - {{- else if .IsAny -}}types.ObjectType{} - {{- else if .IsByteStream}}types.ObjectType{} - {{- else if or .IsEmpty .IsObject -}}{{/* Objects are treated as lists from a TFSDK type perspective. */}}basetypes.ListType{ - ElemType: {{- if .IsExternal -}}{{.Package.Name}}_tf.{{- end -}}{{.PascalName}}{}.Type(ctx), - } - {{- end -}} - {{- end -}} -{{- end -}} - - -{{- define "field" -}} -{{template "field-name" .}} {{template "type" .field.Entity}} `{{template "field-tag" . }}` -{{- end -}} - -{{- define "field-name" -}} -{{if .effective}}Effective{{end}}{{.field.PascalName}}{{if eq .field.PascalName "Type"}}_{{end}} -{{- end -}} - -{{- define "field-tag" -}} - {{- $annotations := "" -}} - {{- if or .field.Entity.IsComputed .effective -}} - {{- $annotations = (printf "%scomputed,optional," $annotations) -}} - {{- else -}} - {{- if not .field.Required -}} - {{- $annotations = (printf "%soptional," $annotations) -}} - {{- end -}} - {{- if .field.Entity.IsObject -}} - {{- $annotations = (printf "%sobject," $annotations) -}} - {{- end -}} - {{- end -}} - {{- if gt (len $annotations) 0 -}} - {{- $annotations = (printf "%s" (trimSuffix "," $annotations)) -}} - {{- end -}} - {{if .field.IsJson}}tfsdk:"{{ template "tfsdk-name" . }}" tf:"{{$annotations}}"{{else}}tfsdk:"-"{{end -}} -{{- end -}} - -{{- define "tfsdk-name" -}} -{{- if and (ne .field.Entity.Terraform nil) (ne .field.Entity.Terraform.Alias "") -}} -{{.field.Entity.Terraform.Alias}} -{{- else -}} -{{if .effective}}effective_{{end}}{{.field.Name}} -{{- end -}} -{{- end -}} - -{{- define "type" -}} - {{- if not . }}any /* ERROR */ - {{- else if .IsExternal }}types.List{{/* Note: we use types.List for objects for now. TODO: change this to types.Object. */}} - {{- else if .IsAny}}types.Object - {{- else if .IsEmpty}}types.List - {{- else if .IsString}}types.String - {{- else if .IsBool}}types.Bool - {{- else if .IsInt64}}types.Int64 - {{- else if .IsFloat64}}types.Float64 - {{- else if .IsInt}}types.Int64 - {{- else if .IsByteStream}}types.Object - {{- else if .ArrayValue }}types.List - {{- else if .MapValue }}types.Map - {{- else if .IsObject }}types.List{{/* Note: we use types.List for objects for now. TODO: change this to types.Object. */}} - {{- else if .Enum }}types.String - {{- else}}any /* MISSING TYPE */ - {{- end -}} -{{- end -}} \ No newline at end of file diff --git a/catalog/resource_credential.go b/catalog/resource_credential.go index f0dc62de4..4cf77e319 100644 --- a/catalog/resource_credential.go +++ b/catalog/resource_credential.go @@ -94,6 +94,14 @@ func ResourceCredential() common.Resource { if err != nil { return err } + // azure client secret is sensitive, so we need to preserve it + var credOrig catalog.CredentialInfo + common.DataToStructPointer(d, credentialSchema, &credOrig) + if credOrig.AzureServicePrincipal != nil { + if credOrig.AzureServicePrincipal.ClientSecret != "" { + cred.AzureServicePrincipal.ClientSecret = credOrig.AzureServicePrincipal.ClientSecret + } + } d.Set("credential_id", cred.Id) return common.StructToData(cred, credentialSchema, d) }, diff --git a/docs/data-sources/jobs.md b/docs/data-sources/jobs.md index c56aaec94..077b6d366 100644 --- a/docs/data-sources/jobs.md +++ b/docs/data-sources/jobs.md @@ -16,6 +16,10 @@ Granting view [databricks_permissions](../resources/permissions.md) to all [data ```hcl data "databricks_jobs" "this" {} +data "databricks_jobs" "tests" { + job_name_contains = "test" +} + resource "databricks_permissions" "everyone_can_view_all_jobs" { for_each = data.databricks_jobs.this.ids job_id = each.value @@ -38,6 +42,10 @@ output "x" { } ``` +## Argument Reference + +* `job_name_contains` - (Optional) Only return [databricks_job](../resources/job.md#) ids that match the given name string (case-insensitive). + ## Attribute Reference This data source exports the following attributes: diff --git a/exporter/importables.go b/exporter/importables.go index 0aa38acd8..a08cfc7c1 100644 --- a/exporter/importables.go +++ b/exporter/importables.go @@ -2271,6 +2271,8 @@ var resourcesMap map[string]importable = map[string]importable{ }, Body: resourceOrDataBlockBody, Depends: []reference{ + {Path: "path", Resource: "databricks_user", Match: "home"}, + {Path: "path", Resource: "databricks_service_principal", Match: "home"}, // TODO: it should try to find longest reference to another directory object that it not itself... {Path: "path", Resource: "databricks_user", Match: "home", MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName}, diff --git a/jobs/data_jobs.go b/jobs/data_jobs.go index a3d6a327c..abc158c12 100644 --- a/jobs/data_jobs.go +++ b/jobs/data_jobs.go @@ -3,29 +3,36 @@ package jobs import ( "context" "fmt" + "strconv" + "strings" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/terraform-provider-databricks/common" ) func DataSourceJobs() common.Resource { - type jobsData struct { - Ids map[string]string `json:"ids,omitempty" tf:"computed"` - } - return common.DataResource(jobsData{}, func(ctx context.Context, e any, c *common.DatabricksClient) error { - response := e.(*jobsData) - jobsAPI := NewJobsAPI(ctx, c) - list, err := jobsAPI.List() - if err != nil { - return err - } - response.Ids = map[string]string{} - for _, v := range list { - name := v.Settings.Name - _, duplicateName := response.Ids[name] + return common.WorkspaceData(func(ctx context.Context, data *struct { + Ids map[string]string `json:"ids,omitempty" tf:"computed"` + NameFilter string `json:"job_name_contains,omitempty"` + }, w *databricks.WorkspaceClient) error { + iter := w.Jobs.List(ctx, jobs.ListJobsRequest{ExpandTasks: false, Limit: 100}) + data.Ids = map[string]string{} + nameFilter := strings.ToLower(data.NameFilter) + for iter.HasNext(ctx) { + job, err := iter.Next(ctx) + if err != nil { + return err + } + name := job.Settings.Name + if nameFilter != "" && !strings.Contains(strings.ToLower(name), nameFilter) { + continue + } + _, duplicateName := data.Ids[name] if duplicateName { return fmt.Errorf("duplicate job name detected: %s", name) } - response.Ids[name] = v.ID() + data.Ids[name] = strconv.FormatInt(job.JobId, 10) } return nil }) diff --git a/jobs/data_jobs_test.go b/jobs/data_jobs_test.go index a51683260..327240f2c 100644 --- a/jobs/data_jobs_test.go +++ b/jobs/data_jobs_test.go @@ -11,7 +11,7 @@ func TestJobsData(t *testing.T) { Fixtures: []qa.HTTPFixture{ { Method: "GET", - Resource: "/api/2.1/jobs/list?expand_tasks=false&limit=25", + Resource: "/api/2.1/jobs/list?limit=100", Response: JobListResponse{ Jobs: []Job{ { @@ -41,3 +41,39 @@ func TestJobsData(t *testing.T) { }, }) } + +func TestJobsDataWithFilter(t *testing.T) { + qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.1/jobs/list?limit=100", + Response: JobListResponse{ + Jobs: []Job{ + { + JobID: 123, + Settings: &JobSettings{ + Name: "First", + }, + }, + { + JobID: 234, + Settings: &JobSettings{ + Name: "Second", + }, + }, + }, + }, + }, + }, + Resource: DataSourceJobs(), + Read: true, + NonWritable: true, + ID: "_", + HCL: `job_name_contains = "Sec"`, + }.ApplyAndExpectData(t, map[string]any{ + "ids": map[string]any{ + "Second": "234", + }, + }) +} diff --git a/jobs/resource_job.go b/jobs/resource_job.go index e619ceac4..7b93f295c 100644 --- a/jobs/resource_job.go +++ b/jobs/resource_job.go @@ -677,8 +677,7 @@ func (a JobsAPI) ListByName(name string, expandTasks bool) ([]Job, error) { // List all jobs func (a JobsAPI) List() (l []Job, err error) { - l, err = a.ListByName("", false) - return + return a.ListByName("", false) } // RunsList returns a job runs list