From e2c1b7def35506640be1ae10dea161d508cd82a6 Mon Sep 17 00:00:00 2001 From: Sandro Date: Fri, 29 Jan 2021 18:07:42 +1100 Subject: [PATCH 1/6] Introduce support for step-types --- client/step_types.go | 137 ++++++++++++++++++++++++ codefresh/data_step_types.go | 65 ++++++++++++ codefresh/data_step_types_versions.go | 45 ++++++++ codefresh/provider.go | 29 +++--- codefresh/resource_step_types.go | 145 ++++++++++++++++++++++++++ docs/data/step-types-versions.md | 23 ++++ docs/data/step-types.md | 24 +++++ docs/resources/step-types.md | 47 +++++++++ 8 files changed, 502 insertions(+), 13 deletions(-) create mode 100644 client/step_types.go create mode 100644 codefresh/data_step_types.go create mode 100644 codefresh/data_step_types_versions.go create mode 100644 codefresh/resource_step_types.go create mode 100644 docs/data/step-types-versions.md create mode 100644 docs/data/step-types.md create mode 100644 docs/resources/step-types.md diff --git a/client/step_types.go b/client/step_types.go new file mode 100644 index 0000000..bf07eff --- /dev/null +++ b/client/step_types.go @@ -0,0 +1,137 @@ +package client + +import ( + "fmt" + "log" + "net/url" +) + +type StepTypes struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` + Spec map[string]interface{} `json:"spec,omitempty"` +} + +func (stepTypes *StepTypes) GetID() string { + return stepTypes.Metadata["name"].(string) +} + +func (client *Client) GetStepTypesVersions(name string) ([]string, error) { + fullPath := fmt.Sprintf("/step-types/%s/versions", url.PathEscape(name)) + opts := RequestOptions{ + Path: fullPath, + Method: "GET", + } + + resp, err := client.RequestAPI(&opts) + + if err != nil { + return nil, err + } + var respStepTypesVersions []string + err = DecodeResponseInto(resp, &respStepTypesVersions) + if err != nil { + return nil, err + } + return respStepTypesVersions, nil +} + +func (client *Client) GetStepTypes(identifier string) (*StepTypes, error) { + fullPath := fmt.Sprintf("/step-types/%s", url.PathEscape(identifier)) + opts := RequestOptions{ + Path: fullPath, + Method: "GET", + } + + resp, err := client.RequestAPI(&opts) + + if err != nil { + return nil, err + } + var respStepTypes StepTypes + err = DecodeResponseInto(resp, &respStepTypes) + if err != nil { + return nil, err + } + + return &respStepTypes, nil + +} + +func (client *Client) CreateStepTypes(stepTypes *StepTypes) (*StepTypes, error) { + + body, err := EncodeToJSON(stepTypes) + + if err != nil { + return nil, err + } + opts := RequestOptions{ + Path: "/step-types", + Method: "POST", + Body: body, + } + + resp, err := client.RequestAPI(&opts) + if err != nil { + return nil, err + } + log.Printf("[DEBUG] Response step types: %q", resp) + var respStepTypes StepTypes + err = DecodeResponseInto(resp, &respStepTypes) + if err != nil { + log.Printf("[DEBUG] Error while decoding step types. Error = %v, Response: %q", err, respStepTypes) + return nil, err + } + log.Printf("[DEBUG] Decoded step types response: %q", respStepTypes.Metadata["name"]) + return &respStepTypes, nil + +} + +func (client *Client) UpdateStepTypes(stepTypes *StepTypes) (*StepTypes, error) { + + body, err := EncodeToJSON(stepTypes) + + if err != nil { + return nil, err + } + + fullPath := fmt.Sprintf("/step-types/%s", url.PathEscape(stepTypes.GetID()+":"+stepTypes.Metadata["version"].(string))) + opts := RequestOptions{ + Path: fullPath, + Method: "PUT", + Body: body, + } + + resp, err := client.RequestAPI(&opts) + + if err != nil { + return nil, err + } + + var respStepTypes StepTypes + err = DecodeResponseInto(resp, &respStepTypes) + if err != nil { + return nil, err + } + + return &respStepTypes, nil + +} + +func (client *Client) DeleteStepTypes(name string) error { + + fullPath := fmt.Sprintf("/step-types/%s", url.PathEscape(name)) + opts := RequestOptions{ + Path: fullPath, + Method: "DELETE", + } + + _, err := client.RequestAPI(&opts) + + if err != nil { + return err + } + + return nil +} diff --git a/codefresh/data_step_types.go b/codefresh/data_step_types.go new file mode 100644 index 0000000..98635eb --- /dev/null +++ b/codefresh/data_step_types.go @@ -0,0 +1,65 @@ +package codefresh + +import ( + "fmt" + + cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" + "github.com/ghodss/yaml" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func dataSourceStepTypes() *schema.Resource { + return &schema.Resource{ + Read: dataSourceStepTypesRead, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + }, + "step_types_yaml": { + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func dataSourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { + + client := meta.(*cfClient.Client) + var stepTypes *cfClient.StepTypes + var err error + + if name, nameOk := d.GetOk("name"); nameOk { + stepTypes, err = client.GetStepTypes(name.(string)) + } else { + return fmt.Errorf("data.codefresh_step_types - must specify name") + } + if err != nil { + return err + } + + if stepTypes == nil { + return fmt.Errorf("data.codefresh_step_types - cannot find step-types") + } + + return mapDataSetTypesToResource(stepTypes, d) +} + +func mapDataSetTypesToResource(stepTypes *cfClient.StepTypes, d *schema.ResourceData) error { + + if stepTypes == nil || stepTypes.Metadata["name"].(string) == "" { + return fmt.Errorf("data.codefresh_step_types - failed to mapDataSetTypesToResource") + } + d.SetId(stepTypes.Metadata["name"].(string)) + + d.Set("name", d.Id()) + + stepTypesYaml, err := yaml.Marshal(stepTypes) + if err != nil { + return err + } + d.Set("step_types_yaml", string(stepTypesYaml)) + + return nil +} diff --git a/codefresh/data_step_types_versions.go b/codefresh/data_step_types_versions.go new file mode 100644 index 0000000..334517a --- /dev/null +++ b/codefresh/data_step_types_versions.go @@ -0,0 +1,45 @@ +package codefresh + +import ( + "fmt" + + cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func dataSourceStepTypesVersions() *schema.Resource { + return &schema.Resource{ + Read: dataSourceStepTypesVersionsRead, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + }, + "versions": { + Type: schema.TypeList, + Computed: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + } +} + +func dataSourceStepTypesVersionsRead(d *schema.ResourceData, meta interface{}) error { + + client := meta.(*cfClient.Client) + var versions []string + var err error + + if name, nameOk := d.GetOk("name"); nameOk { + if versions, err = client.GetStepTypesVersions(name.(string)); err == nil { + d.SetId(name.(string)) + d.Set("name", name.(string)) + d.Set("versions", versions) + } + return err + } + return fmt.Errorf("data.codefresh_step_types_versions - must specify name") + +} diff --git a/codefresh/provider.go b/codefresh/provider.go index d34b124..4433018 100644 --- a/codefresh/provider.go +++ b/codefresh/provider.go @@ -27,25 +27,28 @@ func Provider() *schema.Provider { }, }, DataSourcesMap: map[string]*schema.Resource{ - "codefresh_users": dataSourceUsers(), - "codefresh_user": dataSourceUser(), - "codefresh_idps": dataSourceIdps(), - "codefresh_account": dataSourceAccount(), - "codefresh_team": dataSourceTeam(), - "codefresh_current_account": dataSourceCurrentAccount(), - "codefresh_context": dataSourceContext(), + "codefresh_account": dataSourceAccount(), + "codefresh_context": dataSourceContext(), + "codefresh_current_account": dataSourceCurrentAccount(), + "codefresh_idps": dataSourceIdps(), + "codefresh_step_types": dataSourceStepTypes(), + "codefresh_step_types_versions": dataSourceStepTypesVersions(), + "codefresh_team": dataSourceTeam(), + "codefresh_user": dataSourceUser(), + "codefresh_users": dataSourceUsers(), }, ResourcesMap: map[string]*schema.Resource{ - "codefresh_project": resourceProject(), - "codefresh_pipeline": resourcePipeline(), - "codefresh_context": resourceContext(), - "codefresh_team": resourceTeam(), "codefresh_account": resourceAccount(), + "codefresh_account_admins": resourceAccountAdmins(), "codefresh_api_key": resourceApiKey(), + "codefresh_context": resourceContext(), "codefresh_idp_accounts": resourceIDPAccounts(), - "codefresh_account_admins": resourceAccountAdmins(), - "codefresh_user": resourceUser(), "codefresh_permission": resourcePermission(), + "codefresh_pipeline": resourcePipeline(), + "codefresh_project": resourceProject(), + "codefresh_step_types": resourceStepTypes(), + "codefresh_user": resourceUser(), + "codefresh_team": resourceTeam(), }, ConfigureFunc: configureProvider, } diff --git a/codefresh/resource_step_types.go b/codefresh/resource_step_types.go new file mode 100644 index 0000000..4896830 --- /dev/null +++ b/codefresh/resource_step_types.go @@ -0,0 +1,145 @@ +package codefresh + +import ( + "log" + + cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" + "github.com/ghodss/yaml" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func resourceStepTypes() *schema.Resource { + return &schema.Resource{ + Create: resourceStepTypesCreate, + Read: resourceStepTypesRead, + Update: resourceStepTypesUpdate, + Delete: resourceStepTypesDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + Schema: map[string]*schema.Schema{ + "step_types_yaml": { + Type: schema.TypeString, + Required: true, + ValidateFunc: stringIsYaml, + DiffSuppressFunc: suppressEquivalentYamlDiffs, + StateFunc: func(v interface{}) string { + template, _ := normalizeYamlString(v) + return template + }, + }, + }, + } +} + +func resourceStepTypesCreate(d *schema.ResourceData, meta interface{}) error { + + client := meta.(*cfClient.Client) + stepTypes := *mapResourceToStepTypes(d) + resp, err := client.CreateStepTypes(&stepTypes) + if err != nil { + log.Printf("[DEBUG] Error while creating step types. Error = %v", err) + return err + } + + d.SetId(resp.GetID()) + return resourceStepTypesRead(d, meta) +} + +func resourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { + client := meta.(*cfClient.Client) + + stepTypesIdentifier := d.Id() + + if stepTypesIdentifier == "" { + d.SetId("") + return nil + } + + stepTypes, err := client.GetStepTypes(stepTypesIdentifier) + // Remove transient attributes from metadata + for _, attribute := range []string{"created_at", "accountId", "id", "updated_at", "latest"} { + if _, ok := stepTypes.Metadata[attribute]; ok { + delete(stepTypes.Metadata, attribute) + } + } + if err != nil { + log.Printf("[DEBUG] Error while getting stepTypes. Error = %v", stepTypesIdentifier) + return err + } + + err = mapStepTypesToResource(*stepTypes, d) + if err != nil { + log.Printf("[DEBUG] Error while mapping stepTypes to resource. Error = %v", err) + return err + } + + return nil +} + +func resourceStepTypesUpdate(d *schema.ResourceData, meta interface{}) error { + + client := meta.(*cfClient.Client) + + stepTypes := *mapResourceToStepTypes(d) + newVersion := stepTypes.Metadata["version"].(string) + existingVersions, err := client.GetStepTypesVersions(stepTypes.Metadata["name"].(string)) + if err == nil { + for _, version := range existingVersions { + if version == newVersion { + log.Printf("[DEBUG] Version %s already exists. Updating...", newVersion) + _, err := client.UpdateStepTypes(&stepTypes) + if err != nil { + log.Printf("[DEBUG] Error while updating stepTypes. Error = %v", err) + return err + } + return resourceStepTypesRead(d, meta) + } + } + } + log.Printf("[DEBUG] Creating new version %s", newVersion) + _, err = client.CreateStepTypes(&stepTypes) + if err != nil { + log.Printf("[DEBUG] Error while Creating stepTypes. Error = %v", err) + return err + } + + return resourceStepTypesRead(d, meta) +} + +func resourceStepTypesDelete(d *schema.ResourceData, meta interface{}) error { + + client := meta.(*cfClient.Client) + + err := client.DeleteStepTypes(d.Id()) + if err != nil { + return err + } + + return nil +} + +func mapStepTypesToResource(stepTypes cfClient.StepTypes, d *schema.ResourceData) error { + + stepTypesYaml, err := yaml.Marshal(stepTypes) + if err != nil { + log.Printf("[DEBUG] Failed to Marshal Step Types yaml = %v", stepTypes) + return err + } + err = d.Set("step_types_yaml", string(stepTypesYaml)) + + if err != nil { + return err + } + + return nil +} + +func mapResourceToStepTypes(d *schema.ResourceData) *cfClient.StepTypes { + + var stepTypes cfClient.StepTypes + stepTypesYaml := d.Get("step_types_yaml") + yaml.Unmarshal([]byte(stepTypesYaml.(string)), &stepTypes) + + return &stepTypes +} diff --git a/docs/data/step-types-versions.md b/docs/data/step-types-versions.md new file mode 100644 index 0000000..a8ee0ca --- /dev/null +++ b/docs/data/step-types-versions.md @@ -0,0 +1,23 @@ +# Data Source: codefresh_step_types_versions +This data source allows to retrieve the latest published version of a step-types + +## Example Usage + +```hcl +data "codefresh_step_types_versions" "freestyle" { + name = "freestyle" +} + +output "versions" { + value = data.codefresh_step_types_versions.freestyle.versions +} + +``` + +## Argument Reference + +* `name` - (Required) Name of the step-types to be retrieved + +## Attributes Reference + +* `versions` - List of versions available for the custom plugin (step-types). diff --git a/docs/data/step-types.md b/docs/data/step-types.md new file mode 100644 index 0000000..96dec78 --- /dev/null +++ b/docs/data/step-types.md @@ -0,0 +1,24 @@ +# Data Source: codefresh_step_types +This data source allows to retrieve the latest published version of a step-types + +## Example Usage + +```hcl +data "codefresh_step_types" "freestyle" { + name = "freestyle" +} + +output "test" { + # Value is return as YAML + value = yamldecode(data.codefresh_step_types.freestyle.step_types_yaml).metadata.updated_at +} + +``` + +## Argument Reference + +* `name` - (Required) Name of the step-types to be retrieved + +## Attributes Reference + +* `step_types_yaml` - The yaml string representing the custom plugin (step-types). diff --git a/docs/resources/step-types.md b/docs/resources/step-types.md new file mode 100644 index 0000000..aec1804 --- /dev/null +++ b/docs/resources/step-types.md @@ -0,0 +1,47 @@ +# Step-type Resource + +The Step-type resource allows to create your own typed step. +More about custom steps in the [official documentation](https://codefresh.io/docs/docs/codefresh-yaml/steps/#creating-a-typed-codefresh-plugin). + +## Known limitations and disclaimers +### Differences during plan phase +When executing `terraform plan` the diff presented will be the comparison between the latest published version and the version configured in the `step_types_yaml`. +At this stage the Read function doesn't have the reference to the new version in order to be able to retrieve the exact version for comparison. + +### Deletion of resource +When executing `terraform destroy` the step-stype is completely removed (including all the existing version) + +## Example Usage + +```hcl +resource "codefresh_step_types" "custom_step" { + + # NOTE: you can also load the yaml from a file with `step_types_yaml = file("PATH-TO-FILE.yaml")` + # Example has been cut down for simplicity. Yaml schema must be compliant with the what specified in the documentation for typed plugins + step_types_yaml = </custom-step + ... +spec: + arguments: |- + { + .... + } +delimiters: + left: '[[' + right: ']]' + stepsTemplate: |- + print_info_message: + name: Test step + ... +YAML +} +``` + +## Argument Reference + +- `step_types_yaml` (Required) YAML String containing a valid definition of a typed plugin + + From 6e81ef2f5f90cc034ac62b96cf5732b3b2966088 Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 2 Feb 2021 21:49:08 +1100 Subject: [PATCH 2/6] Support for step-type with versions --- client/step_types.go | 13 +- codefresh/data_step_types.go | 13 +- codefresh/provider.go | 23 +- codefresh/resource_step_types.go | 7 +- codefresh/resource_step_types_versions.go | 365 ++++++++++++++++++++++ docs/data/step-types.md | 3 +- docs/resources/step-types-versions.md | 50 +++ go.mod | 2 + go.sum | 3 + 9 files changed, 460 insertions(+), 19 deletions(-) create mode 100644 codefresh/resource_step_types_versions.go create mode 100644 docs/resources/step-types-versions.md diff --git a/client/step_types.go b/client/step_types.go index bf07eff..3c77acd 100644 --- a/client/step_types.go +++ b/client/step_types.go @@ -6,6 +6,15 @@ import ( "net/url" ) +type StepTypesVersions struct { + Name string + Versions []StepTypesVersion +} +type StepTypesVersion struct { + VersionNumber string + StepTypes StepTypes +} + type StepTypes struct { Version string `json:"version,omitempty"` Kind string `json:"kind,omitempty"` @@ -76,7 +85,7 @@ func (client *Client) CreateStepTypes(stepTypes *StepTypes) (*StepTypes, error) if err != nil { return nil, err } - log.Printf("[DEBUG] Response step types: %q", resp) + var respStepTypes StepTypes err = DecodeResponseInto(resp, &respStepTypes) if err != nil { @@ -96,7 +105,7 @@ func (client *Client) UpdateStepTypes(stepTypes *StepTypes) (*StepTypes, error) return nil, err } - fullPath := fmt.Sprintf("/step-types/%s", url.PathEscape(stepTypes.GetID()+":"+stepTypes.Metadata["version"].(string))) + fullPath := fmt.Sprintf("/step-types/%s", url.PathEscape(stepTypes.Metadata["name"].(string)+":"+stepTypes.Metadata["version"].(string))) opts := RequestOptions{ Path: fullPath, Method: "PUT", diff --git a/codefresh/data_step_types.go b/codefresh/data_step_types.go index 98635eb..3a10020 100644 --- a/codefresh/data_step_types.go +++ b/codefresh/data_step_types.go @@ -16,6 +16,10 @@ func dataSourceStepTypes() *schema.Resource { Type: schema.TypeString, Required: true, }, + "version": { + Type: schema.TypeString, + Optional: true, + }, "step_types_yaml": { Type: schema.TypeString, Computed: true, @@ -29,12 +33,13 @@ func dataSourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { client := meta.(*cfClient.Client) var stepTypes *cfClient.StepTypes var err error + identifier := d.Get("name").(string) + version, versionOk := d.GetOk("version") - if name, nameOk := d.GetOk("name"); nameOk { - stepTypes, err = client.GetStepTypes(name.(string)) - } else { - return fmt.Errorf("data.codefresh_step_types - must specify name") + if versionOk { + identifier = identifier + ":" + version.(string) } + stepTypes, err = client.GetStepTypes(identifier) if err != nil { return err } diff --git a/codefresh/provider.go b/codefresh/provider.go index 4433018..11a2405 100644 --- a/codefresh/provider.go +++ b/codefresh/provider.go @@ -38,17 +38,18 @@ func Provider() *schema.Provider { "codefresh_users": dataSourceUsers(), }, ResourcesMap: map[string]*schema.Resource{ - "codefresh_account": resourceAccount(), - "codefresh_account_admins": resourceAccountAdmins(), - "codefresh_api_key": resourceApiKey(), - "codefresh_context": resourceContext(), - "codefresh_idp_accounts": resourceIDPAccounts(), - "codefresh_permission": resourcePermission(), - "codefresh_pipeline": resourcePipeline(), - "codefresh_project": resourceProject(), - "codefresh_step_types": resourceStepTypes(), - "codefresh_user": resourceUser(), - "codefresh_team": resourceTeam(), + "codefresh_account": resourceAccount(), + "codefresh_account_admins": resourceAccountAdmins(), + "codefresh_api_key": resourceApiKey(), + "codefresh_context": resourceContext(), + "codefresh_idp_accounts": resourceIDPAccounts(), + "codefresh_permission": resourcePermission(), + "codefresh_pipeline": resourcePipeline(), + "codefresh_project": resourceProject(), + "codefresh_step_types": resourceStepTypes(), + "codefresh_step_types_versions": resourceStepTypesVersions(), + "codefresh_user": resourceUser(), + "codefresh_team": resourceTeam(), }, ConfigureFunc: configureProvider, } diff --git a/codefresh/resource_step_types.go b/codefresh/resource_step_types.go index 4896830..807bf38 100644 --- a/codefresh/resource_step_types.go +++ b/codefresh/resource_step_types.go @@ -55,8 +55,12 @@ func resourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { d.SetId("") return nil } + var stepTypesGetVersion cfClient.StepTypes + stepTypesYaml := d.Get("step_types_yaml") + yaml.Unmarshal([]byte(stepTypesYaml.(string)), &stepTypesGetVersion) + version := stepTypesGetVersion.Metadata["version"].(string) - stepTypes, err := client.GetStepTypes(stepTypesIdentifier) + stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) // Remove transient attributes from metadata for _, attribute := range []string{"created_at", "accountId", "id", "updated_at", "latest"} { if _, ok := stepTypes.Metadata[attribute]; ok { @@ -122,6 +126,7 @@ func resourceStepTypesDelete(d *schema.ResourceData, meta interface{}) error { func mapStepTypesToResource(stepTypes cfClient.StepTypes, d *schema.ResourceData) error { stepTypesYaml, err := yaml.Marshal(stepTypes) + log.Printf("[DEBUG] Marshalled Step Types yaml = %v", string(stepTypesYaml)) if err != nil { log.Printf("[DEBUG] Failed to Marshal Step Types yaml = %v", stepTypes) return err diff --git a/codefresh/resource_step_types_versions.go b/codefresh/resource_step_types_versions.go new file mode 100644 index 0000000..d544cc4 --- /dev/null +++ b/codefresh/resource_step_types_versions.go @@ -0,0 +1,365 @@ +package codefresh + +import ( + "bytes" + "context" + "fmt" + "log" + "sort" + + "github.com/Masterminds/semver" + cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" + "github.com/ghodss/yaml" + "github.com/hashicorp/terraform-plugin-sdk/helper/hashcode" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func resourceStepTypesVersions() *schema.Resource { + return &schema.Resource{ + CreateContext: resourceStepTypesVersionCreate, + ReadContext: resourceStepTypesVersionRead, + UpdateContext: resourceStepTypesVersionUpdate, + DeleteContext: resourceStepTypesVersionDelete, + Importer: &schema.ResourceImporter{ + State: schema.ImportStatePassthrough, + }, + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + ForceNew: true, + Required: true, + }, + "version": { + Type: schema.TypeSet, + Required: true, + MinItems: 1, + Set: resourceStepTypesVersionsConfigHash, + ConfigMode: schema.SchemaConfigModeAttr, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "version_number": { + Type: schema.TypeString, + Required: true, + }, + "step_types_yaml": { + Type: schema.TypeString, + Required: true, + ValidateFunc: stringIsYaml, + DiffSuppressFunc: suppressEquivalentYamlDiffs, + StateFunc: func(v interface{}) string { + template, _ := normalizeYamlStringStepTypes(v) + return template + }, + }, + }, + }, + }, + }, + } +} + +func normalizeYamlStringStepTypes(yamlString interface{}) (string, error) { + var j map[string]interface{} + + if yamlString == nil || yamlString.(string) == "" { + return "", nil + } + + s := yamlString.(string) + err := yaml.Unmarshal([]byte(s), &j) + metadataMap := j["metadata"].(map[string]interface{}) + //Removing "latest" attribute from metadata since it's transient based on the version + delete(metadataMap, "latest") + if err != nil { + return s, err + } + + bytes, _ := yaml.Marshal(j) + return string(bytes[:]), nil +} + +func resourceStepTypesVersionCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + + client := meta.(*cfClient.Client) + stepTypes := *mapResourceToStepTypesVersions(d) + + name := d.Get("name").(string) + d.SetId(name) + + // Extract all the versions so that we can order the set based on semantic versioning + mapVersion := make(map[string]cfClient.StepTypes) + var versions []string + for _, version := range stepTypes.Versions { + version.StepTypes.Metadata["name"] = name + version.StepTypes.Metadata["version"] = version.VersionNumber + log.Printf("[DEBUG] Length: %q, %v", versions, len(stepTypes.Versions)) + versions = append(versions, version.VersionNumber) + + mapVersion[version.VersionNumber] = version.StepTypes + + } + + // Create the versions in order based on semver + orderedVersions := sortVersions(versions) + for _, version := range orderedVersions { + step := mapVersion[version.String()] + log.Printf("[DEBUG] Version for create: %q", version) + _, err := client.CreateStepTypes(&step) + if err != nil { + return diag.Errorf("[DEBUG] Error while creating step types OnCreate. Error = %v", err) + } + } + + return resourceStepTypesVersionRead(ctx, d, meta) +} + +func resourceStepTypesVersionRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + client := meta.(*cfClient.Client) + + stepTypesIdentifier := d.Id() + if stepTypesIdentifier == "" { + + d.SetId("") + return nil + } + + //Extracting the step just based on the name to validate it exists + stepTypes, err := client.GetStepTypes(stepTypesIdentifier) + if err != nil { + log.Printf("[DEBUG] Step Not found %v. Error = %v", stepTypesIdentifier, err) + d.SetId("") + return nil + } + + var stepVersions cfClient.StepTypesVersions + name := stepTypes.Metadata["name"].(string) + stepVersions.Name = name + versions := d.Get("version").(*schema.Set) + // Try to retrieve defined versions and add to the list if it exists + for _, step := range versions.List() { + version := step.(map[string]interface{})["version_number"].(string) + log.Printf("[DEBUG] Get step version FromList %v", version) + if version != "" { + stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) + log.Printf("[DEBUG] Get step version %v", version) + if err != nil { + log.Printf("[DEBUG] StepVersion not found %v. Error = %v", stepTypesIdentifier+":"+version, err) + } else { + cleanUpStepFromTransientValues(stepTypes, name, version) + stepVersion := cfClient.StepTypesVersion{ + VersionNumber: version, + StepTypes: *stepTypes, + } + stepVersions.Versions = append(stepVersions.Versions, stepVersion) + + } + } + } + + err = mapStepTypesVersionsToResource(stepVersions, d) + + if err != nil { + return diag.Errorf("[DEBUG] Error while mapping stepTypes to resource for READ. Error = %v", err) + } + + return nil +} + +func resourceStepTypesVersionUpdate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + + client := meta.(*cfClient.Client) + name := d.Get("name").(string) + stepTypesVersions := mapResourceToStepTypesVersions(d) + mapVersionToCreate := make(map[string]cfClient.StepTypes) + versionsPreviouslyDefined := make(map[string]string) + versionsDefined := make(map[string]string) + // Name is set to ForceNew so if we reach this function "version" is changed. Skipping check on HasChange + // Retrieving old version of the resource to enable comparsion with new and determine which versions should be removed + old, _ := d.GetChange("version") + + for _, oldStep := range old.(*schema.Set).List() { + oldVersion := oldStep.(map[string]interface{})["version_number"].(string) + versionsPreviouslyDefined[oldVersion] = oldVersion + } + + // Parse current set: new versions that need to be created are added to a data structure + // that will be sorted later for the creation + // Updates are performed immediately + for _, version := range stepTypesVersions.Versions { + versionNumber := version.VersionNumber + versionsDefined[versionNumber] = versionNumber + + _, err := client.GetStepTypes(name + ":" + versionNumber) + cleanUpStepFromTransientValues(&version.StepTypes, name, versionNumber) + if err != nil { + // If an error occured during Get, we assume step doesn't exist + log.Printf("[DEBUG] Recording for creation: %q", versionNumber) + mapVersionToCreate[versionNumber] = version.StepTypes + } else { + log.Printf("[DEBUG] Update Version step: %q", versionNumber) + _, err := client.UpdateStepTypes(&version.StepTypes) + if err != nil { + return diag.Errorf("[DEBUG] Error while updating stepTypes. Error = %v", err) + + } + } + } + + // Order versions for creation + createVersions := make([]string, len(mapVersionToCreate)) + i := 0 + for k := range mapVersionToCreate { + createVersions[i] = k + i++ + } + orderedVersions := sortVersions(createVersions) + for _, version := range orderedVersions { + step := mapVersionToCreate[version.String()] + log.Printf("[DEBUG] Creating version %s for step types: %s", step.Metadata["version"], step.Metadata["name"]) + _, err := client.CreateStepTypes(&step) + if err != nil { + return diag.Errorf("[DEBUG] Error while creating step types OnUpdate function. Error = %v", err) + } + } + + // If a version is not listed in versionsDefined we can remove it from the system + for version := range versionsPreviouslyDefined { + if _, ok := versionsDefined[version]; !ok { + log.Printf("[DEBUG] Deleting version: %s", version) + // If not defined we remove from the system + err := client.DeleteStepTypes(d.Id() + ":" + version) + if err != nil { + return diag.Errorf("[DEBUG] Error while deleting step_types_versions. Error = %v", err) + } + } + } + + return resourceStepTypesVersionRead(ctx, d, meta) +} + +func resourceStepTypesVersionDelete(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + + client := meta.(*cfClient.Client) + log.Printf("[DEBUG] Deleting step type: %s", d.Id()) + err := client.DeleteStepTypes(d.Id()) + if err != nil { + return diag.Errorf("[DEBUG] Error while deleting step_types %s. Error = %v", d.Id(), err) + } + + return nil +} + +func cleanUpStepFromTransientValues(stepTypes *cfClient.StepTypes, name, version string) { + if stepTypes != nil { + // Remove transient attributes from metadata + for _, attribute := range []string{"created_at", "accountId", "id", "updated_at"} { + if _, ok := stepTypes.Metadata[attribute]; ok { + delete(stepTypes.Metadata, attribute) + } + } + // Forcing latest to false + // This is needed because in some cases (e.g. adding an old version) the latest attribute is set to `null` by Codefresh + // Having `null` as value causes subsequent calls to fail validation against this attribute + stepTypes.Metadata["latest"] = false + + // If name of version are empty strings we remove them from the data structure + // The use case is for the calculation of the Hash of the Set item, where we don't have access to this information. + // Since that is coming from the other attribute of the resource there's no point to actually consider it for hashing + if name != "" { + stepTypes.Metadata["name"] = name + } else { + delete(stepTypes.Metadata, "name") + } + if version != "" { + stepTypes.Metadata["version"] = version + } else { + delete(stepTypes.Metadata, "version") + } + + } + +} + +func sortVersions(versions []string) []*semver.Version { + log.Printf("[DEBUG] Sorting: %q", versions) + var vs []*semver.Version + for _, version := range versions { + v, err := semver.NewVersion(version) + if err != nil { + diag.Errorf("Error parsing version: %s", err) + } + vs = append(vs, v) + } + + sort.Sort(semver.Collection(vs)) + return vs +} + +func mapStepTypesVersionsToResource(stepTypesVersions cfClient.StepTypesVersions, d *schema.ResourceData) error { + + err := d.Set("name", stepTypesVersions.Name) + if err != nil { + return err + } + + err = d.Set("version", flattenVersions(stepTypesVersions.Name, stepTypesVersions.Versions)) + return err + +} + +func resourceStepTypesVersionsConfigHash(v interface{}) int { + + var buf bytes.Buffer + m := v.(map[string]interface{}) + + buf.WriteString(fmt.Sprintf("%s", m["version_number"].(string))) + var stepTypes cfClient.StepTypes + stepTypesYaml := m["step_types_yaml"].(string) + yaml.Unmarshal([]byte(stepTypesYaml), &stepTypes) + // Remove runtime attributes, name and version to avoid discrepancies when comparing hashes + cleanUpStepFromTransientValues(&stepTypes, "", "") + stepTypesYamlByteArray, _ := yaml.Marshal(stepTypes) + buf.WriteString(fmt.Sprintf("%s", string(stepTypesYamlByteArray))) + hash := hashcode.String(buf.String()) + return hash +} + +func flattenVersions(name string, versions []cfClient.StepTypesVersion) *schema.Set { + + stepVersions := make([]interface{}, 0) + for _, version := range versions { + m := make(map[string]interface{}) + m["version_number"] = version.VersionNumber + cleanUpStepFromTransientValues(&version.StepTypes, name, version.VersionNumber) + stepTypesYaml, _ := yaml.Marshal(version.StepTypes) + m["step_types_yaml"] = string(stepTypesYaml) + stepVersions = append(stepVersions, m) + } + + return schema.NewSet(resourceStepTypesVersionsConfigHash, stepVersions) +} + +func mapResourceToStepTypesVersions(d *schema.ResourceData) *cfClient.StepTypesVersions { + var stepTypesVersions cfClient.StepTypesVersions + stepTypesVersions.Name = d.Get("name").(string) + versions := d.Get("version").(*schema.Set) + + for _, step := range versions.List() { + version := step.(map[string]interface{})["version_number"].(string) + if version != "" { + var stepTypes cfClient.StepTypes + stepTypesYaml := step.(map[string]interface{})["step_types_yaml"].(string) + yaml.Unmarshal([]byte(stepTypesYaml), &stepTypes) + cleanUpStepFromTransientValues(&stepTypes, stepTypesVersions.Name, version) + stepVersion := cfClient.StepTypesVersion{ + VersionNumber: version, + StepTypes: stepTypes, + } + + stepTypesVersions.Versions = append(stepTypesVersions.Versions, stepVersion) + } + } + + return &stepTypesVersions +} diff --git a/docs/data/step-types.md b/docs/data/step-types.md index 96dec78..f14dae9 100644 --- a/docs/data/step-types.md +++ b/docs/data/step-types.md @@ -9,7 +9,7 @@ data "codefresh_step_types" "freestyle" { } output "test" { - # Value is return as YAML + # Value is return as YAML value = yamldecode(data.codefresh_step_types.freestyle.step_types_yaml).metadata.updated_at } @@ -18,6 +18,7 @@ output "test" { ## Argument Reference * `name` - (Required) Name of the step-types to be retrieved +* `version` - (Optional) Version to be retrieved. If not specified, the latest published will be returned ## Attributes Reference diff --git a/docs/resources/step-types-versions.md b/docs/resources/step-types-versions.md new file mode 100644 index 0000000..4ffdd25 --- /dev/null +++ b/docs/resources/step-types-versions.md @@ -0,0 +1,50 @@ +# Step-type-versions Resource + +The Step-type-version resource allows to create your own typed step and manage all it's published versions. +The resource allows to handle the life-cycle of the version by allowing specifying multiple blocks `version` where the user provides a version number and the yaml file representing the plugin. +More about custom steps in the [official documentation](https://codefresh.io/docs/docs/codefresh-yaml/steps/#creating-a-typed-codefresh-plugin). + +## Known limitations and disclaimers +### Version and name in yaml Metadata are ignored. +The version and name of the step declared in the yaml files are superseeded by the attributes specified at resource level: +- `name` : at top level +- `version_numer`: specified in the `version` block +The above are added/replaced at runtime time. + +### Number of API requests +This resource makes a lot of additional API calls to validate the steps and retrieve all the version available. +Caution is recommended on the amount of versions maintained and the number of resources defined in a single project. + + +## Example Usage + +```hcl + +data "codefresh_current_account" "acc" { +} + +resource "codefresh_step_types_versions" "my-custom-step" { + name = "${data.codefresh_current_account.acc.name}/my-custom-step" + + version { + version_number = "0.0.1" + step_types_yaml = file("./templates/plugin-0.0.1.yaml") + } + version { + version_number = "0.0.2" + step_types_yaml = file("./templates/plugin-0.0.2.yaml") + } + .... +} +} +``` + +## Argument Reference +- `name` - (Required) The name for the step-type +- `version` - (At least 1 Required) A collection of `version` blocks as documented below. + +--- + +`version` supports the following: +- `version_number` - (Required) String representing the semVer for the step +- `step_types_yaml` (Required) YAML String containing a valid definition of a typed plugin diff --git a/go.mod b/go.mod index a43d9fd..9d1229a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/codefresh-io/terraform-provider-codefresh require ( + github.com/Masterminds/semver v1.5.0 github.com/aws/aws-sdk-go v1.30.12 // indirect github.com/bflad/tfproviderdocs v0.6.0 github.com/bflad/tfproviderlint v0.14.0 @@ -8,6 +9,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/golangci/golangci-lint v1.27.0 github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7 // indirect + github.com/hashicorp/terraform-plugin-sdk v1.7.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1 github.com/imdario/mergo v0.3.9 github.com/stretchr/objx v0.1.1 diff --git a/go.sum b/go.sum index 43c1753..7d9ff22 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157 h1:hY39LwQHh+1kaovmIjOrlqnXNX6tygSRfLkkK33IkZU= github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= @@ -243,6 +245,7 @@ github.com/hashicorp/terraform-json v0.5.0 h1:7TV3/F3y7QVSuN4r9BEXqnWqrAyeOtON8f github.com/hashicorp/terraform-json v0.5.0/go.mod h1:eAbqb4w0pSlRmdvl8fOyHAi/+8jnkVYN28gJkSJrLhU= github.com/hashicorp/terraform-plugin-sdk v1.7.0 h1:B//oq0ZORG+EkVrIJy0uPGSonvmXqxSzXe8+GhknoW0= github.com/hashicorp/terraform-plugin-sdk v1.7.0/go.mod h1:OjgQmey5VxnPej/buEhe+YqKm0KNvV3QqU4hkqHqPCY= +github.com/hashicorp/terraform-plugin-sdk v1.16.0 h1:NrkXMRjHErUPPTHQkZ6JIn6bByiJzGnlJzH1rVdNEuE= github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1 h1:h8TtYDlIACXQ6LNJesvSHuxskaPUAX/nltvqp0Kp1vk= github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1/go.mod h1:aWg/hVISyjdpUOt89SSrplxffuq8HPEnIWGyKDpDzCo= github.com/hashicorp/terraform-plugin-test v1.2.0/go.mod h1:QIJHYz8j+xJtdtLrFTlzQVC0ocr3rf/OjIpgZLK56Hs= From 6a0c6f56a8fdb247ac3806aec83365f6615dc89a Mon Sep 17 00:00:00 2001 From: Sandro Gattuso Date: Sun, 14 Feb 2021 14:01:17 +1100 Subject: [PATCH 3/6] Bugfix to preserve order of steps attribute in step-types --- client/step_types.go | 15 +++- codefresh/resource_step_types.go | 5 +- codefresh/resource_step_types_versions.go | 85 ++++++++++++++++++++--- go.mod | 1 + go.sum | 2 + 5 files changed, 95 insertions(+), 13 deletions(-) diff --git a/client/step_types.go b/client/step_types.go index 3c77acd..970bbe5 100644 --- a/client/step_types.go +++ b/client/step_types.go @@ -4,6 +4,8 @@ import ( "fmt" "log" "net/url" + + "github.com/iancoleman/orderedmap" ) type StepTypesVersions struct { @@ -19,7 +21,15 @@ type StepTypes struct { Version string `json:"version,omitempty"` Kind string `json:"kind,omitempty"` Metadata map[string]interface{} `json:"metadata,omitempty"` - Spec map[string]interface{} `json:"spec,omitempty"` + Spec SpecStepTypes `json:"spec,omitempty"` +} + +type SpecStepTypes struct { + Arguments string `json:"arguments,omitempty"` + Delimiters map[string]interface{} `json:"delimiters,omitempty"` + Returns string `json:"returns,omitempty"` + Steps *orderedmap.OrderedMap `json:"steps,omitempty"` + StepsTemplate string `json:"stepsTemplate,omitempty"` } func (stepTypes *StepTypes) GetID() string { @@ -89,10 +99,9 @@ func (client *Client) CreateStepTypes(stepTypes *StepTypes) (*StepTypes, error) var respStepTypes StepTypes err = DecodeResponseInto(resp, &respStepTypes) if err != nil { - log.Printf("[DEBUG] Error while decoding step types. Error = %v, Response: %q", err, respStepTypes) + log.Printf("[DEBUG] Error while decoding step types. Error = %v, Response: %v", err, respStepTypes) return nil, err } - log.Printf("[DEBUG] Decoded step types response: %q", respStepTypes.Metadata["name"]) return &respStepTypes, nil } diff --git a/codefresh/resource_step_types.go b/codefresh/resource_step_types.go index 807bf38..d45bb98 100644 --- a/codefresh/resource_step_types.go +++ b/codefresh/resource_step_types.go @@ -38,7 +38,7 @@ func resourceStepTypesCreate(d *schema.ResourceData, meta interface{}) error { stepTypes := *mapResourceToStepTypes(d) resp, err := client.CreateStepTypes(&stepTypes) if err != nil { - log.Printf("[DEBUG] Error while creating step types. Error = %v", err) + log.Printf("[DEBUG] Error while creating step types for resource_step_types. Error = %v", err) return err } @@ -145,6 +145,9 @@ func mapResourceToStepTypes(d *schema.ResourceData) *cfClient.StepTypes { var stepTypes cfClient.StepTypes stepTypesYaml := d.Get("step_types_yaml") yaml.Unmarshal([]byte(stepTypesYaml.(string)), &stepTypes) + if stepTypes.Spec.Steps != nil { + stepTypes.Spec.Steps = extractSteps(stepTypesYaml.(string)) + } return &stepTypes } diff --git a/codefresh/resource_step_types_versions.go b/codefresh/resource_step_types_versions.go index d544cc4..d9be2e5 100644 --- a/codefresh/resource_step_types_versions.go +++ b/codefresh/resource_step_types_versions.go @@ -3,16 +3,20 @@ package codefresh import ( "bytes" "context" + "encoding/json" "fmt" "log" "sort" + "strings" "github.com/Masterminds/semver" cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" - "github.com/ghodss/yaml" + ghodss "github.com/ghodss/yaml" "github.com/hashicorp/terraform-plugin-sdk/helper/hashcode" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/iancoleman/orderedmap" + "gopkg.in/yaml.v2" ) func resourceStepTypesVersions() *schema.Resource { @@ -67,15 +71,17 @@ func normalizeYamlStringStepTypes(yamlString interface{}) (string, error) { } s := yamlString.(string) - err := yaml.Unmarshal([]byte(s), &j) + err := ghodss.Unmarshal([]byte(s), &j) metadataMap := j["metadata"].(map[string]interface{}) //Removing "latest" attribute from metadata since it's transient based on the version delete(metadataMap, "latest") + delete(metadataMap, "name") + delete(metadataMap, "version") if err != nil { return s, err } - bytes, _ := yaml.Marshal(j) + bytes, _ := ghodss.Marshal(j) return string(bytes[:]), nil } @@ -104,7 +110,7 @@ func resourceStepTypesVersionCreate(ctx context.Context, d *schema.ResourceData, orderedVersions := sortVersions(versions) for _, version := range orderedVersions { step := mapVersion[version.String()] - log.Printf("[DEBUG] Version for create: %q", version) + log.Printf("[DEBUG] Version for create: %q. StepSpec: %v", version, step.Spec.Steps) _, err := client.CreateStepTypes(&step) if err != nil { return diag.Errorf("[DEBUG] Error while creating step types OnCreate. Error = %v", err) @@ -316,10 +322,10 @@ func resourceStepTypesVersionsConfigHash(v interface{}) int { buf.WriteString(fmt.Sprintf("%s", m["version_number"].(string))) var stepTypes cfClient.StepTypes stepTypesYaml := m["step_types_yaml"].(string) - yaml.Unmarshal([]byte(stepTypesYaml), &stepTypes) + ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) // Remove runtime attributes, name and version to avoid discrepancies when comparing hashes cleanUpStepFromTransientValues(&stepTypes, "", "") - stepTypesYamlByteArray, _ := yaml.Marshal(stepTypes) + stepTypesYamlByteArray, _ := ghodss.Marshal(stepTypes) buf.WriteString(fmt.Sprintf("%s", string(stepTypesYamlByteArray))) hash := hashcode.String(buf.String()) return hash @@ -332,11 +338,15 @@ func flattenVersions(name string, versions []cfClient.StepTypesVersion) *schema. m := make(map[string]interface{}) m["version_number"] = version.VersionNumber cleanUpStepFromTransientValues(&version.StepTypes, name, version.VersionNumber) - stepTypesYaml, _ := yaml.Marshal(version.StepTypes) + stepTypesYaml, err := ghodss.Marshal(version.StepTypes) + log.Printf("[DEBUG] Flattened StepTypes %v", version.StepTypes.Spec) + if err != nil { + log.Fatalf("Error while flattening Versions: %v. Errv=%s", version.StepTypes, err) + } m["step_types_yaml"] = string(stepTypesYaml) stepVersions = append(stepVersions, m) } - + log.Printf("[DEBUG] Flattened Versions %s", stepVersions) return schema.NewSet(resourceStepTypesVersionsConfigHash, stepVersions) } @@ -350,12 +360,20 @@ func mapResourceToStepTypesVersions(d *schema.ResourceData) *cfClient.StepTypesV if version != "" { var stepTypes cfClient.StepTypes stepTypesYaml := step.(map[string]interface{})["step_types_yaml"].(string) - yaml.Unmarshal([]byte(stepTypesYaml), &stepTypes) + + err := ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) + if err != nil { + log.Fatalf("[DEBUG] Unable to mapResourceToStepTypesVersions for version %s. Err= %s", version, err) + } + cleanUpStepFromTransientValues(&stepTypes, stepTypesVersions.Name, version) stepVersion := cfClient.StepTypesVersion{ VersionNumber: version, StepTypes: stepTypes, } + if stepVersion.StepTypes.Spec.Steps != nil { + stepVersion.StepTypes.Spec.Steps = extractSteps(stepTypesYaml) + } stepTypesVersions.Versions = append(stepTypesVersions.Versions, stepVersion) } @@ -363,3 +381,52 @@ func mapResourceToStepTypesVersions(d *schema.ResourceData) *cfClient.StepTypesV return &stepTypesVersions } + +// extractStagesAndSteps extracts the steps and stages from the original yaml string to enable propagation in the `Spec` attribute of the pipeline +// We cannot leverage on the standard marshal/unmarshal because the steps attribute needs to maintain the order of elements +// while by default the standard function doesn't do it because in JSON maps are unordered +func extractSteps(stepTypesYaml string) (steps *orderedmap.OrderedMap) { + // Use mapSlice to preserve order of items from the YAML string + m := yaml.MapSlice{} + err := yaml.Unmarshal([]byte(stepTypesYaml), &m) + if err != nil { + log.Fatal("Unable to unmarshall stepTypesYaml") + } + steps = orderedmap.New() + // Dynamically build JSON object for steps using String builder + stepsBuilder := strings.Builder{} + stepsBuilder.WriteString("{") + // Parse elements of the YAML string to extract Steps and Stages if defined + for _, item := range m { + if item.Key == "spec" { + for _, specItem := range item.Value.(yaml.MapSlice) { + if specItem.Key == "steps" { + switch x := specItem.Value.(type) { + default: + log.Fatalf("unsupported value type: %T", specItem.Value) + + case yaml.MapSlice: + numberOfSteps := len(x) + for index, item := range x { + // We only need to preserve order at the first level to guarantee order of the steps, hence the child nodes can be marshalled + // with the standard library + y, _ := yaml.Marshal(item.Value) + j2, _ := ghodss.YAMLToJSON(y) + stepsBuilder.WriteString("\"" + item.Key.(string) + "\" : " + string(j2)) + if index < numberOfSteps-1 { + stepsBuilder.WriteString(",") + } + } + } + } + } + } + } + stepsBuilder.WriteString("}") + err = json.Unmarshal([]byte(stepsBuilder.String()), &steps) + if err != nil { + log.Fatalf("[DEBUG] Unable to parse steps. ") + } + + return +} diff --git a/go.mod b/go.mod index 9d1229a..d3f90f2 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/hashicorp/terraform-config-inspect v0.0.0-20191212124732-c6ae6269b9d7 // indirect github.com/hashicorp/terraform-plugin-sdk v1.7.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1 + github.com/iancoleman/orderedmap v0.2.0 github.com/imdario/mergo v0.3.9 github.com/stretchr/objx v0.1.1 github.com/stretchr/testify v1.6.1 // indirect diff --git a/go.sum b/go.sum index 7d9ff22..e4f567f 100644 --- a/go.sum +++ b/go.sum @@ -258,6 +258,8 @@ github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= +github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= From f84eceaeb66a9f54bd491d32916cc39065a8fcda Mon Sep 17 00:00:00 2001 From: Sandro Gattuso Date: Thu, 18 Feb 2021 21:48:58 +1100 Subject: [PATCH 4/6] Test cases and clean up --- codefresh/provider.go | 23 +- codefresh/resource_step_types.go | 431 ++++++++++++++--- codefresh/resource_step_types_test.go | 215 +++++++++ codefresh/resource_step_types_versions.go | 432 ------------------ go.sum | 1 - test_data/step_types/testStepTypesOrder.yaml | 6 + .../step_types/testStepWithRuntimeData.yaml | 32 ++ test_data/step_types/testSteps.yaml | 29 ++ test_data/step_types/testStepsTemplate.yaml | 32 ++ 9 files changed, 680 insertions(+), 521 deletions(-) create mode 100644 codefresh/resource_step_types_test.go delete mode 100644 codefresh/resource_step_types_versions.go create mode 100644 test_data/step_types/testStepTypesOrder.yaml create mode 100644 test_data/step_types/testStepWithRuntimeData.yaml create mode 100644 test_data/step_types/testSteps.yaml create mode 100644 test_data/step_types/testStepsTemplate.yaml diff --git a/codefresh/provider.go b/codefresh/provider.go index 11a2405..4433018 100644 --- a/codefresh/provider.go +++ b/codefresh/provider.go @@ -38,18 +38,17 @@ func Provider() *schema.Provider { "codefresh_users": dataSourceUsers(), }, ResourcesMap: map[string]*schema.Resource{ - "codefresh_account": resourceAccount(), - "codefresh_account_admins": resourceAccountAdmins(), - "codefresh_api_key": resourceApiKey(), - "codefresh_context": resourceContext(), - "codefresh_idp_accounts": resourceIDPAccounts(), - "codefresh_permission": resourcePermission(), - "codefresh_pipeline": resourcePipeline(), - "codefresh_project": resourceProject(), - "codefresh_step_types": resourceStepTypes(), - "codefresh_step_types_versions": resourceStepTypesVersions(), - "codefresh_user": resourceUser(), - "codefresh_team": resourceTeam(), + "codefresh_account": resourceAccount(), + "codefresh_account_admins": resourceAccountAdmins(), + "codefresh_api_key": resourceApiKey(), + "codefresh_context": resourceContext(), + "codefresh_idp_accounts": resourceIDPAccounts(), + "codefresh_permission": resourcePermission(), + "codefresh_pipeline": resourcePipeline(), + "codefresh_project": resourceProject(), + "codefresh_step_types": resourceStepTypes(), + "codefresh_user": resourceUser(), + "codefresh_team": resourceTeam(), }, ConfigureFunc: configureProvider, } diff --git a/codefresh/resource_step_types.go b/codefresh/resource_step_types.go index d45bb98..46ff2fc 100644 --- a/codefresh/resource_step_types.go +++ b/codefresh/resource_step_types.go @@ -1,153 +1,432 @@ package codefresh import ( + "bytes" + "context" + "encoding/json" + "fmt" "log" + "sort" + "strings" + "github.com/Masterminds/semver" cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" - "github.com/ghodss/yaml" + ghodss "github.com/ghodss/yaml" + "github.com/hashicorp/terraform-plugin-sdk/helper/hashcode" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/iancoleman/orderedmap" + "gopkg.in/yaml.v2" ) func resourceStepTypes() *schema.Resource { return &schema.Resource{ - Create: resourceStepTypesCreate, - Read: resourceStepTypesRead, - Update: resourceStepTypesUpdate, - Delete: resourceStepTypesDelete, + CreateContext: resourceStepTypesCreate, + ReadContext: resourceStepTypesRead, + UpdateContext: resourceStepTypesUpdate, + DeleteContext: resourceStepTypesDelete, Importer: &schema.ResourceImporter{ State: schema.ImportStatePassthrough, }, Schema: map[string]*schema.Schema{ - "step_types_yaml": { - Type: schema.TypeString, - Required: true, - ValidateFunc: stringIsYaml, - DiffSuppressFunc: suppressEquivalentYamlDiffs, - StateFunc: func(v interface{}) string { - template, _ := normalizeYamlString(v) - return template + "name": { + Type: schema.TypeString, + ForceNew: true, + Required: true, + }, + "version": { + Type: schema.TypeSet, + Required: true, + MinItems: 1, + Set: resourceStepTypesVersionsConfigHash, + ConfigMode: schema.SchemaConfigModeAttr, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "version_number": { + Type: schema.TypeString, + Required: true, + }, + "step_types_yaml": { + Type: schema.TypeString, + Required: true, + ValidateFunc: stringIsYaml, + DiffSuppressFunc: suppressEquivalentYamlDiffs, + StateFunc: func(v interface{}) string { + template, _ := normalizeYamlStringStepTypes(v) + return template + }, + }, + }, }, }, }, } } -func resourceStepTypesCreate(d *schema.ResourceData, meta interface{}) error { +func normalizeYamlStringStepTypes(yamlString interface{}) (string, error) { + var j map[string]interface{} - client := meta.(*cfClient.Client) - stepTypes := *mapResourceToStepTypes(d) - resp, err := client.CreateStepTypes(&stepTypes) + if yamlString == nil || yamlString.(string) == "" { + return "", nil + } + + s := yamlString.(string) + err := ghodss.Unmarshal([]byte(s), &j) + metadataMap := j["metadata"].(map[string]interface{}) + //Removing "latest" attribute from metadata since it's transient based on the version + delete(metadataMap, "latest") + delete(metadataMap, "name") + delete(metadataMap, "version") if err != nil { - log.Printf("[DEBUG] Error while creating step types for resource_step_types. Error = %v", err) - return err + return s, err } - d.SetId(resp.GetID()) - return resourceStepTypesRead(d, meta) + bytes, _ := ghodss.Marshal(j) + return string(bytes[:]), nil } -func resourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { +func resourceStepTypesCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + client := meta.(*cfClient.Client) + stepTypes := *mapResourceToStepTypesVersions(d) - stepTypesIdentifier := d.Id() + name := d.Get("name").(string) + d.SetId(name) + + // Extract all the versions so that we can order the set based on semantic versioning + mapVersion := make(map[string]cfClient.StepTypes) + var versions []string + for _, version := range stepTypes.Versions { + version.StepTypes.Metadata["name"] = name + version.StepTypes.Metadata["version"] = version.VersionNumber + log.Printf("[DEBUG] Length: %q, %v", versions, len(stepTypes.Versions)) + versions = append(versions, version.VersionNumber) + + mapVersion[version.VersionNumber] = version.StepTypes + + } + + // Create the versions in order based on semver + orderedVersions := sortVersions(versions) + for _, version := range orderedVersions { + step := mapVersion[version.String()] + log.Printf("[DEBUG] Version for create: %q. StepSpec: %v", version, step.Spec.Steps) + _, err := client.CreateStepTypes(&step) + if err != nil { + return diag.Errorf("[DEBUG] Error while creating step types OnCreate. Error = %v", err) + } + } + + return resourceStepTypesRead(ctx, d, meta) +} + +func resourceStepTypesRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + client := meta.(*cfClient.Client) + stepTypesIdentifier := d.Id() if stepTypesIdentifier == "" { + d.SetId("") return nil } - var stepTypesGetVersion cfClient.StepTypes - stepTypesYaml := d.Get("step_types_yaml") - yaml.Unmarshal([]byte(stepTypesYaml.(string)), &stepTypesGetVersion) - version := stepTypesGetVersion.Metadata["version"].(string) - stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) - // Remove transient attributes from metadata - for _, attribute := range []string{"created_at", "accountId", "id", "updated_at", "latest"} { - if _, ok := stepTypes.Metadata[attribute]; ok { - delete(stepTypes.Metadata, attribute) - } - } + //Extracting the step just based on the name to validate it exists + stepTypes, err := client.GetStepTypes(stepTypesIdentifier) if err != nil { - log.Printf("[DEBUG] Error while getting stepTypes. Error = %v", stepTypesIdentifier) - return err + log.Printf("[DEBUG] Step Not found %v. Error = %v", stepTypesIdentifier, err) + d.SetId("") + return nil + } + + var stepVersions cfClient.StepTypesVersions + name := stepTypes.Metadata["name"].(string) + stepVersions.Name = name + versions := d.Get("version").(*schema.Set) + // Try to retrieve defined versions and add to the list if it exists + for _, step := range versions.List() { + version := step.(map[string]interface{})["version_number"].(string) + log.Printf("[DEBUG] Get step version FromList %v", version) + if version != "" { + stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) + log.Printf("[DEBUG] Get step version %v", version) + if err != nil { + log.Printf("[DEBUG] StepVersion not found %v. Error = %v", stepTypesIdentifier+":"+version, err) + } else { + cleanUpStepFromTransientValues(stepTypes, name, version) + stepVersion := cfClient.StepTypesVersion{ + VersionNumber: version, + StepTypes: *stepTypes, + } + stepVersions.Versions = append(stepVersions.Versions, stepVersion) + + } + } } - err = mapStepTypesToResource(*stepTypes, d) + err = mapStepTypesVersionsToResource(stepVersions, d) + if err != nil { - log.Printf("[DEBUG] Error while mapping stepTypes to resource. Error = %v", err) - return err + return diag.Errorf("[DEBUG] Error while mapping stepTypes to resource for READ. Error = %v", err) } return nil } -func resourceStepTypesUpdate(d *schema.ResourceData, meta interface{}) error { +func resourceStepTypesUpdate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { client := meta.(*cfClient.Client) + name := d.Get("name").(string) + stepTypesVersions := mapResourceToStepTypesVersions(d) + mapVersionToCreate := make(map[string]cfClient.StepTypes) + versionsPreviouslyDefined := make(map[string]string) + versionsDefined := make(map[string]string) + // Name is set to ForceNew so if we reach this function "version" is changed. Skipping check on HasChange + // Retrieving old version of the resource to enable comparsion with new and determine which versions should be removed + old, _ := d.GetChange("version") + + for _, oldStep := range old.(*schema.Set).List() { + oldVersion := oldStep.(map[string]interface{})["version_number"].(string) + versionsPreviouslyDefined[oldVersion] = oldVersion + } + + // Parse current set: new versions that need to be created are added to a data structure + // that will be sorted later for the creation + // Updates are performed immediately + for _, version := range stepTypesVersions.Versions { + versionNumber := version.VersionNumber + versionsDefined[versionNumber] = versionNumber + + _, err := client.GetStepTypes(name + ":" + versionNumber) + cleanUpStepFromTransientValues(&version.StepTypes, name, versionNumber) + if err != nil { + // If an error occured during Get, we assume step doesn't exist + log.Printf("[DEBUG] Recording for creation: %q", versionNumber) + mapVersionToCreate[versionNumber] = version.StepTypes + } else { + log.Printf("[DEBUG] Update Version step: %q", versionNumber) + _, err := client.UpdateStepTypes(&version.StepTypes) + if err != nil { + return diag.Errorf("[DEBUG] Error while updating stepTypes. Error = %v", err) - stepTypes := *mapResourceToStepTypes(d) - newVersion := stepTypes.Metadata["version"].(string) - existingVersions, err := client.GetStepTypesVersions(stepTypes.Metadata["name"].(string)) - if err == nil { - for _, version := range existingVersions { - if version == newVersion { - log.Printf("[DEBUG] Version %s already exists. Updating...", newVersion) - _, err := client.UpdateStepTypes(&stepTypes) - if err != nil { - log.Printf("[DEBUG] Error while updating stepTypes. Error = %v", err) - return err - } - return resourceStepTypesRead(d, meta) } } } - log.Printf("[DEBUG] Creating new version %s", newVersion) - _, err = client.CreateStepTypes(&stepTypes) - if err != nil { - log.Printf("[DEBUG] Error while Creating stepTypes. Error = %v", err) - return err + + // Order versions for creation + createVersions := make([]string, len(mapVersionToCreate)) + i := 0 + for k := range mapVersionToCreate { + createVersions[i] = k + i++ + } + orderedVersions := sortVersions(createVersions) + for _, version := range orderedVersions { + step := mapVersionToCreate[version.String()] + log.Printf("[DEBUG] Creating version %s for step types: %s", step.Metadata["version"], step.Metadata["name"]) + _, err := client.CreateStepTypes(&step) + if err != nil { + return diag.Errorf("[DEBUG] Error while creating step types OnUpdate function. Error = %v", err) + } + } + + // If a version is not listed in versionsDefined we can remove it from the system + for version := range versionsPreviouslyDefined { + if _, ok := versionsDefined[version]; !ok { + log.Printf("[DEBUG] Deleting version: %s", version) + // If not defined we remove from the system + err := client.DeleteStepTypes(d.Id() + ":" + version) + if err != nil { + return diag.Errorf("[DEBUG] Error while deleting step_types_versions. Error = %v", err) + } + } } - return resourceStepTypesRead(d, meta) + return resourceStepTypesRead(ctx, d, meta) } -func resourceStepTypesDelete(d *schema.ResourceData, meta interface{}) error { +func resourceStepTypesDelete(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { client := meta.(*cfClient.Client) - + log.Printf("[DEBUG] Deleting step type: %s", d.Id()) err := client.DeleteStepTypes(d.Id()) if err != nil { - return err + return diag.Errorf("[DEBUG] Error while deleting step_types %s. Error = %v", d.Id(), err) } return nil } -func mapStepTypesToResource(stepTypes cfClient.StepTypes, d *schema.ResourceData) error { +func cleanUpStepFromTransientValues(stepTypes *cfClient.StepTypes, name, version string) { + if stepTypes != nil { + // Remove transient attributes from metadata + for _, attribute := range []string{"created_at", "accountId", "id", "updated_at"} { + if _, ok := stepTypes.Metadata[attribute]; ok { + delete(stepTypes.Metadata, attribute) + } + } + // Forcing latest to false + // This is needed because in some cases (e.g. adding an old version) the latest attribute is set to `null` by Codefresh + // Having `null` as value causes subsequent calls to fail validation against this attribute + stepTypes.Metadata["latest"] = false - stepTypesYaml, err := yaml.Marshal(stepTypes) - log.Printf("[DEBUG] Marshalled Step Types yaml = %v", string(stepTypesYaml)) - if err != nil { - log.Printf("[DEBUG] Failed to Marshal Step Types yaml = %v", stepTypes) - return err + // If name of version are empty strings we remove them from the data structure + // The use case is for the calculation of the Hash of the Set item, where we don't have access to this information. + // Since that is coming from the other attribute of the resource there's no point to actually consider it for hashing + if name != "" { + stepTypes.Metadata["name"] = name + } else { + delete(stepTypes.Metadata, "name") + } + if version != "" { + stepTypes.Metadata["version"] = version + } else { + delete(stepTypes.Metadata, "version") + } + + } + +} + +func sortVersions(versions []string) []*semver.Version { + log.Printf("[DEBUG] Sorting: %q", versions) + var vs []*semver.Version + for _, version := range versions { + v, err := semver.NewVersion(version) + if err != nil { + diag.Errorf("Error parsing version: %s", err) + } + vs = append(vs, v) } - err = d.Set("step_types_yaml", string(stepTypesYaml)) + sort.Sort(semver.Collection(vs)) + return vs +} + +func mapStepTypesVersionsToResource(stepTypesVersions cfClient.StepTypesVersions, d *schema.ResourceData) error { + + err := d.Set("name", stepTypesVersions.Name) if err != nil { return err } - return nil + err = d.Set("version", flattenVersions(stepTypesVersions.Name, stepTypesVersions.Versions)) + return err + } -func mapResourceToStepTypes(d *schema.ResourceData) *cfClient.StepTypes { +func resourceStepTypesVersionsConfigHash(v interface{}) int { + + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s", m["version_number"].(string))) var stepTypes cfClient.StepTypes - stepTypesYaml := d.Get("step_types_yaml") - yaml.Unmarshal([]byte(stepTypesYaml.(string)), &stepTypes) - if stepTypes.Spec.Steps != nil { - stepTypes.Spec.Steps = extractSteps(stepTypesYaml.(string)) + stepTypesYaml := m["step_types_yaml"].(string) + ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) + // Remove runtime attributes, name and version to avoid discrepancies when comparing hashes + cleanUpStepFromTransientValues(&stepTypes, "", "") + stepTypesYamlByteArray, _ := ghodss.Marshal(stepTypes) + buf.WriteString(fmt.Sprintf("%s", string(stepTypesYamlByteArray))) + hash := hashcode.String(buf.String()) + return hash +} + +func flattenVersions(name string, versions []cfClient.StepTypesVersion) *schema.Set { + + stepVersions := make([]interface{}, 0) + for _, version := range versions { + m := make(map[string]interface{}) + m["version_number"] = version.VersionNumber + cleanUpStepFromTransientValues(&version.StepTypes, name, version.VersionNumber) + stepTypesYaml, err := ghodss.Marshal(version.StepTypes) + log.Printf("[DEBUG] Flattened StepTypes %v", version.StepTypes.Spec) + if err != nil { + log.Fatalf("Error while flattening Versions: %v. Errv=%s", version.StepTypes, err) + } + m["step_types_yaml"] = string(stepTypesYaml) + stepVersions = append(stepVersions, m) + } + log.Printf("[DEBUG] Flattened Versions %s", stepVersions) + return schema.NewSet(resourceStepTypesVersionsConfigHash, stepVersions) +} + +func mapResourceToStepTypesVersions(d *schema.ResourceData) *cfClient.StepTypesVersions { + var stepTypesVersions cfClient.StepTypesVersions + stepTypesVersions.Name = d.Get("name").(string) + versions := d.Get("version").(*schema.Set) + + for _, step := range versions.List() { + version := step.(map[string]interface{})["version_number"].(string) + if version != "" { + var stepTypes cfClient.StepTypes + stepTypesYaml := step.(map[string]interface{})["step_types_yaml"].(string) + + err := ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) + if err != nil { + log.Fatalf("[DEBUG] Unable to mapResourceToStepTypesVersions for version %s. Err= %s", version, err) + } + + cleanUpStepFromTransientValues(&stepTypes, stepTypesVersions.Name, version) + stepVersion := cfClient.StepTypesVersion{ + VersionNumber: version, + StepTypes: stepTypes, + } + if stepVersion.StepTypes.Spec.Steps != nil { + stepVersion.StepTypes.Spec.Steps = extractSteps(stepTypesYaml) + } + + stepTypesVersions.Versions = append(stepTypesVersions.Versions, stepVersion) + } + } + + return &stepTypesVersions +} + +// extractStagesAndSteps extracts the steps and stages from the original yaml string to enable propagation in the `Spec` attribute of the pipeline +// We cannot leverage on the standard marshal/unmarshal because the steps attribute needs to maintain the order of elements +// while by default the standard function doesn't do it because in JSON maps are unordered +func extractSteps(stepTypesYaml string) (steps *orderedmap.OrderedMap) { + // Use mapSlice to preserve order of items from the YAML string + m := yaml.MapSlice{} + err := yaml.Unmarshal([]byte(stepTypesYaml), &m) + if err != nil { + log.Fatal("Unable to unmarshall stepTypesYaml") + } + steps = orderedmap.New() + // Dynamically build JSON object for steps using String builder + stepsBuilder := strings.Builder{} + stepsBuilder.WriteString("{") + // Parse elements of the YAML string to extract Steps and Stages if defined + for _, item := range m { + if item.Key == "spec" { + for _, specItem := range item.Value.(yaml.MapSlice) { + if specItem.Key == "steps" { + switch x := specItem.Value.(type) { + default: + log.Fatalf("unsupported value type: %T", specItem.Value) + + case yaml.MapSlice: + numberOfSteps := len(x) + for index, item := range x { + // We only need to preserve order at the first level to guarantee order of the steps, hence the child nodes can be marshalled + // with the standard library + y, _ := yaml.Marshal(item.Value) + j2, _ := ghodss.YAMLToJSON(y) + stepsBuilder.WriteString("\"" + item.Key.(string) + "\" : " + string(j2)) + if index < numberOfSteps-1 { + stepsBuilder.WriteString(",") + } + } + } + } + } + } + } + stepsBuilder.WriteString("}") + err = json.Unmarshal([]byte(stepsBuilder.String()), &steps) + if err != nil { + log.Fatalf("[DEBUG] Unable to parse steps. ") } - return &stepTypes + return } diff --git a/codefresh/resource_step_types_test.go b/codefresh/resource_step_types_test.go new file mode 100644 index 0000000..e3b0ecd --- /dev/null +++ b/codefresh/resource_step_types_test.go @@ -0,0 +1,215 @@ +package codefresh + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "regexp" + "strings" + "testing" + + "github.com/Masterminds/semver" + cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +var stepTypesNamePrefix = "TerraformAccTest_" + +// Unit Testing +func TestCleanUpStepFromTransientValues(t *testing.T) { + stepTypes := cfClient.StepTypes{ + Metadata: map[string]interface{}{ + "accountId": "test", + "created_at": "test", + "id": "test", + "updated_at": "test", + "latest": true, + "name": "originalTestName", + "version": "oldVersion", + }, + } + newName := "newTestName" + newVersion := "newVersion" + cleanUpStepFromTransientValues(&stepTypes, newName, newVersion) + for _, attributeName := range []string{"created_at", "accountId", "id", "updated_at"} { + if _, ok := stepTypes.Metadata[attributeName]; ok { + t.Errorf("Attribute %s wasn't removed from the Metadata %v.", attributeName, stepTypes) + } + } + if stepTypes.Metadata["name"] != newName { + t.Errorf("Name wasn't updated in Metadata. Expected %s found %s.", newName, stepTypes.Metadata["name"]) + } + if stepTypes.Metadata["version"] != newVersion { + t.Errorf("Version wasn't updated in Metadata. Expected %s found %s.", newVersion, stepTypes.Metadata["version"]) + } +} + +func TestNormalizeYamlStringStepTypes(t *testing.T) { + testFile := "../test_data/step_types/testStepWithRuntimeData.yaml" + yamlString, err := ioutil.ReadFile(testFile) + if err != nil { + t.Errorf("Unable to open test file %s. Err: #%v ", testFile, err) + } + if normalizedYAML, error := normalizeYamlStringStepTypes(string(yamlString)); error == nil { + if strings.Contains(normalizedYAML, "latest: true") { + t.Errorf("Latest attribute wasn't removed from Metadata. %s.", normalizedYAML) + } + if strings.Contains(normalizedYAML, "name: test/step") { + t.Errorf("Name attribute wasn't removed from Metadata. %s.", normalizedYAML) + } + if strings.Contains(normalizedYAML, "version: 0.0.0") { + t.Errorf("Version attribute wasn't removed from Metadata. %s.", normalizedYAML) + } + } else { + t.Errorf("Error while normalising Yaml string for StepTypes%s.", error) + } +} + +func TestSortVersions(t *testing.T) { + versions := []string{"2.13.0", "1.0.0", "1.0.23", "0.12.1", "1.0.8", "2.8.0"} + sortedVersions := []string{"0.12.1", "1.0.0", "1.0.8", "1.0.23", "2.8.0", "2.13.0"} + sortedCollection := sortVersions(versions) + for index, item := range sortedCollection { + checkVersion, err := semver.NewVersion(sortedVersions[index]) + if err != nil { + t.Errorf("Error parsing checkVersion: %s. Err = %v", checkVersion, err) + } + if item.Compare(checkVersion) != 0 { + t.Errorf("Expected version: %s at index %d found %s.", checkVersion, index, item.String()) + } + } +} + +func TestExtractSteps(t *testing.T) { + testFile := "../test_data/step_types/testStepTypesOrder.yaml" + yamlString, err := ioutil.ReadFile(testFile) + if err != nil { + t.Errorf("Unable to read file %s", testFile) + } + orderedSteps := extractSteps(string(yamlString)) + expectedOrdeer := []string{"first_message", "check_second_message_order_maintained"} + for index, stepName := range orderedSteps.Keys() { + if stepName != expectedOrdeer[index] { + t.Errorf("Expected step %s in position %d but got %s", expectedOrdeer[index], index, stepName) + } + } +} + +// Acceptance testing +func TestAccCodefreshStepTypes(t *testing.T) { + // Adding check if we are executing Acceptance test + // This is needed to ensure we have cfClient initialised so that we can retrieve the accountName dynamically + if os.Getenv("TF_ACC") == "1" { + apiClient := testAccProvider.Meta().(*cfClient.Client) + var accountName string + if account, err := apiClient.GetCurrentAccount(); err == nil { + accountName = account.Name + } else { + log.Fatalf("Error, unable to retrieve current account name: %s", err) + } + name := accountName + "/" + stepTypesNamePrefix + acctest.RandString(10) + resourceName := "codefresh_step_types.test" + contentStepsV1, err := ioutil.ReadFile("../test_data/step_types/testSteps.yaml") + if err != nil { + log.Fatal(err) + } + contentStepsV2, err := ioutil.ReadFile("../test_data/step_types/testStepsTemplate.yaml") + if err != nil { + log.Fatal(err) + } + stepTypesV1 := string(contentStepsV1) + stepTypesV2 := string(contentStepsV2) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckCodefreshStepTypesDestroy, + Steps: []resource.TestStep{ + { + Config: testAccCodefreshStepTypesConfig(name, "0.0.1", stepTypesV1, "0.0.2", stepTypesV2), + Check: resource.ComposeTestCheckFunc( + testAccCheckCodefreshStepTypesExists(resourceName), + resource.TestCheckResourceAttr(resourceName, "name", name), + resource.TestCheckResourceAttr(resourceName, "version.0.version_number", "0.0.1"), + resource.TestCheckResourceAttr(resourceName, "version.0.step_types_yaml", stepTypesV1), + resource.TestCheckResourceAttr(resourceName, "version.1.version_number", "0.0.2"), + resource.TestCheckResourceAttr(resourceName, "version.1.step_types_yaml", stepTypesV2), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) + } + +} + +func testAccCheckCodefreshStepTypesExists(resource string) resource.TestCheckFunc { + return func(state *terraform.State) error { + rs, ok := state.RootModule().Resources[resource] + if !ok { + return fmt.Errorf("Not found: %s", resource) + } + if rs.Primary.ID == "" { + return fmt.Errorf("No Record ID is set") + } + + stepTypeID := rs.Primary.ID + + apiClient := testAccProvider.Meta().(*cfClient.Client) + _, err := apiClient.GetStepTypes(stepTypeID) + + if err != nil { + return fmt.Errorf("error fetching step types with resource %s. %s", resource, err) + } + return nil + } +} + +func testAccCheckCodefreshStepTypesDestroy(s *terraform.State) error { + apiClient := testAccProvider.Meta().(*cfClient.Client) + + for _, rs := range s.RootModule().Resources { + + if rs.Type != "codefresh_step_types" { + continue + } + + _, err := apiClient.GetStepTypes(rs.Primary.ID) + + if err == nil { + return fmt.Errorf("Alert still exists") + } + notFoundErr := "not found" + expectedErr := regexp.MustCompile(notFoundErr) + if !expectedErr.Match([]byte(err.Error())) { + return fmt.Errorf("expected %s, got %s", notFoundErr, err) + } + + } + + return nil +} + +// CONFIGS +func testAccCodefreshStepTypesConfig(rName, version1, stepTypesYaml1, version2, stepTypesYaml2 string) string { + return fmt.Sprintf(` +resource "codefresh_step_types" "test" { + name = "%s" + version { + version_number = "%s" + step_types_yaml = %#v + } + version { + version_number = "%s" + step_types_yaml = %#v + } +} +`, rName, version1, stepTypesYaml1, version2, stepTypesYaml2) +} diff --git a/codefresh/resource_step_types_versions.go b/codefresh/resource_step_types_versions.go deleted file mode 100644 index d9be2e5..0000000 --- a/codefresh/resource_step_types_versions.go +++ /dev/null @@ -1,432 +0,0 @@ -package codefresh - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "log" - "sort" - "strings" - - "github.com/Masterminds/semver" - cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" - ghodss "github.com/ghodss/yaml" - "github.com/hashicorp/terraform-plugin-sdk/helper/hashcode" - "github.com/hashicorp/terraform-plugin-sdk/v2/diag" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/iancoleman/orderedmap" - "gopkg.in/yaml.v2" -) - -func resourceStepTypesVersions() *schema.Resource { - return &schema.Resource{ - CreateContext: resourceStepTypesVersionCreate, - ReadContext: resourceStepTypesVersionRead, - UpdateContext: resourceStepTypesVersionUpdate, - DeleteContext: resourceStepTypesVersionDelete, - Importer: &schema.ResourceImporter{ - State: schema.ImportStatePassthrough, - }, - Schema: map[string]*schema.Schema{ - "name": { - Type: schema.TypeString, - ForceNew: true, - Required: true, - }, - "version": { - Type: schema.TypeSet, - Required: true, - MinItems: 1, - Set: resourceStepTypesVersionsConfigHash, - ConfigMode: schema.SchemaConfigModeAttr, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "version_number": { - Type: schema.TypeString, - Required: true, - }, - "step_types_yaml": { - Type: schema.TypeString, - Required: true, - ValidateFunc: stringIsYaml, - DiffSuppressFunc: suppressEquivalentYamlDiffs, - StateFunc: func(v interface{}) string { - template, _ := normalizeYamlStringStepTypes(v) - return template - }, - }, - }, - }, - }, - }, - } -} - -func normalizeYamlStringStepTypes(yamlString interface{}) (string, error) { - var j map[string]interface{} - - if yamlString == nil || yamlString.(string) == "" { - return "", nil - } - - s := yamlString.(string) - err := ghodss.Unmarshal([]byte(s), &j) - metadataMap := j["metadata"].(map[string]interface{}) - //Removing "latest" attribute from metadata since it's transient based on the version - delete(metadataMap, "latest") - delete(metadataMap, "name") - delete(metadataMap, "version") - if err != nil { - return s, err - } - - bytes, _ := ghodss.Marshal(j) - return string(bytes[:]), nil -} - -func resourceStepTypesVersionCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { - - client := meta.(*cfClient.Client) - stepTypes := *mapResourceToStepTypesVersions(d) - - name := d.Get("name").(string) - d.SetId(name) - - // Extract all the versions so that we can order the set based on semantic versioning - mapVersion := make(map[string]cfClient.StepTypes) - var versions []string - for _, version := range stepTypes.Versions { - version.StepTypes.Metadata["name"] = name - version.StepTypes.Metadata["version"] = version.VersionNumber - log.Printf("[DEBUG] Length: %q, %v", versions, len(stepTypes.Versions)) - versions = append(versions, version.VersionNumber) - - mapVersion[version.VersionNumber] = version.StepTypes - - } - - // Create the versions in order based on semver - orderedVersions := sortVersions(versions) - for _, version := range orderedVersions { - step := mapVersion[version.String()] - log.Printf("[DEBUG] Version for create: %q. StepSpec: %v", version, step.Spec.Steps) - _, err := client.CreateStepTypes(&step) - if err != nil { - return diag.Errorf("[DEBUG] Error while creating step types OnCreate. Error = %v", err) - } - } - - return resourceStepTypesVersionRead(ctx, d, meta) -} - -func resourceStepTypesVersionRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { - client := meta.(*cfClient.Client) - - stepTypesIdentifier := d.Id() - if stepTypesIdentifier == "" { - - d.SetId("") - return nil - } - - //Extracting the step just based on the name to validate it exists - stepTypes, err := client.GetStepTypes(stepTypesIdentifier) - if err != nil { - log.Printf("[DEBUG] Step Not found %v. Error = %v", stepTypesIdentifier, err) - d.SetId("") - return nil - } - - var stepVersions cfClient.StepTypesVersions - name := stepTypes.Metadata["name"].(string) - stepVersions.Name = name - versions := d.Get("version").(*schema.Set) - // Try to retrieve defined versions and add to the list if it exists - for _, step := range versions.List() { - version := step.(map[string]interface{})["version_number"].(string) - log.Printf("[DEBUG] Get step version FromList %v", version) - if version != "" { - stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) - log.Printf("[DEBUG] Get step version %v", version) - if err != nil { - log.Printf("[DEBUG] StepVersion not found %v. Error = %v", stepTypesIdentifier+":"+version, err) - } else { - cleanUpStepFromTransientValues(stepTypes, name, version) - stepVersion := cfClient.StepTypesVersion{ - VersionNumber: version, - StepTypes: *stepTypes, - } - stepVersions.Versions = append(stepVersions.Versions, stepVersion) - - } - } - } - - err = mapStepTypesVersionsToResource(stepVersions, d) - - if err != nil { - return diag.Errorf("[DEBUG] Error while mapping stepTypes to resource for READ. Error = %v", err) - } - - return nil -} - -func resourceStepTypesVersionUpdate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { - - client := meta.(*cfClient.Client) - name := d.Get("name").(string) - stepTypesVersions := mapResourceToStepTypesVersions(d) - mapVersionToCreate := make(map[string]cfClient.StepTypes) - versionsPreviouslyDefined := make(map[string]string) - versionsDefined := make(map[string]string) - // Name is set to ForceNew so if we reach this function "version" is changed. Skipping check on HasChange - // Retrieving old version of the resource to enable comparsion with new and determine which versions should be removed - old, _ := d.GetChange("version") - - for _, oldStep := range old.(*schema.Set).List() { - oldVersion := oldStep.(map[string]interface{})["version_number"].(string) - versionsPreviouslyDefined[oldVersion] = oldVersion - } - - // Parse current set: new versions that need to be created are added to a data structure - // that will be sorted later for the creation - // Updates are performed immediately - for _, version := range stepTypesVersions.Versions { - versionNumber := version.VersionNumber - versionsDefined[versionNumber] = versionNumber - - _, err := client.GetStepTypes(name + ":" + versionNumber) - cleanUpStepFromTransientValues(&version.StepTypes, name, versionNumber) - if err != nil { - // If an error occured during Get, we assume step doesn't exist - log.Printf("[DEBUG] Recording for creation: %q", versionNumber) - mapVersionToCreate[versionNumber] = version.StepTypes - } else { - log.Printf("[DEBUG] Update Version step: %q", versionNumber) - _, err := client.UpdateStepTypes(&version.StepTypes) - if err != nil { - return diag.Errorf("[DEBUG] Error while updating stepTypes. Error = %v", err) - - } - } - } - - // Order versions for creation - createVersions := make([]string, len(mapVersionToCreate)) - i := 0 - for k := range mapVersionToCreate { - createVersions[i] = k - i++ - } - orderedVersions := sortVersions(createVersions) - for _, version := range orderedVersions { - step := mapVersionToCreate[version.String()] - log.Printf("[DEBUG] Creating version %s for step types: %s", step.Metadata["version"], step.Metadata["name"]) - _, err := client.CreateStepTypes(&step) - if err != nil { - return diag.Errorf("[DEBUG] Error while creating step types OnUpdate function. Error = %v", err) - } - } - - // If a version is not listed in versionsDefined we can remove it from the system - for version := range versionsPreviouslyDefined { - if _, ok := versionsDefined[version]; !ok { - log.Printf("[DEBUG] Deleting version: %s", version) - // If not defined we remove from the system - err := client.DeleteStepTypes(d.Id() + ":" + version) - if err != nil { - return diag.Errorf("[DEBUG] Error while deleting step_types_versions. Error = %v", err) - } - } - } - - return resourceStepTypesVersionRead(ctx, d, meta) -} - -func resourceStepTypesVersionDelete(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { - - client := meta.(*cfClient.Client) - log.Printf("[DEBUG] Deleting step type: %s", d.Id()) - err := client.DeleteStepTypes(d.Id()) - if err != nil { - return diag.Errorf("[DEBUG] Error while deleting step_types %s. Error = %v", d.Id(), err) - } - - return nil -} - -func cleanUpStepFromTransientValues(stepTypes *cfClient.StepTypes, name, version string) { - if stepTypes != nil { - // Remove transient attributes from metadata - for _, attribute := range []string{"created_at", "accountId", "id", "updated_at"} { - if _, ok := stepTypes.Metadata[attribute]; ok { - delete(stepTypes.Metadata, attribute) - } - } - // Forcing latest to false - // This is needed because in some cases (e.g. adding an old version) the latest attribute is set to `null` by Codefresh - // Having `null` as value causes subsequent calls to fail validation against this attribute - stepTypes.Metadata["latest"] = false - - // If name of version are empty strings we remove them from the data structure - // The use case is for the calculation of the Hash of the Set item, where we don't have access to this information. - // Since that is coming from the other attribute of the resource there's no point to actually consider it for hashing - if name != "" { - stepTypes.Metadata["name"] = name - } else { - delete(stepTypes.Metadata, "name") - } - if version != "" { - stepTypes.Metadata["version"] = version - } else { - delete(stepTypes.Metadata, "version") - } - - } - -} - -func sortVersions(versions []string) []*semver.Version { - log.Printf("[DEBUG] Sorting: %q", versions) - var vs []*semver.Version - for _, version := range versions { - v, err := semver.NewVersion(version) - if err != nil { - diag.Errorf("Error parsing version: %s", err) - } - vs = append(vs, v) - } - - sort.Sort(semver.Collection(vs)) - return vs -} - -func mapStepTypesVersionsToResource(stepTypesVersions cfClient.StepTypesVersions, d *schema.ResourceData) error { - - err := d.Set("name", stepTypesVersions.Name) - if err != nil { - return err - } - - err = d.Set("version", flattenVersions(stepTypesVersions.Name, stepTypesVersions.Versions)) - return err - -} - -func resourceStepTypesVersionsConfigHash(v interface{}) int { - - var buf bytes.Buffer - m := v.(map[string]interface{}) - - buf.WriteString(fmt.Sprintf("%s", m["version_number"].(string))) - var stepTypes cfClient.StepTypes - stepTypesYaml := m["step_types_yaml"].(string) - ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) - // Remove runtime attributes, name and version to avoid discrepancies when comparing hashes - cleanUpStepFromTransientValues(&stepTypes, "", "") - stepTypesYamlByteArray, _ := ghodss.Marshal(stepTypes) - buf.WriteString(fmt.Sprintf("%s", string(stepTypesYamlByteArray))) - hash := hashcode.String(buf.String()) - return hash -} - -func flattenVersions(name string, versions []cfClient.StepTypesVersion) *schema.Set { - - stepVersions := make([]interface{}, 0) - for _, version := range versions { - m := make(map[string]interface{}) - m["version_number"] = version.VersionNumber - cleanUpStepFromTransientValues(&version.StepTypes, name, version.VersionNumber) - stepTypesYaml, err := ghodss.Marshal(version.StepTypes) - log.Printf("[DEBUG] Flattened StepTypes %v", version.StepTypes.Spec) - if err != nil { - log.Fatalf("Error while flattening Versions: %v. Errv=%s", version.StepTypes, err) - } - m["step_types_yaml"] = string(stepTypesYaml) - stepVersions = append(stepVersions, m) - } - log.Printf("[DEBUG] Flattened Versions %s", stepVersions) - return schema.NewSet(resourceStepTypesVersionsConfigHash, stepVersions) -} - -func mapResourceToStepTypesVersions(d *schema.ResourceData) *cfClient.StepTypesVersions { - var stepTypesVersions cfClient.StepTypesVersions - stepTypesVersions.Name = d.Get("name").(string) - versions := d.Get("version").(*schema.Set) - - for _, step := range versions.List() { - version := step.(map[string]interface{})["version_number"].(string) - if version != "" { - var stepTypes cfClient.StepTypes - stepTypesYaml := step.(map[string]interface{})["step_types_yaml"].(string) - - err := ghodss.Unmarshal([]byte(stepTypesYaml), &stepTypes) - if err != nil { - log.Fatalf("[DEBUG] Unable to mapResourceToStepTypesVersions for version %s. Err= %s", version, err) - } - - cleanUpStepFromTransientValues(&stepTypes, stepTypesVersions.Name, version) - stepVersion := cfClient.StepTypesVersion{ - VersionNumber: version, - StepTypes: stepTypes, - } - if stepVersion.StepTypes.Spec.Steps != nil { - stepVersion.StepTypes.Spec.Steps = extractSteps(stepTypesYaml) - } - - stepTypesVersions.Versions = append(stepTypesVersions.Versions, stepVersion) - } - } - - return &stepTypesVersions -} - -// extractStagesAndSteps extracts the steps and stages from the original yaml string to enable propagation in the `Spec` attribute of the pipeline -// We cannot leverage on the standard marshal/unmarshal because the steps attribute needs to maintain the order of elements -// while by default the standard function doesn't do it because in JSON maps are unordered -func extractSteps(stepTypesYaml string) (steps *orderedmap.OrderedMap) { - // Use mapSlice to preserve order of items from the YAML string - m := yaml.MapSlice{} - err := yaml.Unmarshal([]byte(stepTypesYaml), &m) - if err != nil { - log.Fatal("Unable to unmarshall stepTypesYaml") - } - steps = orderedmap.New() - // Dynamically build JSON object for steps using String builder - stepsBuilder := strings.Builder{} - stepsBuilder.WriteString("{") - // Parse elements of the YAML string to extract Steps and Stages if defined - for _, item := range m { - if item.Key == "spec" { - for _, specItem := range item.Value.(yaml.MapSlice) { - if specItem.Key == "steps" { - switch x := specItem.Value.(type) { - default: - log.Fatalf("unsupported value type: %T", specItem.Value) - - case yaml.MapSlice: - numberOfSteps := len(x) - for index, item := range x { - // We only need to preserve order at the first level to guarantee order of the steps, hence the child nodes can be marshalled - // with the standard library - y, _ := yaml.Marshal(item.Value) - j2, _ := ghodss.YAMLToJSON(y) - stepsBuilder.WriteString("\"" + item.Key.(string) + "\" : " + string(j2)) - if index < numberOfSteps-1 { - stepsBuilder.WriteString(",") - } - } - } - } - } - } - } - stepsBuilder.WriteString("}") - err = json.Unmarshal([]byte(stepsBuilder.String()), &steps) - if err != nil { - log.Fatalf("[DEBUG] Unable to parse steps. ") - } - - return -} diff --git a/go.sum b/go.sum index e4f567f..030134c 100644 --- a/go.sum +++ b/go.sum @@ -245,7 +245,6 @@ github.com/hashicorp/terraform-json v0.5.0 h1:7TV3/F3y7QVSuN4r9BEXqnWqrAyeOtON8f github.com/hashicorp/terraform-json v0.5.0/go.mod h1:eAbqb4w0pSlRmdvl8fOyHAi/+8jnkVYN28gJkSJrLhU= github.com/hashicorp/terraform-plugin-sdk v1.7.0 h1:B//oq0ZORG+EkVrIJy0uPGSonvmXqxSzXe8+GhknoW0= github.com/hashicorp/terraform-plugin-sdk v1.7.0/go.mod h1:OjgQmey5VxnPej/buEhe+YqKm0KNvV3QqU4hkqHqPCY= -github.com/hashicorp/terraform-plugin-sdk v1.16.0 h1:NrkXMRjHErUPPTHQkZ6JIn6bByiJzGnlJzH1rVdNEuE= github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1 h1:h8TtYDlIACXQ6LNJesvSHuxskaPUAX/nltvqp0Kp1vk= github.com/hashicorp/terraform-plugin-sdk/v2 v2.0.0-rc.2.0.20200717132200-7435e2abc9d1/go.mod h1:aWg/hVISyjdpUOt89SSrplxffuq8HPEnIWGyKDpDzCo= github.com/hashicorp/terraform-plugin-test v1.2.0/go.mod h1:QIJHYz8j+xJtdtLrFTlzQVC0ocr3rf/OjIpgZLK56Hs= diff --git a/test_data/step_types/testStepTypesOrder.yaml b/test_data/step_types/testStepTypesOrder.yaml new file mode 100644 index 0000000..6d5a8ee --- /dev/null +++ b/test_data/step_types/testStepTypesOrder.yaml @@ -0,0 +1,6 @@ +spec: + steps: + first_message: + name: firstMessage + check_second_message_order_maintained: + name: secondMessage diff --git a/test_data/step_types/testStepWithRuntimeData.yaml b/test_data/step_types/testStepWithRuntimeData.yaml new file mode 100644 index 0000000..c88592d --- /dev/null +++ b/test_data/step_types/testStepWithRuntimeData.yaml @@ -0,0 +1,32 @@ +kind: step-type +metadata: + description: Testing step + examples: + - description: test + workflow: + steps: + test_step: + title: Test step + type: test/steps + version: "1.0" + isPublic: false + latest: true + name: test/step + official: false + stage: incubating + version: 0.0.0 +spec: + steps: + first_message: + commands: + - echo "Message first step" + image: alpine + name: firstMessage + title: Info message + second_message: + commands: + - echo "Message second step" + image: alpine + name: secondMessage + title: Info message +version: "1.0" diff --git a/test_data/step_types/testSteps.yaml b/test_data/step_types/testSteps.yaml new file mode 100644 index 0000000..b30e279 --- /dev/null +++ b/test_data/step_types/testSteps.yaml @@ -0,0 +1,29 @@ +kind: step-type +metadata: + description: Testing step + examples: + - description: test + workflow: + steps: + test_step: + title: Test step + type: test/steps + version: "1.0" + isPublic: false + official: false + stage: incubating +spec: + steps: + first_message: + commands: + - echo "Message first step" + image: alpine + name: firstMessage + title: Info message + second_message: + commands: + - echo "Message second step" + image: alpine + name: secondMessage + title: Info message +version: "1.0" diff --git a/test_data/step_types/testStepsTemplate.yaml b/test_data/step_types/testStepsTemplate.yaml new file mode 100644 index 0000000..fe1bf32 --- /dev/null +++ b/test_data/step_types/testStepsTemplate.yaml @@ -0,0 +1,32 @@ +kind: step-type +metadata: + description: Testing stepsTemplate + examples: + - description: test + workflow: + steps: + test_step: + title: Test stepsTemplate + type: test/stepsTemplate + version: "1.0" + isPublic: false + official: false + stage: incubating +spec: + delimiters: + left: '[[' + right: ']]' + stepsTemplate: |- + first_message: + name: firstMessage + title: Info message + image: alpine + commands: + - echo "Message first step" + check_second_message_is_maintain_in_order: + name: secondMessage + title: Info message + image: alpine + commands: + - echo "Message second step" +version: "1.0" From 0b251585aea6f05fbc43b7c7ef9e950f87994edb Mon Sep 17 00:00:00 2001 From: Sandro Gattuso Date: Sat, 6 Mar 2021 08:39:07 +1100 Subject: [PATCH 5/6] Remove documentation of old implementation --- docs/data/step-types-versions.md | 2 +- docs/resources/step-types-versions.md | 50 -------------------- docs/resources/step-types.md | 67 ++++++++++++++------------- 3 files changed, 36 insertions(+), 83 deletions(-) delete mode 100644 docs/resources/step-types-versions.md diff --git a/docs/data/step-types-versions.md b/docs/data/step-types-versions.md index a8ee0ca..57150d1 100644 --- a/docs/data/step-types-versions.md +++ b/docs/data/step-types-versions.md @@ -1,5 +1,5 @@ # Data Source: codefresh_step_types_versions -This data source allows to retrieve the latest published version of a step-types +This data source allows to retrieve the list of published versions of a step-types ## Example Usage diff --git a/docs/resources/step-types-versions.md b/docs/resources/step-types-versions.md deleted file mode 100644 index 4ffdd25..0000000 --- a/docs/resources/step-types-versions.md +++ /dev/null @@ -1,50 +0,0 @@ -# Step-type-versions Resource - -The Step-type-version resource allows to create your own typed step and manage all it's published versions. -The resource allows to handle the life-cycle of the version by allowing specifying multiple blocks `version` where the user provides a version number and the yaml file representing the plugin. -More about custom steps in the [official documentation](https://codefresh.io/docs/docs/codefresh-yaml/steps/#creating-a-typed-codefresh-plugin). - -## Known limitations and disclaimers -### Version and name in yaml Metadata are ignored. -The version and name of the step declared in the yaml files are superseeded by the attributes specified at resource level: -- `name` : at top level -- `version_numer`: specified in the `version` block -The above are added/replaced at runtime time. - -### Number of API requests -This resource makes a lot of additional API calls to validate the steps and retrieve all the version available. -Caution is recommended on the amount of versions maintained and the number of resources defined in a single project. - - -## Example Usage - -```hcl - -data "codefresh_current_account" "acc" { -} - -resource "codefresh_step_types_versions" "my-custom-step" { - name = "${data.codefresh_current_account.acc.name}/my-custom-step" - - version { - version_number = "0.0.1" - step_types_yaml = file("./templates/plugin-0.0.1.yaml") - } - version { - version_number = "0.0.2" - step_types_yaml = file("./templates/plugin-0.0.2.yaml") - } - .... -} -} -``` - -## Argument Reference -- `name` - (Required) The name for the step-type -- `version` - (At least 1 Required) A collection of `version` blocks as documented below. - ---- - -`version` supports the following: -- `version_number` - (Required) String representing the semVer for the step -- `step_types_yaml` (Required) YAML String containing a valid definition of a typed plugin diff --git a/docs/resources/step-types.md b/docs/resources/step-types.md index aec1804..dcb8192 100644 --- a/docs/resources/step-types.md +++ b/docs/resources/step-types.md @@ -1,47 +1,50 @@ -# Step-type Resource +# Step-types Resource -The Step-type resource allows to create your own typed step. +The Step-types resource allows to create your own typed step and manage all it's published versions. +The resource allows to handle the life-cycle of the version by allowing specifying multiple blocks `version` where the user provides a version number and the yaml file representing the plugin. More about custom steps in the [official documentation](https://codefresh.io/docs/docs/codefresh-yaml/steps/#creating-a-typed-codefresh-plugin). ## Known limitations and disclaimers -### Differences during plan phase -When executing `terraform plan` the diff presented will be the comparison between the latest published version and the version configured in the `step_types_yaml`. -At this stage the Read function doesn't have the reference to the new version in order to be able to retrieve the exact version for comparison. +### Version and name in yaml Metadata are ignored. +The version and name of the step declared in the yaml files are superseeded by the attributes specified at resource level: +- `name` : at top level +- `version_numer`: specified in the `version` block +The above are added/replaced at runtime time. + +### Number of API requests +This resource makes a lot of additional API calls to validate the steps and retrieve all the version available. +Caution is recommended on the amount of versions maintained and the number of resources defined in a single project. -### Deletion of resource -When executing `terraform destroy` the step-stype is completely removed (including all the existing version) ## Example Usage ```hcl -resource "codefresh_step_types" "custom_step" { - - # NOTE: you can also load the yaml from a file with `step_types_yaml = file("PATH-TO-FILE.yaml")` - # Example has been cut down for simplicity. Yaml schema must be compliant with the what specified in the documentation for typed plugins - step_types_yaml = </custom-step - ... -spec: - arguments: |- - { - .... - } -delimiters: - left: '[[' - right: ']]' - stepsTemplate: |- - print_info_message: - name: Test step - ... -YAML + +data "codefresh_current_account" "acc" { +} + +resource "codefresh_step_types_versions" "my-custom-step" { + name = "${data.codefresh_current_account.acc.name}/my-custom-step" + + version { + version_number = "0.0.1" + step_types_yaml = file("./templates/plugin-0.0.1.yaml") + } + version { + version_number = "0.0.2" + step_types_yaml = file("./templates/plugin-0.0.2.yaml") + } + .... +} } ``` ## Argument Reference +- `name` - (Required) The name for the step-type +- `version` - (At least 1 Required) A collection of `version` blocks as documented below. -- `step_types_yaml` (Required) YAML String containing a valid definition of a typed plugin - +--- +`version` supports the following: +- `version_number` - (Required) String representing the semVer for the step +- `step_types_yaml` (Required) YAML String containing a valid definition of a typed plugin From 84b9f2da7409f7aa45c95a74035ef7eba8955991 Mon Sep 17 00:00:00 2001 From: Sandro Gattuso Date: Wed, 10 Mar 2021 20:47:15 +1100 Subject: [PATCH 6/6] Datasource version matching the resource definition --- codefresh/data_step_types.go | 71 +++++++++++++++------------ codefresh/data_step_types_versions.go | 45 ----------------- codefresh/provider.go | 17 +++---- docs/data/step-types-versions.md | 23 --------- docs/data/step-types.md | 17 +++++-- 5 files changed, 60 insertions(+), 113 deletions(-) delete mode 100644 codefresh/data_step_types_versions.go delete mode 100644 docs/data/step-types-versions.md diff --git a/codefresh/data_step_types.go b/codefresh/data_step_types.go index 3a10020..928eefc 100644 --- a/codefresh/data_step_types.go +++ b/codefresh/data_step_types.go @@ -2,9 +2,9 @@ package codefresh import ( "fmt" + "log" cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" - "github.com/ghodss/yaml" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) @@ -17,12 +17,20 @@ func dataSourceStepTypes() *schema.Resource { Required: true, }, "version": { - Type: schema.TypeString, - Optional: true, - }, - "step_types_yaml": { - Type: schema.TypeString, + Type: schema.TypeSet, Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "version_number": { + Type: schema.TypeString, + Computed: true, + }, + "step_types_yaml": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, }, }, } @@ -31,40 +39,39 @@ func dataSourceStepTypes() *schema.Resource { func dataSourceStepTypesRead(d *schema.ResourceData, meta interface{}) error { client := meta.(*cfClient.Client) - var stepTypes *cfClient.StepTypes var err error - identifier := d.Get("name").(string) - version, versionOk := d.GetOk("version") + var versions []string + stepTypesIdentifier := d.Get("name").(string) - if versionOk { - identifier = identifier + ":" + version.(string) - } - stepTypes, err = client.GetStepTypes(identifier) - if err != nil { - return err + d.SetId(stepTypesIdentifier) + if versions, err = client.GetStepTypesVersions(stepTypesIdentifier); err == nil { + var stepVersions cfClient.StepTypesVersions + stepVersions.Name = stepTypesIdentifier + d.Set("versions", versions) + for _, version := range versions { + stepTypes, err := client.GetStepTypes(stepTypesIdentifier + ":" + version) + if err != nil { + log.Printf("[DEBUG] Skipping version %v due to error %v", version, err) + } else { + stepVersion := cfClient.StepTypesVersion{ + VersionNumber: version, + StepTypes: *stepTypes, + } + stepVersions.Versions = append(stepVersions.Versions, stepVersion) + } + } + return mapStepTypesVersionsToResource(stepVersions, d) } - if stepTypes == nil { - return fmt.Errorf("data.codefresh_step_types - cannot find step-types") - } + return fmt.Errorf("data.codefresh_step_types - was unable to retrieve the versions for step_type %s", stepTypesIdentifier) - return mapDataSetTypesToResource(stepTypes, d) } -func mapDataSetTypesToResource(stepTypes *cfClient.StepTypes, d *schema.ResourceData) error { - - if stepTypes == nil || stepTypes.Metadata["name"].(string) == "" { - return fmt.Errorf("data.codefresh_step_types - failed to mapDataSetTypesToResource") - } - d.SetId(stepTypes.Metadata["name"].(string)) - - d.Set("name", d.Id()) - - stepTypesYaml, err := yaml.Marshal(stepTypes) +func mapDataSetTypesToResource(stepTypesVersions cfClient.StepTypesVersions, d *schema.ResourceData) error { + err := d.Set("name", stepTypesVersions.Name) if err != nil { return err } - d.Set("step_types_yaml", string(stepTypesYaml)) - - return nil + err = d.Set("version", flattenVersions(stepTypesVersions.Name, stepTypesVersions.Versions)) + return err } diff --git a/codefresh/data_step_types_versions.go b/codefresh/data_step_types_versions.go deleted file mode 100644 index 334517a..0000000 --- a/codefresh/data_step_types_versions.go +++ /dev/null @@ -1,45 +0,0 @@ -package codefresh - -import ( - "fmt" - - cfClient "github.com/codefresh-io/terraform-provider-codefresh/client" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" -) - -func dataSourceStepTypesVersions() *schema.Resource { - return &schema.Resource{ - Read: dataSourceStepTypesVersionsRead, - Schema: map[string]*schema.Schema{ - "name": { - Type: schema.TypeString, - Required: true, - }, - "versions": { - Type: schema.TypeList, - Computed: true, - Elem: &schema.Schema{ - Type: schema.TypeString, - }, - }, - }, - } -} - -func dataSourceStepTypesVersionsRead(d *schema.ResourceData, meta interface{}) error { - - client := meta.(*cfClient.Client) - var versions []string - var err error - - if name, nameOk := d.GetOk("name"); nameOk { - if versions, err = client.GetStepTypesVersions(name.(string)); err == nil { - d.SetId(name.(string)) - d.Set("name", name.(string)) - d.Set("versions", versions) - } - return err - } - return fmt.Errorf("data.codefresh_step_types_versions - must specify name") - -} diff --git a/codefresh/provider.go b/codefresh/provider.go index 4433018..bdf4182 100644 --- a/codefresh/provider.go +++ b/codefresh/provider.go @@ -27,15 +27,14 @@ func Provider() *schema.Provider { }, }, DataSourcesMap: map[string]*schema.Resource{ - "codefresh_account": dataSourceAccount(), - "codefresh_context": dataSourceContext(), - "codefresh_current_account": dataSourceCurrentAccount(), - "codefresh_idps": dataSourceIdps(), - "codefresh_step_types": dataSourceStepTypes(), - "codefresh_step_types_versions": dataSourceStepTypesVersions(), - "codefresh_team": dataSourceTeam(), - "codefresh_user": dataSourceUser(), - "codefresh_users": dataSourceUsers(), + "codefresh_account": dataSourceAccount(), + "codefresh_context": dataSourceContext(), + "codefresh_current_account": dataSourceCurrentAccount(), + "codefresh_idps": dataSourceIdps(), + "codefresh_step_types": dataSourceStepTypes(), + "codefresh_team": dataSourceTeam(), + "codefresh_user": dataSourceUser(), + "codefresh_users": dataSourceUsers(), }, ResourcesMap: map[string]*schema.Resource{ "codefresh_account": resourceAccount(), diff --git a/docs/data/step-types-versions.md b/docs/data/step-types-versions.md deleted file mode 100644 index 57150d1..0000000 --- a/docs/data/step-types-versions.md +++ /dev/null @@ -1,23 +0,0 @@ -# Data Source: codefresh_step_types_versions -This data source allows to retrieve the list of published versions of a step-types - -## Example Usage - -```hcl -data "codefresh_step_types_versions" "freestyle" { - name = "freestyle" -} - -output "versions" { - value = data.codefresh_step_types_versions.freestyle.versions -} - -``` - -## Argument Reference - -* `name` - (Required) Name of the step-types to be retrieved - -## Attributes Reference - -* `versions` - List of versions available for the custom plugin (step-types). diff --git a/docs/data/step-types.md b/docs/data/step-types.md index f14dae9..cedfe65 100644 --- a/docs/data/step-types.md +++ b/docs/data/step-types.md @@ -1,5 +1,5 @@ # Data Source: codefresh_step_types -This data source allows to retrieve the latest published version of a step-types +This data source allows to retrieve the published versions of a step-types ## Example Usage @@ -8,9 +8,13 @@ data "codefresh_step_types" "freestyle" { name = "freestyle" } +local { + freestyle_map = { for step_definition in data.codefresh_step_types.freestyle.version: step_definition.version_number => step_definition } +} + output "test" { # Value is return as YAML - value = yamldecode(data.codefresh_step_types.freestyle.step_types_yaml).metadata.updated_at + value = local.freestyle_map[keys(local.freestyle_map)[0]].version_number } ``` @@ -18,8 +22,13 @@ output "test" { ## Argument Reference * `name` - (Required) Name of the step-types to be retrieved -* `version` - (Optional) Version to be retrieved. If not specified, the latest published will be returned ## Attributes Reference -* `step_types_yaml` - The yaml string representing the custom plugin (step-types). +- `version` - A Set of `version` blocks as documented below. + +--- + +`version` provides the following: +- `version_number` - String representing the semVer for the step +- `step_types_yaml` - YAML String containing the definition of a typed plugin