From ac1a35a977f82486cdf4efbc8c87be0ffc34192a Mon Sep 17 00:00:00 2001 From: Murad Biashimov Date: Fri, 8 Nov 2024 08:55:54 +0100 Subject: [PATCH] feat(changelog): add changelog parser (#1891) --- CHANGELOG.md | 61 +++++++------ changelog/formatter.go | 168 ++++++++++++++++++++++++++++++++++++ changelog/formatter_test.go | 122 ++++++++++++++++++++++++++ changelog/main.go | 25 ++++-- 4 files changed, 336 insertions(+), 40 deletions(-) create mode 100644 changelog/formatter.go create mode 100644 changelog/formatter_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 05d156f3b..31fe8684e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,23 @@ nav_order: 1 # Changelog + + ## [MAJOR.MINOR.PATCH] - YYYY-MM-DD - Add support for `autoscaler` service integration -- Add `aiven_opensearch` resource field `opensearch_user_config.azure_migration.include_aliases`: Whether to restore aliases alongside their associated indexes -- Add `aiven_opensearch` resource field `opensearch_user_config.gcs_migration.include_aliases`: Whether to restore aliases alongside their associated indexes -- Add `aiven_opensearch` resource field `opensearch_user_config.s3_migration.include_aliases`: Whether to restore aliases alongside their associated indexes -- Add `aiven_opensearch` datasource field `opensearch_user_config.gcs_migration.include_aliases`: Whether to restore aliases alongside their associated indexes -- Add `aiven_opensearch` datasource field `opensearch_user_config.azure_migration.include_aliases`: Whether to restore aliases alongside their associated indexes -- Add `aiven_opensearch` datasource field `opensearch_user_config.s3_migration.include_aliases`: Whether to restore aliases alongside their associated indexes +- Add `aiven_opensearch` resource field `opensearch_user_config.azure_migration.include_aliases`: Whether to restore + aliases alongside their associated indexes +- Add `aiven_opensearch` resource field `opensearch_user_config.gcs_migration.include_aliases`: Whether to restore aliases + alongside their associated indexes +- Add `aiven_opensearch` resource field `opensearch_user_config.s3_migration.include_aliases`: Whether to restore aliases + alongside their associated indexes +- Add `aiven_opensearch` datasource field `opensearch_user_config.gcs_migration.include_aliases`: Whether to restore + aliases alongside their associated indexes +- Add `aiven_opensearch` datasource field `opensearch_user_config.azure_migration.include_aliases`: Whether to restore + aliases alongside their associated indexes +- Add `aiven_opensearch` datasource field `opensearch_user_config.s3_migration.include_aliases`: Whether to restore + aliases alongside their associated indexes - Change `aiven_cassandra` resource field `cassandra_user_config.additional_backup_regions`: remove deprecation - Change `aiven_cassandra` datasource field `cassandra_user_config.additional_backup_regions`: remove deprecation @@ -71,14 +79,14 @@ nav_order: 1 - Fix `aiven_transit_gateway_vpc_attachment`: remove `peer_region` deprecation, mark the field as create only. - Add `aiven_valkey` resource as a beta resource - Add `aiven_valkey_user` resource as a beta resource -- Temporarily remove the `aiven_project_user` deprecation until we find a suitable alternative. +- Temporarily remove the `aiven_project_user` deprecation until we find a suitable alternative. ## [4.20.0] - 2024-06-26 - Mark several sensitive user config fields as "sensitive" - Fix aiven-go-client dependency version - Fix `aiven_organization_user_group` resource - `name` field is required -- Use `TypeSet` for `config_properties_exclude` +- Use `TypeSet` for `config_properties_exclude` ## [4.19.1] - 2024-05-05 @@ -89,7 +97,7 @@ nav_order: 1 - Add `is_super_admin` flag to `aiven_organization_application_user` resource - Add `aiven_mirrormaker_replication_flow` replication factor - Remove `aiven_mirrormaker_replication_flow` global mutex, the backend has been fixed -- Remove service version validation to allow running new service versions without the provider upgrade +- Remove service version validation to allow running new service versions without the provider upgrade - Fix `aiven_organization_application_user_token` crashes with empty optional fields - Fix `ip_filter` conversion issue @@ -107,12 +115,12 @@ nav_order: 1 ## [4.16.0] - 2024-04-30 - Fix incorrect timeout values used in certain cases -- Fix sending `aiven_kafka_topic` config default values +- Fix sending `aiven_kafka_topic` config default values - Fix sending `false` values in `aiven_kafka_topic` config - Fix `aiven_pg` user config fields with `__dot__` substring in name - Validate `aiven_kafka_topic` topic name conflict on `terraform plan` - Mark service connection info blocks as `sensitive`. See SDK [bug](https://github.com/hashicorp/terraform-plugin-sdk/issues/201). -- Remove redundant service connection info fields +- Remove redundant service connection info fields - Add Thanos resource (`aiven_thanos`) to allow for the creation and management of Thanos services (currently available as beta) ## [4.15.0] - 2024-03-21 @@ -134,7 +142,7 @@ nav_order: 1 - `emit_backward_heartbeats_enabled` - `offset_syncs_topic_location` - `replication_policy_class` -- Remove the beta flag for `aiven_organization_user_group_member` and `aiven_organization_group_project` resources +- Remove the beta flag for `aiven_organization_user_group_member` and `aiven_organization_group_project` resources ## [4.14.0] - 2024-02-20 @@ -145,7 +153,7 @@ nav_order: 1 - Fix `aiven_organization_user_group` resource - `description` field is required - Use golang 1.22 - Output explicitly `termination_protection = true -> false` when service property is removed -- Fix `aiven_flink_application_deployment` deletion +- Fix `aiven_flink_application_deployment` deletion ## [4.13.3] - 2024-01-29 @@ -165,7 +173,7 @@ nav_order: 1 - Add organization application users support - Add organization application user tokens support -- Configure "insufficient broker" error retries timeout +- Configure "insufficient broker" error retries timeout - Enable `local_retention_*` fields in `aiven_kafka_topic` resource - Validate that `local_retention_bytes` is not bigger than `retention_bytes` @@ -200,12 +208,11 @@ nav_order: 1 - Retry kafka topic creation error `Cluster only has N broker(s), cannot set replication factor to M` - Fix Kafka Topic migration issues from V3 to V4. - Fix V3 to V4 migration issue related to cloud_name diff. -- Add support for the `aiven_organization_user_group_member` resource, allowing the association of groups with the users. Please note that this resource is in the beta stage, and to use it, you would need to set the environment variable PROVIDER_AIVEN_ENABLE_BETA to a non-zero value. - +- Add support for the `aiven_organization_user_group_member` resource, allowing the association of groups with the users. Please note that this resource is in the beta stage, and to use it, you would need to set the environment variable PROVIDER_AIVEN_ENABLE_BETA to a non-zero value. ## [4.9.4] - 2023-12-13 -- Fix race issues with `aiven_mirrormaker_replication_flow` on create/update/delete operations +- Fix race issues with `aiven_mirrormaker_replication_flow` on create/update/delete operations - Add `tech_emails` to services ## [4.9.3] - 2023-10-27 @@ -237,7 +244,7 @@ nav_order: 1 - Add Organization User Groups support - Fixed incorrect `account_id` behavior in mixed constraint setup in `aiven_project` resource -- Use updated aiven-go-client with enhanced retries +- Use updated aiven-go-client with enhanced retries - Change `plan` from optional to required - Improve `disk_space` deprecation message to become more explicit to migrating users - Fix account deletion flakiness @@ -291,7 +298,7 @@ nav_order: 1 - Added docs and validation for `aiven_service_integration_endpoint` - Dropped `signalfx` from supported integration types -- Fix MySQL user creation authentication field +- Fix MySQL user creation authentication field - Fix Account SAML Field mapping set method - Adjust generated SQL for ClickHouse privilege grants - Fix `required` not generated for top level fields for user config options @@ -369,7 +376,7 @@ nav_order: 1 - Integration with Kafka source - Integration with PostgreSQL source - Fix VPC peering ID parser -- Add `offset_syncs_topic_location` support for `aiven_mirrormaker_replication_flow` resource +- Add `offset_syncs_topic_location` support for `aiven_mirrormaker_replication_flow` resource - Add `ssl` and `kafka_authentication_method` output support in service components - Fix `admin_username` and `admin_password` fields diff @@ -379,7 +386,7 @@ nav_order: 1 - Add `ip_filter_object` and `namespaces_object` user config options which are meant to extend the existing `ip_filter` and `namespaces` ones - Revert `datasource_project_vpc` `cloud_name` and `project` deprecations - Add extra timeout for `kafka_connect` service integration create -- Support `clickhouse_kafka` integration type in `aiven_service_integration` +- Support `clickhouse_kafka` integration type in `aiven_service_integration` - Fix `aiven_transit_gateway_vpc_attachment` fails to parse ID - Prevent generation of `Default` field in static schema generator - Add `self_link` field to `aiven_gcp_vpc_peering_connection` resource @@ -399,7 +406,7 @@ nav_order: 1 ## [3.8.0] - 2022-09-30 - Fix `aiven_gcp_vpc_peering_connection` creation -- Improve static IP error handling end messaging +- Improve static IP error handling end messaging - Fix `aiven_account_authentication` resource update, add tests - Change `aiven_project_vpc` datasource behaviour - Fix `aiven_service_component` optional parameters filters @@ -458,7 +465,7 @@ nav_order: 1 - Update Changelog Enforcer workflow - Add CodeQL workflow - Add `opensearch_index` support to `aiven_flink_table` -- Add not found checks to the Kafka availability waiter +- Add not found checks to the Kafka availability waiter - Add PostgreSQL max connections and PgBouncer outputs - Perform general code clean-up and add `revive` linter - Add support for new user configuration options @@ -543,15 +550,11 @@ nav_order: 1 - `aiven_service` and `aiven_elasticsearch` resources were deleted - `aiven_project` resource previously deprecated schema field were deleted - Deprecated resources and data-sources: - - `aiven_database` - `aiven_service_user` - `aiven_vpc_peering_connection` - New resources and data-sources: - - `aiven_aws_vpc_peering_connection` - `aiven_azure_vpc_peering_connection` - `aiven_gcp_vpc_peering_connection` @@ -1002,19 +1005,16 @@ Add backwards compatibility for old TF state files created before Kafka `topic` ## [1.2.1] - 2020-03-02 Terraform client-side termination protection for resources: - - aiven_kafka_topic - aiven_database ## [1.2.0] - 2020-02-18 - Following new types of resources have been added: - - account - account_team - account_team_member - account_team_project - - New configuration options - Fix for a read-only replica service types - Service specific acceptance tests @@ -1151,5 +1151,4 @@ Support termination_protection property for services. Support all Aiven resource types. Also large changes to previously supported resource types, such as full support for all user config options. - **NOTE**: This version is not backwards compatible with older versions. diff --git a/changelog/formatter.go b/changelog/formatter.go new file mode 100644 index 000000000..ef2b810e3 --- /dev/null +++ b/changelog/formatter.go @@ -0,0 +1,168 @@ +package main + +import ( + "fmt" + "regexp" + "strings" +) + +const ( + draftVersion = "MAJOR.MINOR.PATCH" + draftDate = "YYYY-MM-DD" + defaultBullet = "- " + defaultLineMaxLength = 120 // Line soft wrap settings +) + +type changelogItem struct { + Date, Version, Content string +} + +var ( + reVersion = regexp.MustCompile(`\w+\.\w+\.\w+`) + reDate = regexp.MustCompile(`\w{4}-\w{2}-\w{2}`) + reSplitEntries = regexp.MustCompile(`(?m)^(\b[^a-z]| *- +)`) // A line that begins with "-" or a non-letter + reBulletLevel = regexp.MustCompile(`^ *- +`) + reSpaces = regexp.MustCompile(`\s+`) + reTrailingSpace = regexp.MustCompile(`\s+$`) +) + +// updateChangelog updates the changelog with the given addLines +// Soft-wraps lines to the given lineLength +// When reformat is true, reformats the whole given content +func updateChangelog(content string, lineLength int, reformat bool, addLines ...string) (string, error) { + if addLines == nil && !reformat { + return content, nil + } + + lines := strings.Split(reTrailingSpace.ReplaceAllString(content, ""), "\n") + items, start, end := parseItems(lines) + addText := strings.Join(addLines, "\n") + + if len(items) != 0 && items[0].Version == draftVersion { + // Appends to the current draft + items[0].Content = fmt.Sprintf("%s\n%s", items[0].Content, addText) + } else { + // The First item is not the draft, so we need to add a new item + items = append(items, &changelogItem{ + Version: draftVersion, + Date: draftDate, + Content: content, + }) + } + + result := lines[:start] + for i, v := range items { + c := strings.TrimSpace(v.Content) + if i == 0 || reformat { + c = formatContent(c, lineLength) + } + header := fmt.Sprintf("## [%s] - %s", v.Version, v.Date) + result = append(result, header, "", c, "") // Empty lines for readability and formatting + } + + result = append(result, lines[end+1:]...) // Adds the rest of the file + return strings.Join(result, "\n"), nil +} + +func parseItems(lines []string) ([]*changelogItem, int, int) { + start := max(0, len(lines)-1) + end := start + var item *changelogItem + items := make([]*changelogItem, 0) + for i, line := range lines { + if strings.HasPrefix(line, "##") { + if item == nil { + start = i + } + + item = &changelogItem{ + Date: reDate.FindString(line), + Version: reVersion.FindString(line), + } + + items = append(items, item) + continue + } + + if line != "" && item != nil { + item.Content = item.Content + strings.TrimSuffix(line, " ") + "\n" + end = i + } + } + return items, start, end +} + +func formatContent(content string, lineLength int) string { + + // Golang doesn't support regexp "lookarounds", so we need to split the content, + // and then join it to keep what we otherwise would be just ignored by negative lookbehind + seps := reSplitEntries.FindAllStringSubmatchIndex(content, -1) + chunks := reSplitEntries.Split(content, -1) + list := make([]string, 0) + seen := make(map[string]bool) + for i, v := range seps { + // This is the separator between the entries + sep := content[v[0]:v[1]] + + // Joins with the separator in case it has "negative lookbehind" part + text := strings.TrimRight(sep+chunks[i+1], "\n ") + + // Looks for the bullet + bullet := reBulletLevel.FindString(text) + if bullet != "" { + // When found, separates the text + text = strings.SplitN(text, bullet, 2)[1] + } else { + // Otherwise, uses the default bullet + bullet = defaultBullet + } + + // Removes original spaces and newlines + point := addBullet(bullet, softWrap(reSpaces.ReplaceAllString(text, " "), lineLength)) + + // Removes duplicates + if !seen[point] { + seen[point] = true + list = append(list, point) + } + } + + return strings.Join(list, "\n") +} + +var reShortWords = regexp.MustCompile(`(\b.{1,3}\b) +`) + +// softWrap wraps text to a given size +// Keeps prepositions and articles together with the next word for better readability +func softWrap(text string, size int) []string { + text = reShortWords.ReplaceAllString(text, "$1⍽") + + j := 0 + result := make([]string, 1) + for i, w := range strings.Split(text, " ") { + w = strings.ReplaceAll(w, "⍽", " ") + switch { + case i == 0: + result[j] += w + case len(result[j])+len(w) < size: + result[j] += " " + w + default: + result = append(result, w) // nolint: makezero // By some reason linter doesn't understand it has length 1 + j++ + } + } + return result +} + +// addBullet add the given bullet to the beginning of the first line and indents the rest +func addBullet(bullet string, lines []string) string { + prefix := strings.Repeat(" ", len(bullet)) + for i, v := range lines { + if i == 0 { + lines[i] = bullet + v + } else { + lines[i] = prefix + v + } + } + return strings.Join(lines, "\n") +} diff --git a/changelog/formatter_test.go b/changelog/formatter_test.go new file mode 100644 index 000000000..f8f3c4ec2 --- /dev/null +++ b/changelog/formatter_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testSample = `# Changelog + + + + +## [MAJOR.MINOR.PATCH] - YYYY-MM-DD + +- Fix aiven_project : can't migrate from account_id (deprecated) to parent_id +- Add aiven_organization_user_list beta resource +- Add AIVEN_ALLOW_IP_FILTER_PURGE environment variable to allow purging of IP filters. This is a safety feature to + prevent accidental purging of IP filters, which can lead to loss of access to services. To enable purging, set the + environment variable to any value before running Terraform commands. +- Use TypeSet for ip_filter_object +- Deprecated account_id in aiven_project and aiven_billing_group resources + - Please use parent_id instead, account_id is going to be removed in the next major release +- Fix incorrect behavior of aiven_mirrormaker_replication_flow schema fields: + - sync_group_offsets_enabled + - sync_group_offsets_interval_seconds +- Fix project creation with account_id empty and add possibility to dissociate project from an account by not + setting account_id +- Fix typos in documentation and examples + +## [4.26.0] - 2024-09-25 + +- Remove aiven_valkey from beta resources +- Remove aiven_valkey_user from beta resources +- Adds aiven_organization_permission example +- Add + capability to map external service user with internal aiven user with external_identity data source + +## [1.0.0] - 2018-09-27 + +Support all Aiven resource types. Also large changes to previously +supported resource types, such as full support for all user config +options. +` + +const testSampleExpected = `# Changelog + + + + +## [MAJOR.MINOR.PATCH] - YYYY-MM-DD + +- Fix aiven_project : can't migrate from account_id + (deprecated) to parent_id +- Add aiven_organization_user_list beta resource +- Add AIVEN_ALLOW_IP_FILTER_PURGE environment variable + to allow purging of IP filters. This is a safety feature + to prevent accidental purging of IP filters, which can lead + to loss of access to services. To enable purging, + set the environment variable to any value before running + Terraform commands. +- Use TypeSet for ip_filter_object +- Deprecated account_id in aiven_project + and aiven_billing_group resources + - Please use parent_id instead, account_id is going + to be removed in the next major release +- Fix incorrect behavior of aiven_mirrormaker_replication_flow + schema fields: + - sync_group_offsets_enabled + - sync_group_offsets_interval_seconds +- Fix project creation with account_id empty + and add possibility to dissociate project from an account + by not setting account_id +- Fix typos in documentation and examples foo bar + +## [4.26.0] - 2024-09-25 + +- Remove aiven_valkey from beta resources +- Remove aiven_valkey_user from beta resources +- Adds aiven_organization_permission example +- Add capability to map external service user with internal + aiven user with external_identity data source + +## [1.0.0] - 2018-09-27 + +- Support all Aiven resource types. Also large changes + to previously supported resource types, such as full support + for all user config options. +` + +func TestFormatChangelog(t *testing.T) { + result, err := updateChangelog(testSample, 60, true, "foo", "bar", "Use TypeSet for ip_filter_object") + require.NoError(t, err) + assert.Empty(t, cmp.Diff(testSampleExpected, result)) +} + +func TestLineWrapping(t *testing.T) { + input := `Add capability to map external service user with internal aiven user with external_identity data source` + expectList := []string{ + "Add capability", + "to map external service", + "user with internal aiven", + "user with", + "external_identity data", + "source", + } + expectPoint := "- Add capability\n to map external service\n user with internal aiven\n user with\n external_identity data\n source" + assert.Equal(t, expectList, softWrap(input, 25)) + assert.Equal(t, expectPoint, addBullet("- ", expectList)) +} + +func TestChangelogFile(t *testing.T) { + b, err := os.ReadFile("../CHANGELOG.md") + require.NoError(t, err) + + result, err := updateChangelog(string(b), 80, true) + assert.NoError(t, err) + assert.NotEmpty(t, result) +} diff --git a/changelog/main.go b/changelog/main.go index 233d7cfbd..c3e145c9b 100644 --- a/changelog/main.go +++ b/changelog/main.go @@ -18,6 +18,7 @@ type flags struct { diff bool schemaFile string changelogFile string + reformat bool } func main() { @@ -66,6 +67,7 @@ func parseFlags() (*flags, error) { flag.BoolVar(&f.diff, "diff", false, "compare current schema with imported schema") flag.StringVar(&f.schemaFile, "schema", "", "schema file path (for save/diff)") flag.StringVar(&f.changelogFile, "changelog", "", "changelog output file path") + flag.BoolVar(&f.reformat, "reformat", false, "reformat the whole changelog file") flag.Parse() if f.save == f.diff { @@ -115,10 +117,10 @@ func processDiff(flags *flags, newMap ItemMap) error { } if flags.changelogFile == "" { - return printEntries(entries) + return printChangelog(entries) } - return writeChangelog(flags.changelogFile, entries) + return writeChangelog(flags.changelogFile, flags.reformat, entries) } func loadSchemaFile(path string) (ItemMap, error) { @@ -136,20 +138,25 @@ func loadSchemaFile(path string) (ItemMap, error) { return oldMap, nil } -func printEntries(entries []string) error { +func printChangelog(entries []string) error { for _, l := range entries { - fmt.Printf("- %s\n", l) + fmt.Printf("- %s\n", l) } return nil } -func writeChangelog(_ string, entries []string) error { - // todo: write to file - for _, l := range entries { - fmt.Printf("- %s\n", l) +func writeChangelog(filePath string, reformat bool, entries []string) error { + b, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read changelog file: %w", err) } - return nil + s, err := updateChangelog(string(b), defaultLineMaxLength, reformat, entries...) + if err != nil { + return fmt.Errorf("failed to format changelog: %w", err) + } + + return os.WriteFile(filePath, []byte(s), 0644) } func fromProvider(p *schema.Provider) (ItemMap, error) {