feat(kafka_quota): added support for kafka quota

aiven · Dec 27, 2024 · 7414b84 · 7414b84
1 parent 0f2962e
commit 7414b84
Show file tree

Hide file tree

Showing 25 changed files with 1,579 additions and 19 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,10 @@ __debug_bin
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Go workspace file
+go.work
+go.work.sum
+
 # Dependency directories
 vendor/
 packrd/

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,16 @@ nav_order: 1
 <!-- Always keep the following header in place: -->
 <!--## [MAJOR.MINOR.PATCH] - YYYY-MM-DD -->
 
+
+## [MAJOR.MINOR.PATCH] - YYYY-MM-DD
+
+- Add `aiven_kafka_quota` resource
+- Add `aiven_opensearch` resource and datasource field
+  `opensearch_user_config.opensearch.cluster_routing_allocation_balance_prefer_primary`: When set to true, OpenSearch
+  attempts to evenly distribute the primary shards between the cluster nodes
+- Add `aiven_opensearch` resource and datasource field `opensearch_user_config.opensearch.segrep`: Segment Replication
+  Backpressure Settings
+
 ## [4.31.1] - 2024-12-23
 
 - Validate whether the `aiven_project.billing_group` field has changed before calling admin API

diff --git a/Makefile b/Makefile
@@ -112,10 +112,12 @@ lint: lint-go lint-test lint-docs
 lint-go: $(GOLANGCILINT)
 	$(GOLANGCILINT) run --build-tags all --timeout=30m ./...
 
+# Exclude files that use templates from linting
+TERRAFMT_EXCLUDE = -not -path "./internal/acctest/*" \
+	-not -path "./internal/sdkprovider/service/kafka/kafka_quota_test.go"
 
 lint-test: $(TERRAFMT)
-	$(TERRAFMT) diff ./internal -cfq
-
+	find ./internal -type f $(TERRAFMT_EXCLUDE) -exec $(TERRAFMT) diff {} -cfq \;
 
 lint-docs: $(TFPLUGINDOCS)
 	PROVIDER_AIVEN_ENABLE_BETA=1 $(TFPLUGINDOCS) generate --rendered-website-dir tmp
@@ -132,7 +134,6 @@ lint-docs: $(TFPLUGINDOCS)
 
 fmt: fmt-test fmt-imports
 
-
 fmt-test: $(TERRAFMT)
 	$(TERRAFMT) fmt ./internal -fv
 

diff --git a/docs/data-sources/account_team_project.md b/docs/data-sources/account_team_project.md
@@ -32,4 +32,4 @@ data "aiven_account_team_project" "account_team_project1" {
 ### Read-Only
 
 - `id` (String) The ID of this resource.
-- `team_type` (String) The Account team project type. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
+- `team_type` (String) The Account team project type. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:projects:write`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
diff --git a/docs/data-sources/opensearch.md b/docs/data-sources/opensearch.md
@@ -221,6 +221,7 @@ Read-Only:
 - `action_destructive_requires_name` (Boolean)
 - `auth_failure_listeners` (List of Object) (see [below for nested schema](#nestedobjatt--opensearch_user_config--opensearch--auth_failure_listeners))
 - `cluster_max_shards_per_node` (Number)
+- `cluster_routing_allocation_balance_prefer_primary` (Boolean)
 - `cluster_routing_allocation_node_concurrent_recoveries` (Number)
 - `email_sender_name` (String)
 - `email_sender_password` (String)
@@ -252,6 +253,7 @@ Read-Only:
 - `search_backpressure` (List of Object) (see [below for nested schema](#nestedobjatt--opensearch_user_config--opensearch--search_backpressure))
 - `search_insights_top_queries` (List of Object) (see [below for nested schema](#nestedobjatt--opensearch_user_config--opensearch--search_insights_top_queries))
 - `search_max_buckets` (Number)
+- `segrep` (List of Object) (see [below for nested schema](#nestedobjatt--opensearch_user_config--opensearch--segrep))
 - `shard_indexing_pressure` (List of Object) (see [below for nested schema](#nestedobjatt--opensearch_user_config--opensearch--shard_indexing_pressure))
 - `thread_pool_analyze_queue_size` (Number)
 - `thread_pool_analyze_size` (Number)
@@ -394,6 +396,17 @@ Read-Only:
 
 
 
+<a id="nestedobjatt--opensearch_user_config--opensearch--segrep"></a>
+### Nested Schema for `opensearch_user_config.opensearch.segrep`
+
+Read-Only:
+
+- `pressure_checkpoint_limit` (Number)
+- `pressure_enabled` (Boolean)
+- `pressure_replica_stale_limit` (Number)
+- `pressure_time_limit` (String)
+
+
 <a id="nestedobjatt--opensearch_user_config--opensearch--shard_indexing_pressure"></a>
 ### Nested Schema for `opensearch_user_config.opensearch.shard_indexing_pressure`
 

diff --git a/docs/data-sources/project_user.md b/docs/data-sources/project_user.md
@@ -31,4 +31,4 @@ data "aiven_project_user" "mytestuser" {
 
 - `accepted` (Boolean) Whether the user has accepted the request to join the project. Users get an invite and become project members after accepting the invite.
 - `id` (String) The ID of this resource.
-- `member_type` (String) Project membership type. The possible values are `admin`, `developer`, `operator`, `organization:app_users:write`, `organization:audit_logs:read`, `organization:domains:write`, `organization:groups:write`, `organization:idps:write`, `organization:users:write`, `project:audit_logs:read`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `project:services:read`, `project:services:write`, `read_only`, `role:organization:admin`, `role:services:maintenance`, `role:services:recover`, `service:configuration:write`, `service:data:write`, `service:logs:read`, `service:secrets:read` and `service:users:write`.
+- `member_type` (String) Project membership type. The possible values are `admin`, `developer`, `operator`, `organization:app_users:write`, `organization:audit_logs:read`, `organization:domains:write`, `organization:groups:write`, `organization:idps:write`, `organization:projects:write`, `organization:users:write`, `project:audit_logs:read`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `project:services:read`, `project:services:write`, `read_only`, `role:organization:admin`, `role:services:maintenance`, `role:services:recover`, `service:configuration:write`, `service:data:write`, `service:logs:read`, `service:secrets:read` and `service:users:write`.
diff --git a/docs/resources/account_team_project.md b/docs/resources/account_team_project.md
@@ -48,7 +48,7 @@ resource "aiven_account_team_project" "main" {
 ### Optional
 
 - `project_name` (String) The name of an already existing project
-- `team_type` (String) The Account team project type. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
+- `team_type` (String) The Account team project type. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:projects:write`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
 - `timeouts` (Block, Optional) (see [below for nested schema](#nestedblock--timeouts))
 
 ### Read-Only

diff --git a/docs/resources/kafka_quota.md b/docs/resources/kafka_quota.md
@@ -0,0 +1,84 @@
+---
+# generated by https://github.com/hashicorp/terraform-plugin-docs
+page_title: "aiven_kafka_quota Resource - terraform-provider-aiven"
+subcategory: ""
+description: |-
+  Creates and manages quotas for an Aiven for Apache Kafka® service user.
+---
+
+# aiven_kafka_quota (Resource)
+
+Creates and manages quotas for an Aiven for Apache Kafka® service user.
+
+## Example Usage
+
+```terraform
+resource "aiven_kafka_quota" "example_quota" {
+  project             = data.aiven_project.foo.project
+  service_name        = aiven_kafka.example_kafka.service_name
+  user                = "example-kafka-user"
+  client_id           = "example_client"
+  consumer_byte_rate  = 1000
+  producer_byte_rate  = 1000
+  request_percentage  = 50
+}
+```
+
+<!-- schema generated by tfplugindocs -->
+## Schema
+
+### Required
+
+- `project` (String) The name of the project this resource belongs to. To set up proper dependencies please refer to this variable as a reference. Changing this property forces recreation of the resource.
+- `service_name` (String) The name of the service that this resource belongs to. To set up proper dependencies please refer to this variable as a reference. Changing this property forces recreation of the resource.
+
+### Optional
+
+- `client_id` (String) Represents a logical group of clients, assigned a unique name by the client application.
+Quotas can be applied based on user, client-id, or both.
+The most relevant quota is chosen for each connection.
+All connections within a quota group share the same quota.
+It is possible to set default quotas for each (user, client-id), user or client-id group by specifying 'default'
+- `consumer_byte_rate` (Number) Defines the bandwidth limit in bytes/sec for each group of clients sharing a quota.
+Every distinct client group is allocated a specific quota, as defined by the cluster, on a per-broker basis.
+Exceeding this limit results in client throttling.
+- `producer_byte_rate` (Number) Defines the bandwidth limit in bytes/sec for each group of clients sharing a quota.
+Every distinct client group is allocated a specific quota, as defined by the cluster, on a per-broker basis.
+Exceeding this limit results in client throttling.
+- `request_percentage` (Number) Sets the maximum percentage of CPU time that a client group can use on request handler I/O and network threads per broker within a quota window.
+Exceeding this limit triggers throttling.
+The quota, expressed as a percentage, also indicates the total allowable CPU usage for the client groups sharing the quota.
+- `timeouts` (Block, Optional) (see [below for nested schema](#nestedblock--timeouts))
+- `user` (String) Represents a logical group of clients, assigned a unique name by the client application.
+Quotas can be applied based on user, client-id, or both.
+The most relevant quota is chosen for each connection.
+All connections within a quota group share the same quota.
+It is possible to set default quotas for each (user, client-id), user or client-id group by specifying 'default'
+
+### Read-Only
+
+- `id` (String) The ID of this resource.
+
+<a id="nestedblock--timeouts"></a>
+### Nested Schema for `timeouts`
+
+Optional:
+
+- `create` (String)
+- `default` (String)
+- `delete` (String)
+- `read` (String)
+- `update` (String)
+
+## Import
+
+Import is supported using the following syntax:
+
+```shell
+# When both USER and CLIENT_ID are specified
+terraform import aiven_kafka_quota.example_quota PROJECT/SERVICE_NAME/CLIENT_ID/USER
+# When only USER is specified
+terraform import aiven_kafka_quota.example_quota PROJECT/SERVICE_NAME//USER
+# When only CLIENT_ID is specified
+terraform import aiven_kafka_quota.example_quota PROJECT/SERVICE_NAME/CLIENT_ID/
+```
diff --git a/docs/resources/opensearch.md b/docs/resources/opensearch.md
@@ -246,6 +246,7 @@ Optional:
 - `action_destructive_requires_name` (Boolean) Require explicit index names when deleting.
 - `auth_failure_listeners` (Block List, Max: 1) Opensearch Security Plugin Settings (see [below for nested schema](#nestedblock--opensearch_user_config--opensearch--auth_failure_listeners))
 - `cluster_max_shards_per_node` (Number) Controls the number of shards allowed in the cluster per data node. Example: `1000`.
+- `cluster_routing_allocation_balance_prefer_primary` (Boolean) When set to true, OpenSearch attempts to evenly distribute the primary shards between the cluster nodes. Enabling this setting does not always guarantee an equal number of primary shards on each node, especially in the event of a failover. Changing this setting to false after it was set to true does not invoke redistribution of primary shards. Default is false. Default: `false`.
 - `cluster_routing_allocation_node_concurrent_recoveries` (Number) How many concurrent incoming/outgoing shard recoveries (normally replicas) are allowed to happen on a node. Defaults to node cpu count * 2.
 - `email_sender_name` (String) Sender name placeholder to be used in Opensearch Dashboards and Opensearch keystore. Example: `alert-sender`.
 - `email_sender_password` (String, Sensitive) Sender password for Opensearch alerts to authenticate with SMTP server. Example: `very-secure-mail-password`.
@@ -277,6 +278,7 @@ Optional:
 - `search_backpressure` (Block List, Max: 1) Search Backpressure Settings (see [below for nested schema](#nestedblock--opensearch_user_config--opensearch--search_backpressure))
 - `search_insights_top_queries` (Block List, Max: 1) (see [below for nested schema](#nestedblock--opensearch_user_config--opensearch--search_insights_top_queries))
 - `search_max_buckets` (Number) Maximum number of aggregation buckets allowed in a single response. OpenSearch default value is used when this is not defined. Example: `10000`.
+- `segrep` (Block List, Max: 1) Segment Replication Backpressure Settings (see [below for nested schema](#nestedblock--opensearch_user_config--opensearch--segrep))
 - `shard_indexing_pressure` (Block List, Max: 1) Shard indexing back pressure settings (see [below for nested schema](#nestedblock--opensearch_user_config--opensearch--shard_indexing_pressure))
 - `thread_pool_analyze_queue_size` (Number) Size for the thread pool queue. See documentation for exact details.
 - `thread_pool_analyze_size` (Number) Size for the thread pool. See documentation for exact details. Do note this may have maximum value depending on CPU count - value is automatically lowered if set to higher than maximum value.
@@ -419,6 +421,17 @@ Optional:
 
 
 
+<a id="nestedblock--opensearch_user_config--opensearch--segrep"></a>
+### Nested Schema for `opensearch_user_config.opensearch.segrep`
+
+Optional:
+
+- `pressure_checkpoint_limit` (Number) The maximum number of indexing checkpoints that a replica shard can fall behind when copying from primary. Once `segrep.pressure.checkpoint.limit` is breached along with `segrep.pressure.time.limit`, the segment replication backpressure mechanism is initiated. Default is 4 checkpoints. Default: `4`.
+- `pressure_enabled` (Boolean) Enables the segment replication backpressure mechanism. Default is false. Default: `false`.
+- `pressure_replica_stale_limit` (Number) The maximum number of stale replica shards that can exist in a replication group. Once `segrep.pressure.replica.stale.limit` is breached, the segment replication backpressure mechanism is initiated. Default is .5, which is 50% of a replication group. Default: `0.5`.
+- `pressure_time_limit` (String) The maximum amount of time that a replica shard can take to copy from the primary shard. Once segrep.pressure.time.limit is breached along with segrep.pressure.checkpoint.limit, the segment replication backpressure mechanism is initiated. Default is 5 minutes. Default: `5m`.
+
+
 <a id="nestedblock--opensearch_user_config--opensearch--shard_indexing_pressure"></a>
 ### Nested Schema for `opensearch_user_config.opensearch.shard_indexing_pressure`
 

diff --git a/docs/resources/organization_group_project.md b/docs/resources/organization_group_project.md
@@ -51,7 +51,7 @@ resource "aiven_organization_group_project" "example" {
 
 - `group_id` (String) The ID of the user group.
 - `project` (String) The project that the users in the group are members of.
-- `role` (String) [Project-level role](https://aiven.io/docs/platform/reference/project-member-privileges) assigned to all users in the group. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
+- `role` (String) [Project-level role](https://aiven.io/docs/platform/reference/project-member-privileges) assigned to all users in the group. The possible values are `admin`, `operator`, `developer`, `read_only`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `service:configuration:write`, `service:logs:read`, `project:services:read`, `project:services:write`, `project:audit_logs:read`, `service:data:write`, `service:secrets:read`, `service:users:write`, `role:services:maintenance`, `role:services:recover`, `organization:audit_logs:read`, `organization:projects:write`, `organization:users:write`, `organization:app_users:write`, `organization:groups:write`, `organization:idps:write`, `organization:domains:write` and `role:organization:admin`.
 
 ### Optional
 

diff --git a/docs/resources/organization_permission.md b/docs/resources/organization_permission.md
@@ -95,7 +95,7 @@ resource "aiven_organization_permission" "example_org_permissions" {
 
 Required:
 
-- `permissions` (Set of String) List of [roles and permissions](https://aiven.io/docs/platform/concepts/permissions) to grant. The possible values are `admin`, `developer`, `operator`, `organization:app_users:write`, `organization:audit_logs:read`, `organization:domains:write`, `organization:groups:write`, `organization:idps:write`, `organization:users:write`, `project:audit_logs:read`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `project:services:read`, `project:services:write`, `read_only`, `role:organization:admin`, `role:services:maintenance`, `role:services:recover`, `service:configuration:write`, `service:data:write`, `service:logs:read`, `service:secrets:read` and `service:users:write`.
+- `permissions` (Set of String) List of [roles and permissions](https://aiven.io/docs/platform/concepts/permissions) to grant. The possible values are `admin`, `developer`, `operator`, `organization:app_users:write`, `organization:audit_logs:read`, `organization:domains:write`, `organization:groups:write`, `organization:idps:write`, `organization:projects:write`, `organization:users:write`, `project:audit_logs:read`, `project:integrations:read`, `project:integrations:write`, `project:networking:read`, `project:networking:write`, `project:permissions:read`, `project:services:read`, `project:services:write`, `read_only`, `role:organization:admin`, `role:services:maintenance`, `role:services:recover`, `service:configuration:write`, `service:data:write`, `service:logs:read`, `service:secrets:read` and `service:users:write`.
 - `principal_id` (String) ID of the user or group to grant permissions to. Only active users who have accepted an [invite](https://aiven.io/docs/platform/howto/manage-org-users) to join the organization can be granted permissions.
 - `principal_type` (String) The type of principal. The possible values are `user` and `user_group`.