From 95ce3981ad9e65569f8b3808b02036317f1ba95f Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Fri, 8 Dec 2023 04:34:51 -0800 Subject: [PATCH 1/8] Minor bug fixes and updates --- db_params.tf | 60 ++++++++++++++++++++++++++++++++++++++++++++ main.tf | 35 +++++++++++++++++--------- update_parameters.py | 13 +++++----- 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/db_params.tf b/db_params.tf index 9df1aa5..7fadfc5 100644 --- a/db_params.tf +++ b/db_params.tf @@ -10,5 +10,65 @@ resource "aws_db_parameter_group" "education" { name = "autovacuum_vacuum_cost_delay" value = "2" } + parameter { + name = "autovacuum_vacuum_cost_limit" + value = "200" + } + parameter { + name = "autovacuum_vacuum_scale_factor" + value = "0.1" + } + parameter { + name = "autovacuum_vacuum_threshold" + value = "50" + } + parameter { + name = "bgwriter_delay" + value = "200" + } + parameter { + name = "bgwriter_lru_maxpages" + value = "100" + } + parameter { + name = "bgwriter_lru_multiplier" + value = "2" + } + parameter { + name = "checkpoint_completion_target" + value = "0.9" + } + parameter { + name = "default_statistics_target" + value = "100" + } + parameter { + name = "effective_cache_size" + value = "47626" + } + parameter { + name = "effective_io_concurrency" + value = "1" + } + parameter { + name = "max_wal_size" + value = "2048" + } + parameter { + name = "random_page_cost" + value = "4" + } + parameter { + name = "temp_buffers" + value = "1024" + } + parameter { + name = "wal_writer_delay" + value = "200" + } + parameter { + name = "work_mem" + value = "4096" + } } diff --git a/main.tf b/main.tf index 8a2c411..3075bc3 100644 --- a/main.tf +++ b/main.tf @@ -1,14 +1,25 @@ +provider "aws" { + region = "us-east-2" # replace with your desired region + profile = "external" +} + +# export TF_VAR_db_password="your_secure_password" +variable "db_password" {} + resource "aws_db_instance" "education" { - identifier = "education" - instance_class = "db.t3.micro" - allocated_storage = 5 - engine = "postgres" - engine_version = "14.1" - username = "edu" - password = var.db_password - db_subnet_group_name = aws_db_subnet_group.education.name - vpc_security_group_ids = [aws_security_group.rds.id] - parameter_group_name = aws_db_parameter_group.education.name - publicly_accessible = true - skip_final_snapshot = true + identifier = "education" + instance_class = "db.t3.micro" + allocated_storage = "5" + engine = "postgres" + engine_version = "14" + username = "edu" + password = var.db_password + #db_subnet_group_name = aws_db_subnet_group.education.name + #vpc_security_group_ids = [aws_security_group.rds.id] + parameter_group_name = aws_db_parameter_group.education.name + publicly_accessible = true + skip_final_snapshot = true + backup_retention_period = "0" + multi_az = false + performance_insights_enabled = true } diff --git a/update_parameters.py b/update_parameters.py index eef956c..6005e49 100644 --- a/update_parameters.py +++ b/update_parameters.py @@ -2,10 +2,10 @@ import os # Replace with your actual OtterTune API key -ottertune_api_key = os.environ['OT-API-KEY'] +ottertune_api_key = os.environ['OT_API_KEY'] # Replace with the desired database identifier -db_identifier = 'YOUR_DB_IDENTIFIER' +db_identifier = 'education' # Base URL for the OtterTune API base_url = 'https://service.ottertune.com/api' @@ -63,10 +63,11 @@ param_name = None param_value = None while not line.strip().startswith("}"): - if line.strip().startswith("name = "): - param_name = line.strip().split(" = ")[1].strip('""') - elif line.strip().startswith("value = "): - param_value = line.strip().split(" = ")[1].strip('""') + if line.strip().startswith("name"): + param_name = line.split("=")[1].strip('" ') + elif line.strip().startswith("value"): + param_value = line.split("=")[1].strip('" ') + assert param_name is not None line = config_lines[i] i += 1 From 2fa2a9594042b9b20d5620b137e3a05cc867b0a5 Mon Sep 17 00:00:00 2001 From: Dana Van Aken Date: Fri, 8 Dec 2023 14:02:46 -0800 Subject: [PATCH 2/8] Updates: - main.tf: Added some comments and updated a few options - db_params.tf: Added all recommended knobs for PostgreSQL v15 - update_parameters.py: modified parsing of `name` and `value` parameter fields to be a bit more flexible, return non-zero error code on exception, update DB_INSTANCE so it's also assigned to an env like OT_API_KEY - .github/workflows/daily_update.yml: added new envs, install requirements --- .github/workflows/daily_update.yml | 17 +++++++-- .gitignore | 40 +++++++++++++++++++++ db_params.tf | 21 ++++++----- main.tf | 57 ++++++++++++++++++++---------- update_parameters.py | 36 +++++++++++++------ 5 files changed, 128 insertions(+), 43 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index bed3e8c..01a681c 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -17,29 +17,40 @@ jobs: with: python-version: 3.x + - name: Install Python Packages + run: | + pip install -r requirements.txt + - name: Run Python script run: | python update_parameters.py + env: + DB_IDENTIFIER: "terraform-blog-example" + OT_API_KEY: ${{ secrets.OT_API_KEY }} - name: Commit changes run: | + git status git config user.email "you@example.com" git config user.name "Your Name" git add db_params.tf git commit -m "AI knob updates" git push env: - GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN}} + GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token - name: Set up Terraform uses: hashicorp/setup-terraform@v1 with: - terraform_version: 1.0.8 # Replace with your desired Terraform version + terraform_version: 1.6.5 # Replace with your desired Terraform version - name: Initialize Terraform run: terraform init - name: Apply Terraform changes # CAUTION: -auto-approve may override changes to the infra - # that are not saved in the terraform files + # that are not saved in the terraform files run: terraform apply -auto-approve + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..709ca34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +.terraform.lock.hcl + + +# Ignore python venv +env diff --git a/db_params.tf b/db_params.tf index 7fadfc5..3e3cf38 100644 --- a/db_params.tf +++ b/db_params.tf @@ -1,19 +1,18 @@ -resource "aws_db_parameter_group" "education" { - name = "education" - family = "postgres14" +# AWS RDS PostgreSQL Parameter Group Configuration + +resource "aws_db_parameter_group" "terraform_blog_example" { + name = "terraform-blog-example-pg" # Name of the DB parameter group + family = "postgres15" # Database family (PostgreSQL 15.x in this case) + + # Below are OtterTune's recommended knobs for tuning RDS PostgreSQL 15 performance. + # They are initially set to their default values when the RDS instance is created. + # After completing the steps in the blog article, these DB parameter group settings + # will be updated daily with OtterTune's latest knob recommendations. - parameter { - name = "log_connections" - value = "1" - } parameter { name = "autovacuum_vacuum_cost_delay" value = "2" } - parameter { - name = "autovacuum_vacuum_cost_limit" - value = "200" - } parameter { name = "autovacuum_vacuum_scale_factor" value = "0.1" diff --git a/main.tf b/main.tf index 3075bc3..b5a2e7d 100644 --- a/main.tf +++ b/main.tf @@ -1,25 +1,44 @@ +# Note: This file is for tutorial purposes. Adjust the options for production use cases, especially for security and reliability. + +# Provider information provider "aws" { - region = "us-east-2" # replace with your desired region - profile = "external" + region = "us-east-2" # replace with your desired region } -# export TF_VAR_db_password="your_secure_password" + +# Set environment variable `TF_VAR_db_password="your_secure_password"` variable "db_password" {} -resource "aws_db_instance" "education" { - identifier = "education" - instance_class = "db.t3.micro" - allocated_storage = "5" - engine = "postgres" - engine_version = "14" - username = "edu" - password = var.db_password - #db_subnet_group_name = aws_db_subnet_group.education.name - #vpc_security_group_ids = [aws_security_group.rds.id] - parameter_group_name = aws_db_parameter_group.education.name - publicly_accessible = true - skip_final_snapshot = true - backup_retention_period = "0" - multi_az = false - performance_insights_enabled = true + +# AWS RDS PostgreSQL instance creation using Terraform +resource "aws_db_instance" "terraform_blog_example" { + + # Instance specifications + + identifier = "terraform-blog-example" # The unique identifier for the RDS instance + engine = "postgres" # The database engine + engine_version = "15" # The version of the database engine + instance_class = "db.t3.micro" # Determines the machine type used for the DB instance + allocated_storage = "10" # The size of the storage (in GB) allocated to the DB instance + username = "postgres" # Master username for the DB instance + password = var.db_password # Master password for the DB instance + parameter_group_name = aws_db_parameter_group.terraform_blog_example.name # The DB parameter group to associate with the DB instance + performance_insights_enabled = true # Enables RDS Performance Insights for the DB instance + + # Tutorial specific settings (not recommended in production environments) + + publicly_accessible = true # Whether the DB instance is accessible from the internet + backup_retention_period = "0" # The number of days to retain backups ('0' disables backups) + skip_final_snapshot = true # Whether to create final snapshot before the DB instance is deleted + multi_az = false # Whether to replicate the DB instance across multiple AZs for high availability + + # Optional settings + + # Uncomment and set these if you are using a custom VPC or need specific network settings + # db_subnet_group_name = "your_db_subnet_group" # The DB subnet group name to associate with this instance + # vpc_security_group_ids = ["sg-xxxxxxxx"] # List of VPC security group IDs to associate with this instance + + # Add additional settings here as necessary } + + diff --git a/update_parameters.py b/update_parameters.py index 6005e49..8e15471 100644 --- a/update_parameters.py +++ b/update_parameters.py @@ -1,11 +1,15 @@ import requests +import json import os +import re +import sys -# Replace with your actual OtterTune API key + +# Your OtterTune API key ottertune_api_key = os.environ['OT_API_KEY'] -# Replace with the desired database identifier -db_identifier = 'education' +# Your database identifier +db_identifier = os.environ['DB_IDENTIFIER'] # Base URL for the OtterTune API base_url = 'https://service.ottertune.com/api' @@ -17,6 +21,10 @@ # Replace with your config file name terraform_file_path = 'db_params.tf' +# Regex patterns to match parameter 'name' and 'value' lines +name_pattern = re.compile(r'name\s*=\s*"?([^"\n]+)"?') +value_pattern = re.compile(r'value\s*=\s*"?([^"\n]+)"?') + try: response = requests.get(databases_url, headers=headers) response.raise_for_status() @@ -45,6 +53,8 @@ knob_final_value = recommendation['knobFinalValue'] knob_recommendations[knob_name] = knob_final_value + print(f"KNOB_RECOMMENDATIONS = {json.dumps(knob_recommendations, indent=4, default=str)}\n") + if len(knob_recommendations) > 0: with open(terraform_file_path, 'r') as file: terraform_config = file.read() @@ -63,11 +73,14 @@ param_name = None param_value = None while not line.strip().startswith("}"): - if line.strip().startswith("name"): - param_name = line.split("=")[1].strip('" ') - elif line.strip().startswith("value"): - param_value = line.split("=")[1].strip('" ') - assert param_name is not None + name_match = name_pattern.search(line) + if name_match: + param_name = name_match.group(1) + + value_match = value_pattern.search(line) + if value_match: + param_value = value_match.group(1) + line = config_lines[i] i += 1 @@ -92,13 +105,16 @@ with open(terraform_file_path, 'w') as file: file.write(modified_terraform_config) - print("Terraform configuration file (main.tf) has been updated with knob recommendations.") + print(f"Terraform configuration file ({terraform_file_path}) has been updated with knob recommendations.") else: print("No knob recommendations found for this database.") else: - print(f'Database with dbIdentifier "{db_identifier}" not found.') + print(f'Error: Database with dbIdentifier "{db_identifier}" not found.') + sys.exit(1) except requests.exceptions.RequestException as e: print(f'Error making HTTP request: {e}') + sys.exit(1) except Exception as e: print(f'An error occurred: {e}') + sys.exit(1) From 473449a0a41656742734a2dd90c45f2241439f55 Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:23:03 -0500 Subject: [PATCH 3/8] actions fix --- .github/workflows/daily_update.yml | 2 ++ main.tf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index 01a681c..fd55bff 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -53,4 +53,6 @@ jobs: run: terraform apply -auto-approve env: TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/main.tf b/main.tf index b5a2e7d..752a9c6 100644 --- a/main.tf +++ b/main.tf @@ -29,7 +29,7 @@ resource "aws_db_instance" "terraform_blog_example" { publicly_accessible = true # Whether the DB instance is accessible from the internet backup_retention_period = "0" # The number of days to retain backups ('0' disables backups) - skip_final_snapshot = true # Whether to create final snapshot before the DB instance is deleted + skip_final_snapshot = true # Whether to skip creation of final snapshot before the DB instance is deleted multi_az = false # Whether to replicate the DB instance across multiple AZs for high availability # Optional settings From 6ff7616a6eeedbadef731a44d77f1b329935ae32 Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:26:15 -0500 Subject: [PATCH 4/8] temporarily increase run frequency --- .github/workflows/daily_update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index fd55bff..27131ae 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -2,7 +2,7 @@ name: Daily Parameter Update on: schedule: - - cron: '0 0 * * *' # Schedule the workflow to run daily at midnight UTC + - cron: '*/1 * * * *' # Schedule the workflow to run daily at midnight UTC jobs: update_and_commit: From e375c74237053ecc92c06c8689f234b465c2ad3f Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:30:36 -0500 Subject: [PATCH 5/8] add manual trigger --- .github/workflows/daily_update.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index 27131ae..151a6df 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -3,6 +3,7 @@ name: Daily Parameter Update on: schedule: - cron: '*/1 * * * *' # Schedule the workflow to run daily at midnight UTC + workflow_dispatch: jobs: update_and_commit: From 38d9e453abc86b1f0cff974c793d0dc7ef021e69 Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:31:26 -0500 Subject: [PATCH 6/8] add manual trigger --- .github/workflows/daily_update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index 151a6df..11218ef 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -2,7 +2,7 @@ name: Daily Parameter Update on: schedule: - - cron: '*/1 * * * *' # Schedule the workflow to run daily at midnight UTC + - cron: '0 0 * * *' # Schedule the workflow to run daily at midnight UTC workflow_dispatch: jobs: From 3afcc0eae9c779f7e14f7234cab7ba461db58b47 Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:35:52 -0500 Subject: [PATCH 7/8] commit terraform changes after they are applied --- .github/workflows/daily_update.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index 11218ef..aa09caf 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -29,17 +29,6 @@ jobs: DB_IDENTIFIER: "terraform-blog-example" OT_API_KEY: ${{ secrets.OT_API_KEY }} - - name: Commit changes - run: | - git status - git config user.email "you@example.com" - git config user.name "Your Name" - git add db_params.tf - git commit -m "AI knob updates" - git push - env: - GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token - - name: Set up Terraform uses: hashicorp/setup-terraform@v1 with: @@ -57,3 +46,14 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Commit changes + run: | + git status + git config user.email "priyankbhandia@gmail.com" + git config user.name "priyank96" + git add db_params.tf + git commit -m "AI knob updates" + git push + env: + GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token + From 7321ec3c2e7eec5416223d893da38330a01593c4 Mon Sep 17 00:00:00 2001 From: priyank96 Date: Mon, 11 Dec 2023 19:39:45 -0500 Subject: [PATCH 8/8] fix invalid yaml --- .github/workflows/daily_update.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index aa09caf..8afe6e5 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -47,13 +47,13 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - name: Commit changes - run: | - git status - git config user.email "priyankbhandia@gmail.com" - git config user.name "priyank96" - git add db_params.tf - git commit -m "AI knob updates" - git push - env: - GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token + run: | + git status + git config user.email "priyankbhandia@gmail.com" + git config user.name "priyank96" + git add db_params.tf + git commit -m "AI knob updates" + git push + env: + GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token