diff --git a/.github/workflows/daily_update.yml b/.github/workflows/daily_update.yml index bed3e8c..8afe6e5 100644 --- a/.github/workflows/daily_update.yml +++ b/.github/workflows/daily_update.yml @@ -3,6 +3,7 @@ name: Daily Parameter Update on: schedule: - cron: '0 0 * * *' # Schedule the workflow to run daily at midnight UTC + workflow_dispatch: jobs: update_and_commit: @@ -17,29 +18,42 @@ jobs: with: python-version: 3.x - - name: Run Python script + - name: Install Python Packages run: | - python update_parameters.py + pip install -r requirements.txt - - name: Commit changes + - name: Run Python script run: | - git config user.email "you@example.com" - git config user.name "Your Name" - git add db_params.tf - git commit -m "AI knob updates" - git push + python update_parameters.py env: - GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN}} + DB_IDENTIFIER: "terraform-blog-example" + OT_API_KEY: ${{ secrets.OT_API_KEY }} - name: Set up Terraform uses: hashicorp/setup-terraform@v1 with: - terraform_version: 1.0.8 # Replace with your desired Terraform version + terraform_version: 1.6.5 # Replace with your desired Terraform version - name: Initialize Terraform run: terraform init - name: Apply Terraform changes # CAUTION: -auto-approve may override changes to the infra - # that are not saved in the terraform files + # that are not saved in the terraform files run: terraform apply -auto-approve + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Commit changes + run: | + git status + git config user.email "priyankbhandia@gmail.com" + git config user.name "priyank96" + git add db_params.tf + git commit -m "AI knob updates" + git push + env: + GITHUB_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} # GitHub access token + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..709ca34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +.terraform.lock.hcl + + +# Ignore python venv +env diff --git a/db_params.tf b/db_params.tf index 9df1aa5..3e3cf38 100644 --- a/db_params.tf +++ b/db_params.tf @@ -1,14 +1,73 @@ -resource "aws_db_parameter_group" "education" { - name = "education" - family = "postgres14" +# AWS RDS PostgreSQL Parameter Group Configuration + +resource "aws_db_parameter_group" "terraform_blog_example" { + name = "terraform-blog-example-pg" # Name of the DB parameter group + family = "postgres15" # Database family (PostgreSQL 15.x in this case) + + # Below are OtterTune's recommended knobs for tuning RDS PostgreSQL 15 performance. + # They are initially set to their default values when the RDS instance is created. + # After completing the steps in the blog article, these DB parameter group settings + # will be updated daily with OtterTune's latest knob recommendations. parameter { - name = "log_connections" - value = "1" + name = "autovacuum_vacuum_cost_delay" + value = "2" } parameter { - name = "autovacuum_vacuum_cost_delay" + name = "autovacuum_vacuum_scale_factor" + value = "0.1" + } + parameter { + name = "autovacuum_vacuum_threshold" + value = "50" + } + parameter { + name = "bgwriter_delay" + value = "200" + } + parameter { + name = "bgwriter_lru_maxpages" + value = "100" + } + parameter { + name = "bgwriter_lru_multiplier" value = "2" } + parameter { + name = "checkpoint_completion_target" + value = "0.9" + } + parameter { + name = "default_statistics_target" + value = "100" + } + parameter { + name = "effective_cache_size" + value = "47626" + } + parameter { + name = "effective_io_concurrency" + value = "1" + } + parameter { + name = "max_wal_size" + value = "2048" + } + parameter { + name = "random_page_cost" + value = "4" + } + parameter { + name = "temp_buffers" + value = "1024" + } + parameter { + name = "wal_writer_delay" + value = "200" + } + parameter { + name = "work_mem" + value = "4096" + } } diff --git a/main.tf b/main.tf index 8a2c411..752a9c6 100644 --- a/main.tf +++ b/main.tf @@ -1,14 +1,44 @@ -resource "aws_db_instance" "education" { - identifier = "education" - instance_class = "db.t3.micro" - allocated_storage = 5 - engine = "postgres" - engine_version = "14.1" - username = "edu" - password = var.db_password - db_subnet_group_name = aws_db_subnet_group.education.name - vpc_security_group_ids = [aws_security_group.rds.id] - parameter_group_name = aws_db_parameter_group.education.name - publicly_accessible = true - skip_final_snapshot = true +# Note: This file is for tutorial purposes. Adjust the options for production use cases, especially for security and reliability. + +# Provider information +provider "aws" { + region = "us-east-2" # replace with your desired region } + + +# Set environment variable `TF_VAR_db_password="your_secure_password"` +variable "db_password" {} + + +# AWS RDS PostgreSQL instance creation using Terraform +resource "aws_db_instance" "terraform_blog_example" { + + # Instance specifications + + identifier = "terraform-blog-example" # The unique identifier for the RDS instance + engine = "postgres" # The database engine + engine_version = "15" # The version of the database engine + instance_class = "db.t3.micro" # Determines the machine type used for the DB instance + allocated_storage = "10" # The size of the storage (in GB) allocated to the DB instance + username = "postgres" # Master username for the DB instance + password = var.db_password # Master password for the DB instance + parameter_group_name = aws_db_parameter_group.terraform_blog_example.name # The DB parameter group to associate with the DB instance + performance_insights_enabled = true # Enables RDS Performance Insights for the DB instance + + # Tutorial specific settings (not recommended in production environments) + + publicly_accessible = true # Whether the DB instance is accessible from the internet + backup_retention_period = "0" # The number of days to retain backups ('0' disables backups) + skip_final_snapshot = true # Whether to skip creation of final snapshot before the DB instance is deleted + multi_az = false # Whether to replicate the DB instance across multiple AZs for high availability + + # Optional settings + + # Uncomment and set these if you are using a custom VPC or need specific network settings + # db_subnet_group_name = "your_db_subnet_group" # The DB subnet group name to associate with this instance + # vpc_security_group_ids = ["sg-xxxxxxxx"] # List of VPC security group IDs to associate with this instance + + # Add additional settings here as necessary +} + + diff --git a/update_parameters.py b/update_parameters.py index eef956c..8e15471 100644 --- a/update_parameters.py +++ b/update_parameters.py @@ -1,11 +1,15 @@ import requests +import json import os +import re +import sys -# Replace with your actual OtterTune API key -ottertune_api_key = os.environ['OT-API-KEY'] -# Replace with the desired database identifier -db_identifier = 'YOUR_DB_IDENTIFIER' +# Your OtterTune API key +ottertune_api_key = os.environ['OT_API_KEY'] + +# Your database identifier +db_identifier = os.environ['DB_IDENTIFIER'] # Base URL for the OtterTune API base_url = 'https://service.ottertune.com/api' @@ -17,6 +21,10 @@ # Replace with your config file name terraform_file_path = 'db_params.tf' +# Regex patterns to match parameter 'name' and 'value' lines +name_pattern = re.compile(r'name\s*=\s*"?([^"\n]+)"?') +value_pattern = re.compile(r'value\s*=\s*"?([^"\n]+)"?') + try: response = requests.get(databases_url, headers=headers) response.raise_for_status() @@ -45,6 +53,8 @@ knob_final_value = recommendation['knobFinalValue'] knob_recommendations[knob_name] = knob_final_value + print(f"KNOB_RECOMMENDATIONS = {json.dumps(knob_recommendations, indent=4, default=str)}\n") + if len(knob_recommendations) > 0: with open(terraform_file_path, 'r') as file: terraform_config = file.read() @@ -63,10 +73,14 @@ param_name = None param_value = None while not line.strip().startswith("}"): - if line.strip().startswith("name = "): - param_name = line.strip().split(" = ")[1].strip('""') - elif line.strip().startswith("value = "): - param_value = line.strip().split(" = ")[1].strip('""') + name_match = name_pattern.search(line) + if name_match: + param_name = name_match.group(1) + + value_match = value_pattern.search(line) + if value_match: + param_value = value_match.group(1) + line = config_lines[i] i += 1 @@ -91,13 +105,16 @@ with open(terraform_file_path, 'w') as file: file.write(modified_terraform_config) - print("Terraform configuration file (main.tf) has been updated with knob recommendations.") + print(f"Terraform configuration file ({terraform_file_path}) has been updated with knob recommendations.") else: print("No knob recommendations found for this database.") else: - print(f'Database with dbIdentifier "{db_identifier}" not found.') + print(f'Error: Database with dbIdentifier "{db_identifier}" not found.') + sys.exit(1) except requests.exceptions.RequestException as e: print(f'Error making HTTP request: {e}') + sys.exit(1) except Exception as e: print(f'An error occurred: {e}') + sys.exit(1)