diff --git a/.env.sample b/.env.sample index 5a57a4f..cf74f98 100644 --- a/.env.sample +++ b/.env.sample @@ -14,6 +14,7 @@ BEMIDB_STORAGE_PATH=./iceberg # BEMIDB_STORAGE_TYPE=S3 # BEMIDB_STORAGE_PATH=iceberg # AWS_REGION=us-west-1 +# AWS_ENDPOINT=s3.amazonaws.com # AWS_S3_BUCKET=[REPLACE_ME] # AWS_ACCESS_KEY_ID=[REPLACE_ME] # AWS_SECRET_ACCESS_KEY=[REPLACE_ME] diff --git a/README.md b/README.md index 9c046ab..b76eff9 100644 --- a/README.md +++ b/README.md @@ -184,28 +184,29 @@ psql postgres://localhost:54321/bemidb -c \ ### Configuration options -| CLI argument | Environment variable | Default value | Description | -|---------------------------|-------------------------|----------------|---------------------------------------------------------------------------| -| `--host` | `BEMIDB_HOST` | `127.0.0.1` | Host for BemiDB to listen on | -| `--port` | `BEMIDB_PORT` | `54321` | Port for BemiDB to listen on | -| `--database` | `BEMIDB_DATABASE` | `bemidb` | Database name | -| `--storage-type` | `BEMIDB_STORAGE_TYPE` | `LOCAL` | Storage type: `LOCAL` or `S3` | -| `--storage-path` | `BEMIDB_STORAGE_PATH` | `iceberg` | Path to the storage folder | -| `--log-level` | `BEMIDB_LOG_LEVEL` | `INFO` | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` | -| `--init-sql ` | `BEMIDB_INIT_SQL` | `./init.sql` | Path to the initialization SQL file | -| `--user` | `BEMIDB_USER` | | Database user. Allows any if empty | -| `--password` | `BEMIDB_PASSWORD` | | Database password. Allows any if empty | -| `--aws-region` | `AWS_REGION` | | AWS region. Required if storage type is `S3` | -| `--aws-s3-bucket` | `AWS_S3_BUCKET` | | AWS S3 bucket name. Required if storage type is `S3` | -| `--aws-access-key-id` | `AWS_ACCESS_KEY_ID` | | AWS access key ID. Required if storage type is `S3` | -| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | | AWS secret access key. Required if storage type is `S3` | -| `--pg-database-url` | `PG_DATABASE_URL` | | PostgreSQL database URL to sync | -| `--pg-sync-interval` | `PG_SYNC_INTERVAL` | | Interval between syncs. Valid units: `ns`, `us`/`µs`, `ms`, `s`, `m`, `h` | -| `--pg-exclude-schemas` | `PG_EXCLUDE_SCHEMAS` | | List of schemas to exclude from sync. Comma-separated | -| `--pg-include-schemas` | `PG_INCLUDE_SCHEMAS` | | List of schemas to include in sync. Comma-separated | -| `--pg-exclude-tables` | `PG_EXCLUDE_TABLES` | | List of tables to exclude from sync. Comma-separated `schema.table` | -| `--pg-include-tables` | `PG_INCLUDE_TABLES` | | List of tables to include in sync. Comma-separated `schema.table` | -| `--pg-schema-prefix` | `PG_SCHEMA_PREFIX` | | Prefix for PostgreSQL schema names | +| CLI argument | Environment variable | Default value | Description | +|---------------------------|-------------------------|--------------------|---------------------------------------------------------------------------| +| `--host` | `BEMIDB_HOST` | `127.0.0.1` | Host for BemiDB to listen on | +| `--port` | `BEMIDB_PORT` | `54321` | Port for BemiDB to listen on | +| `--database` | `BEMIDB_DATABASE` | `bemidb` | Database name | +| `--storage-type` | `BEMIDB_STORAGE_TYPE` | `LOCAL` | Storage type: `LOCAL` or `S3` | +| `--storage-path` | `BEMIDB_STORAGE_PATH` | `iceberg` | Path to the storage folder | +| `--log-level` | `BEMIDB_LOG_LEVEL` | `INFO` | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` | +| `--init-sql ` | `BEMIDB_INIT_SQL` | `./init.sql` | Path to the initialization SQL file | +| `--user` | `BEMIDB_USER` | | Database user. Allows any if empty | +| `--password` | `BEMIDB_PASSWORD` | | Database password. Allows any if empty | +| `--aws-region` | `AWS_REGION` | | AWS region. Required if storage type is `S3` | +| `--aws-s3-endpoint` | `AWS_S3_ENDPOINT` | `s3.amazonaws.com` | AWS S3 endpoint +| `--aws-s3-bucket` | `AWS_S3_BUCKET` | | AWS S3 bucket name. Required if storage type is `S3` | +| `--aws-access-key-id` | `AWS_ACCESS_KEY_ID` | | AWS access key ID. Required if storage type is `S3` | +| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` | | AWS secret access key. Required if storage type is `S3` | +| `--pg-database-url` | `PG_DATABASE_URL` | | PostgreSQL database URL to sync | +| `--pg-sync-interval` | `PG_SYNC_INTERVAL` | | Interval between syncs. Valid units: `ns`, `us`/`µs`, `ms`, `s`, `m`, `h` | +| `--pg-exclude-schemas` | `PG_EXCLUDE_SCHEMAS` | | List of schemas to exclude from sync. Comma-separated | +| `--pg-include-schemas` | `PG_INCLUDE_SCHEMAS` | | List of schemas to include in sync. Comma-separated | +| `--pg-exclude-tables` | `PG_EXCLUDE_TABLES` | | List of tables to exclude from sync. Comma-separated `schema.table` | +| `--pg-include-tables` | `PG_INCLUDE_TABLES` | | List of tables to include in sync. Comma-separated `schema.table` | +| `--pg-schema-prefix` | `PG_SCHEMA_PREFIX` | | Prefix for PostgreSQL schema names | Note that CLI arguments take precedence over environment variables. I.e. you can override the environment variables with CLI arguments. diff --git a/scripts/install.sh b/scripts/install.sh index 4e0f2a5..a7df640 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION="0.21.0" +VERSION="0.22.0" # Detect OS and architecture OS=$(uname -s | tr '[:upper:]' '[:lower:]') diff --git a/src/config.go b/src/config.go index 63b6fe2..8c4272c 100644 --- a/src/config.go +++ b/src/config.go @@ -19,6 +19,7 @@ const ( ENV_STORAGE_TYPE = "BEMIDB_STORAGE_TYPE" ENV_AWS_REGION = "AWS_REGION" + ENV_AWS_S3_ENDPOINT = "AWS_S3_ENDPOINT" ENV_AWS_S3_BUCKET = "AWS_S3_BUCKET" ENV_AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" ENV_AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" @@ -41,12 +42,15 @@ const ( DEFAULT_LOG_LEVEL = "INFO" DEFAULT_DB_STORAGE_TYPE = "LOCAL" + DEFAULT_AWS_S3_ENDPOINT = "s3.amazonaws.com" + STORAGE_TYPE_LOCAL = "LOCAL" STORAGE_TYPE_S3 = "S3" ) type AwsConfig struct { Region string + S3Endpoint string // optional S3Bucket string AccessKeyId string SecretAccessKey string @@ -109,6 +113,7 @@ func registerFlags() { flag.StringVar(&_configParseValues.pgExcludeTables, "pg-exclude-tables", os.Getenv(ENV_PG_EXCLUDE_TABLES), "(Optional) Comma-separated list of tables to exclude from sync (format: schema.table)") flag.StringVar(&_config.Pg.DatabaseUrl, "pg-database-url", os.Getenv(ENV_PG_DATABASE_URL), "PostgreSQL database URL to sync") flag.StringVar(&_config.Aws.Region, "aws-region", os.Getenv(ENV_AWS_REGION), "AWS region") + flag.StringVar(&_config.Aws.S3Endpoint, "aws-s3-endpoint", os.Getenv(ENV_AWS_S3_ENDPOINT), "AWS S3 endpoint. Default: \""+DEFAULT_AWS_S3_ENDPOINT+"\"") flag.StringVar(&_config.Aws.S3Bucket, "aws-s3-bucket", os.Getenv(ENV_AWS_S3_BUCKET), "AWS S3 bucket name") flag.StringVar(&_config.Aws.AccessKeyId, "aws-access-key-id", os.Getenv(ENV_AWS_ACCESS_KEY_ID), "AWS access key ID") flag.StringVar(&_config.Aws.SecretAccessKey, "aws-secret-access-key", os.Getenv(ENV_AWS_SECRET_ACCESS_KEY), "AWS secret access key") @@ -158,6 +163,9 @@ func parseFlags() { if _config.Aws.Region == "" { panic("AWS region is required") } + if _config.Aws.S3Endpoint == "" { + _config.Aws.S3Endpoint = DEFAULT_AWS_S3_ENDPOINT + } if _config.Aws.S3Bucket == "" { panic("AWS S3 bucket name is required") } diff --git a/src/config_test.go b/src/config_test.go index a4bfa54..f56b969 100644 --- a/src/config_test.go +++ b/src/config_test.go @@ -87,6 +87,7 @@ func TestLoadConfig(t *testing.T) { t.Setenv("BEMIDB_LOG_LEVEL", "ERROR") t.Setenv("BEMIDB_STORAGE_TYPE", "S3") t.Setenv("AWS_REGION", "us-west-1") + t.Setenv("AWS_S3_ENDPOINT", "s3-us-west-1.amazonaws.com") t.Setenv("AWS_S3_BUCKET", "my_bucket") t.Setenv("AWS_ACCESS_KEY_ID", "my_access_key_id") t.Setenv("AWS_SECRET_ACCESS_KEY", "my_secret_access_key") @@ -114,6 +115,9 @@ func TestLoadConfig(t *testing.T) { if config.Aws.Region != "us-west-1" { t.Errorf("Expected awsRegion to be us-west-1, got %s", config.Aws.Region) } + if config.Aws.S3Endpoint != "s3-us-west-1.amazonaws.com" { + t.Errorf("Expected awsS3Endpoint to be s3-us-west-1.amazonaws.com, got %s", config.Aws.S3Endpoint) + } if config.Aws.S3Bucket != "my_bucket" { t.Errorf("Expected awsS3Bucket to be mybucket, got %s", config.Aws.S3Bucket) } diff --git a/src/duckdb.go b/src/duckdb.go index 6eaf9a7..dc6f0f6 100644 --- a/src/duckdb.go +++ b/src/duckdb.go @@ -41,11 +41,12 @@ func NewDuckdb(config *Config) *Duckdb { switch config.StorageType { case STORAGE_TYPE_S3: - query := "CREATE SECRET aws_s3_secret (TYPE S3, KEY_ID '$accessKeyId', SECRET '$secretAccessKey', REGION '$region', SCOPE '$s3Bucket')" + query := "CREATE SECRET aws_s3_secret (TYPE S3, KEY_ID '$accessKeyId', SECRET '$secretAccessKey', REGION '$region', ENDPOINT '$endpoint', SCOPE '$s3Bucket')" _, err = db.ExecContext(ctx, replaceNamedStringArgs(query, map[string]string{ "accessKeyId": config.Aws.AccessKeyId, "secretAccessKey": config.Aws.SecretAccessKey, "region": config.Aws.Region, + "endpoint": config.Aws.S3Endpoint, "s3Bucket": "s3://" + config.Aws.S3Bucket, })) PanicIfError(err) diff --git a/src/main.go b/src/main.go index 0acc756..d1adfd2 100644 --- a/src/main.go +++ b/src/main.go @@ -6,7 +6,7 @@ import ( "time" ) -const VERSION = "0.21.0" +const VERSION = "0.22.0" func main() { config := LoadConfig()