Allow specifying a custom S3 endpoint

BemiHQ · Dec 9, 2024 · bb9685e · bb9685e
1 parent afb1329
commit bb9685e
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 25 deletions.
diff --git a/.env.sample b/.env.sample
@@ -14,6 +14,7 @@ BEMIDB_STORAGE_PATH=./iceberg
 # BEMIDB_STORAGE_TYPE=S3
 # BEMIDB_STORAGE_PATH=iceberg
 # AWS_REGION=us-west-1
+# AWS_ENDPOINT=s3.amazonaws.com
 # AWS_S3_BUCKET=[REPLACE_ME]
 # AWS_ACCESS_KEY_ID=[REPLACE_ME]
 # AWS_SECRET_ACCESS_KEY=[REPLACE_ME]

diff --git a/README.md b/README.md
@@ -184,28 +184,29 @@ psql postgres://localhost:54321/bemidb -c \
 
 ### Configuration options
 
-| CLI argument              | Environment variable    | Default value  | Description                                                               |
-|---------------------------|-------------------------|----------------|---------------------------------------------------------------------------|
-| `--host`                  | `BEMIDB_HOST`           | `127.0.0.1`    | Host for BemiDB to listen on                                              |
-| `--port`                  | `BEMIDB_PORT`           | `54321`        | Port for BemiDB to listen on                                              |
-| `--database`              | `BEMIDB_DATABASE`       | `bemidb`       | Database name                                                             |
-| `--storage-type`          | `BEMIDB_STORAGE_TYPE`   | `LOCAL`        | Storage type: `LOCAL` or `S3`                                             |
-| `--storage-path`          | `BEMIDB_STORAGE_PATH`   | `iceberg`      | Path to the storage folder                                                |
-| `--log-level`             | `BEMIDB_LOG_LEVEL`      | `INFO`         | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`                      |
-| `--init-sql `             | `BEMIDB_INIT_SQL`       | `./init.sql`   | Path to the initialization SQL file                                       |
-| `--user`                  | `BEMIDB_USER`           |                | Database user. Allows any if empty                                        |
-| `--password`              | `BEMIDB_PASSWORD`       |                | Database password. Allows any if empty                                    |
-| `--aws-region`            | `AWS_REGION`            |                | AWS region. Required if storage type is `S3`                              |
-| `--aws-s3-bucket`         | `AWS_S3_BUCKET`         |                | AWS S3 bucket name. Required if storage type is `S3`                      |
-| `--aws-access-key-id`     | `AWS_ACCESS_KEY_ID`     |                | AWS access key ID. Required if storage type is `S3`                       |
-| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` |                | AWS secret access key. Required if storage type is `S3`                   |
-| `--pg-database-url`       | `PG_DATABASE_URL`       |                | PostgreSQL database URL to sync                                           |
-| `--pg-sync-interval`      | `PG_SYNC_INTERVAL`      |                | Interval between syncs. Valid units: `ns`, `us`/`µs`, `ms`, `s`, `m`, `h` |
-| `--pg-exclude-schemas`    | `PG_EXCLUDE_SCHEMAS`    |                | List of schemas to exclude from sync. Comma-separated                     |
-| `--pg-include-schemas`    | `PG_INCLUDE_SCHEMAS`    |                | List of schemas to include in sync. Comma-separated                       |
-| `--pg-exclude-tables`     | `PG_EXCLUDE_TABLES`     |                | List of tables to exclude from sync. Comma-separated `schema.table`       |
-| `--pg-include-tables`     | `PG_INCLUDE_TABLES`     |                | List of tables to include in sync. Comma-separated `schema.table`         |
-| `--pg-schema-prefix`      | `PG_SCHEMA_PREFIX`      |                | Prefix for PostgreSQL schema names                                        |
+| CLI argument              | Environment variable    | Default value      | Description                                                               |
+|---------------------------|-------------------------|--------------------|---------------------------------------------------------------------------|
+| `--host`                  | `BEMIDB_HOST`           | `127.0.0.1`        | Host for BemiDB to listen on                                              |
+| `--port`                  | `BEMIDB_PORT`           | `54321`            | Port for BemiDB to listen on                                              |
+| `--database`              | `BEMIDB_DATABASE`       | `bemidb`           | Database name                                                             |
+| `--storage-type`          | `BEMIDB_STORAGE_TYPE`   | `LOCAL`            | Storage type: `LOCAL` or `S3`                                             |
+| `--storage-path`          | `BEMIDB_STORAGE_PATH`   | `iceberg`          | Path to the storage folder                                                |
+| `--log-level`             | `BEMIDB_LOG_LEVEL`      | `INFO`             | Log level: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`                      |
+| `--init-sql `             | `BEMIDB_INIT_SQL`       | `./init.sql`       | Path to the initialization SQL file                                       |
+| `--user`                  | `BEMIDB_USER`           |                    | Database user. Allows any if empty                                        |
+| `--password`              | `BEMIDB_PASSWORD`       |                    | Database password. Allows any if empty                                    |
+| `--aws-region`            | `AWS_REGION`            |                    | AWS region. Required if storage type is `S3`                              |
+| `--aws-s3-endpoint`       | `AWS_S3_ENDPOINT`       | `s3.amazonaws.com` | AWS S3 endpoint
+| `--aws-s3-bucket`         | `AWS_S3_BUCKET`         |                    | AWS S3 bucket name. Required if storage type is `S3`                      |
+| `--aws-access-key-id`     | `AWS_ACCESS_KEY_ID`     |                    | AWS access key ID. Required if storage type is `S3`                       |
+| `--aws-secret-access-key` | `AWS_SECRET_ACCESS_KEY` |                    | AWS secret access key. Required if storage type is `S3`                   |
+| `--pg-database-url`       | `PG_DATABASE_URL`       |                    | PostgreSQL database URL to sync                                           |
+| `--pg-sync-interval`      | `PG_SYNC_INTERVAL`      |                    | Interval between syncs. Valid units: `ns`, `us`/`µs`, `ms`, `s`, `m`, `h` |
+| `--pg-exclude-schemas`    | `PG_EXCLUDE_SCHEMAS`    |                    | List of schemas to exclude from sync. Comma-separated                     |
+| `--pg-include-schemas`    | `PG_INCLUDE_SCHEMAS`    |                    | List of schemas to include in sync. Comma-separated                       |
+| `--pg-exclude-tables`     | `PG_EXCLUDE_TABLES`     |                    | List of tables to exclude from sync. Comma-separated `schema.table`       |
+| `--pg-include-tables`     | `PG_INCLUDE_TABLES`     |                    | List of tables to include in sync. Comma-separated `schema.table`         |
+| `--pg-schema-prefix`      | `PG_SCHEMA_PREFIX`      |                    | Prefix for PostgreSQL schema names                                        |
 
 Note that CLI arguments take precedence over environment variables. I.e. you can override the environment variables with CLI arguments.
 

diff --git a/scripts/install.sh b/scripts/install.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION="0.21.0"
+VERSION="0.22.0"
 
 # Detect OS and architecture
 OS=$(uname -s | tr '[:upper:]' '[:lower:]')

diff --git a/src/config.go b/src/config.go
@@ -19,6 +19,7 @@ const (
 	ENV_STORAGE_TYPE      = "BEMIDB_STORAGE_TYPE"
 
 	ENV_AWS_REGION            = "AWS_REGION"
+	ENV_AWS_S3_ENDPOINT       = "AWS_S3_ENDPOINT"
 	ENV_AWS_S3_BUCKET         = "AWS_S3_BUCKET"
 	ENV_AWS_ACCESS_KEY_ID     = "AWS_ACCESS_KEY_ID"
 	ENV_AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
@@ -41,12 +42,15 @@ const (
 	DEFAULT_LOG_LEVEL         = "INFO"
 	DEFAULT_DB_STORAGE_TYPE   = "LOCAL"
 
+	DEFAULT_AWS_S3_ENDPOINT = "s3.amazonaws.com"
+
 	STORAGE_TYPE_LOCAL = "LOCAL"
 	STORAGE_TYPE_S3    = "S3"
 )
 
 type AwsConfig struct {
 	Region          string
+	S3Endpoint      string // optional
 	S3Bucket        string
 	AccessKeyId     string
 	SecretAccessKey string
@@ -109,6 +113,7 @@ func registerFlags() {
 	flag.StringVar(&_configParseValues.pgExcludeTables, "pg-exclude-tables", os.Getenv(ENV_PG_EXCLUDE_TABLES), "(Optional) Comma-separated list of tables to exclude from sync (format: schema.table)")
 	flag.StringVar(&_config.Pg.DatabaseUrl, "pg-database-url", os.Getenv(ENV_PG_DATABASE_URL), "PostgreSQL database URL to sync")
 	flag.StringVar(&_config.Aws.Region, "aws-region", os.Getenv(ENV_AWS_REGION), "AWS region")
+	flag.StringVar(&_config.Aws.S3Endpoint, "aws-s3-endpoint", os.Getenv(ENV_AWS_S3_ENDPOINT), "AWS S3 endpoint. Default: \""+DEFAULT_AWS_S3_ENDPOINT+"\"")
 	flag.StringVar(&_config.Aws.S3Bucket, "aws-s3-bucket", os.Getenv(ENV_AWS_S3_BUCKET), "AWS S3 bucket name")
 	flag.StringVar(&_config.Aws.AccessKeyId, "aws-access-key-id", os.Getenv(ENV_AWS_ACCESS_KEY_ID), "AWS access key ID")
 	flag.StringVar(&_config.Aws.SecretAccessKey, "aws-secret-access-key", os.Getenv(ENV_AWS_SECRET_ACCESS_KEY), "AWS secret access key")
@@ -158,6 +163,9 @@ func parseFlags() {
 		if _config.Aws.Region == "" {
 			panic("AWS region is required")
 		}
+		if _config.Aws.S3Endpoint == "" {
+			_config.Aws.S3Endpoint = DEFAULT_AWS_S3_ENDPOINT
+		}
 		if _config.Aws.S3Bucket == "" {
 			panic("AWS S3 bucket name is required")
 		}

diff --git a/src/config_test.go b/src/config_test.go
@@ -87,6 +87,7 @@ func TestLoadConfig(t *testing.T) {
 		t.Setenv("BEMIDB_LOG_LEVEL", "ERROR")
 		t.Setenv("BEMIDB_STORAGE_TYPE", "S3")
 		t.Setenv("AWS_REGION", "us-west-1")
+		t.Setenv("AWS_S3_ENDPOINT", "s3-us-west-1.amazonaws.com")
 		t.Setenv("AWS_S3_BUCKET", "my_bucket")
 		t.Setenv("AWS_ACCESS_KEY_ID", "my_access_key_id")
 		t.Setenv("AWS_SECRET_ACCESS_KEY", "my_secret_access_key")
@@ -114,6 +115,9 @@ func TestLoadConfig(t *testing.T) {
 		if config.Aws.Region != "us-west-1" {
 			t.Errorf("Expected awsRegion to be us-west-1, got %s", config.Aws.Region)
 		}
+		if config.Aws.S3Endpoint != "s3-us-west-1.amazonaws.com" {
+			t.Errorf("Expected awsS3Endpoint to be s3-us-west-1.amazonaws.com, got %s", config.Aws.S3Endpoint)
+		}
 		if config.Aws.S3Bucket != "my_bucket" {
 			t.Errorf("Expected awsS3Bucket to be mybucket, got %s", config.Aws.S3Bucket)
 		}

diff --git a/src/duckdb.go b/src/duckdb.go
@@ -41,11 +41,12 @@ func NewDuckdb(config *Config) *Duckdb {
 
 	switch config.StorageType {
 	case STORAGE_TYPE_S3:
-		query := "CREATE SECRET aws_s3_secret (TYPE S3, KEY_ID '$accessKeyId', SECRET '$secretAccessKey', REGION '$region', SCOPE '$s3Bucket')"
+		query := "CREATE SECRET aws_s3_secret (TYPE S3, KEY_ID '$accessKeyId', SECRET '$secretAccessKey', REGION '$region', ENDPOINT '$endpoint', SCOPE '$s3Bucket')"
 		_, err = db.ExecContext(ctx, replaceNamedStringArgs(query, map[string]string{
 			"accessKeyId":     config.Aws.AccessKeyId,
 			"secretAccessKey": config.Aws.SecretAccessKey,
 			"region":          config.Aws.Region,
+			"endpoint":        config.Aws.S3Endpoint,
 			"s3Bucket":        "s3://" + config.Aws.S3Bucket,
 		}))
 		PanicIfError(err)

diff --git a/src/main.go b/src/main.go
@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-const VERSION = "0.21.0"
+const VERSION = "0.22.0"
 
 func main() {
 	config := LoadConfig()