diff --git a/README.md b/README.md index 83c089a..959408c 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,26 @@ [![GoDoc](https://godoc.org/github.com/crazywolf132/SecretFetch?status.svg)](https://godoc.org/github.com/crazywolf132/SecretFetch) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -## ๐ŸŒŸ Why SecretFetch? +## ๐Ÿค” The Problem + +You're building a Go application and need to manage secrets. You've got a few options, but none of them are great: -Managing secrets in Go applications can be a pain. AWS Secrets Manager is powerful but complex. Environment variables are simple but limited. What if you could have the best of both worlds? +1. **Hardcode them** (Please don't! ๐Ÿ™ˆ) +2. **Use environment variables** (Manual management, no validation, scattered across your codebase) +3. **Use AWS Secrets Manager directly** (Complex API, no caching, lots of boilerplate) +4. **Write your own solution** (Time-consuming, error-prone, reinventing the wheel) + +What if you could have: +- The simplicity of environment variables +- The security of AWS Secrets Manager +- Built-in caching and validation +- All with just a few struct tags? -SecretFetch gives you: +That's where SecretFetch comes in! ๐Ÿš€ + +## ๐ŸŒŸ Why SecretFetch? + +SecretFetch gives you the best of all worlds: - ๐ŸŽฏ **Dead Simple API** - Just add struct tags and go! - ๐Ÿ”„ **Multi-Source Support** - AWS Secrets Manager, env vars, and fallbacks in one place @@ -20,15 +35,7 @@ SecretFetch gives you: - ๐Ÿ›ก๏ธ **Validation** - Pattern matching and custom validators to catch issues early - ๐Ÿ”ง **Flexibility** - Transform values, decode base64, parse JSON/YAML - ๐Ÿƒโ€โ™‚๏ธ **Zero Config** - Works out of the box with sane defaults - -## ๐Ÿค” The Problem - -You're building a Go application and need to manage secrets. You have a few options: - -1. **Hardcode them** (Please don't!) -2. **Use environment variables** (Manual management, no validation) -3. **Use AWS Secrets Manager directly** (Complex API, no caching, lots of boilerplate) -4. **Use SecretFetch** (Simple, flexible, and powerful!) +- ๐Ÿ”Œ **Testability** - Mock AWS Secrets Manager for unit testing ## ๐Ÿš€ Quick Start @@ -59,6 +66,7 @@ if err := secretfetch.Fetch(context.Background(), cfg, nil); err != nil { ### ๐Ÿ” AWS Secrets Manager Integration ```go +// Option 1: Parse JSON secrets type DatabaseConfig struct { Host string `json:"host"` Username string `json:"username"` @@ -69,6 +77,43 @@ type Config struct { // Parse entire database config from AWS Secrets Manager DB DatabaseConfig `secret:"aws=prod/db/config,json"` } + +// Option 2: Preload ARNs for better performance +opts := &secretfetch.Options{ + PreloadARNs: true, // Enable ARN preloading + AWS: &aws.Config{ // Optional: provide custom AWS config + Region: "us-west-2", + }, +} + +// Configure ARNs through environment variables +// In development: +os.Setenv("SECRET_ARNS", "arn:aws:secretsmanager:region:account:secret:name1,arn:aws:secretsmanager:region:account:secret:name2") +// or +os.Setenv("SECRET_ARN", "arn:aws:secretsmanager:region:account:secret:name") + +// In production (ECS/Docker), configure in your task definition or docker-compose: +/* + # ECS Task Definition + { + "containerDefinitions": [ + { + "environment": [ + { + "name": "SECRET_ARNS", + "value": "arn:aws:secretsmanager:region:account:secret:name1,arn:aws:secretsmanager:region:account:secret:name2" + } + ] + } + ] + } + + # docker-compose.yml + services: + app: + environment: + - SECRET_ARNS=arn:aws:secretsmanager:region:account:secret:name1,arn:aws:secretsmanager:region:account:secret:name2 +*/ ``` ### ๐Ÿ” Pattern Validation @@ -87,7 +132,7 @@ type Config struct { ```go opts := &secretfetch.Options{ - Transformers: map[string]secretfetch.TransformerFunc{ + Transformers: map[string]secretfetch.TransformFunc{ "API_KEY": func(value string) (string, error) { return strings.TrimSpace(value), nil }, @@ -98,12 +143,34 @@ opts := &secretfetch.Options{ ### โšก Smart Caching ```go -type Config struct { - // Cache for 5 minutes - APIKey string `secret:"aws=prod/api/key,ttl=5m"` - - // Cache indefinitely - StaticConfig string `secret:"aws=prod/static/config,ttl=-1"` +opts := &secretfetch.Options{ + CacheDuration: 5 * time.Minute, // Cache secrets for 5 minutes +} +``` + +### ๐Ÿงช Testing Support + +SecretFetch makes testing a breeze with its mock interfaces: + +```go +// Mock AWS Secrets Manager client for testing +type mockSecretsManagerClient struct { + getSecretValueFn func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) +} + +func (m *mockSecretsManagerClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return m.getSecretValueFn(ctx, params, optFns...) +} + +// Use in tests +opts := &secretfetch.Options{ + SecretsManager: &mockSecretsManagerClient{ + getSecretValueFn: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String("test-secret-value"), + }, nil + }, + }, } ``` @@ -127,23 +194,42 @@ type Config struct { - ๐Ÿ”ง **Flexible** - Multiple sources, validation, transformation - ๐Ÿ“š **Well Documented** - Comprehensive examples and guides -## ๐Ÿ› ๏ธ Advanced Usage - -For a comprehensive technical deep-dive into all features and capabilities, check out our [Technical Documentation](TECHNICAL.md). +## ๐Ÿ“š Advanced Configuration -Additional resources in our [Wiki](https://github.com/crazywolf132/SecretFetch/wiki): +### Options -- Custom Validation Functions -- AWS Configuration Options -- Caching Strategies -- Error Handling -- Testing Strategies -- Best Practices +```go +type Options struct { + // AWS configuration + AWS *aws.Config + + // Custom validation functions + Validators map[string]ValidationFunc + + // Custom transformation functions + Transformers map[string]TransformFunc + + // Cache duration for secrets + CacheDuration time.Duration + + // Enable ARN preloading + PreloadARNs bool + + // Custom Secrets Manager client for testing + SecretsManager SecretsManagerClient +} +``` ## ๐Ÿค Contributing -We love contributions! Check out our [Contributing Guide](CONTRIBUTING.md) to get started. +Found a bug? Have a cool idea? Want to make SecretFetch even more awesome? We'd love your help! Feel free to: +- ๐Ÿ› Open an issue +- ๐ŸŽ‰ Submit a PR +- ๐ŸŒŸ Give us a star +- ๐Ÿ“š Improve our docs ## ๐Ÿ“ License -MIT ยฉ [Brayden](LICENSE) +This project is licensed under the MIT License - see the LICENSE file for details. + +Made with โค๏ธ by [Brayden](https://github.com/crazywolf132) diff --git a/arn_test.go b/arn_test.go new file mode 100644 index 0000000..bea9052 --- /dev/null +++ b/arn_test.go @@ -0,0 +1,227 @@ +package secretfetch + +import ( + "context" + "os" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockSecretsManagerClient is a mock implementation of the AWS Secrets Manager client +type mockSecretsManagerClient struct { + getSecretValueFn func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) +} + +func (m *mockSecretsManagerClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return m.getSecretValueFn(ctx, params, optFns...) +} + +func TestPreloadSecretsFromARNs(t *testing.T) { + // Setup test environment + testCases := []struct { + name string + arns []string + setupEnv func() + cleanupEnv func() + mockFn func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) + expectErr bool + errorContains string + }{ + { + name: "valid_single_arn", + arns: []string{"arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret"}, + setupEnv: func() { + os.Setenv("SECRET_ARN", "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret") + }, + cleanupEnv: func() { + os.Unsetenv("SECRET_ARN") + }, + mockFn: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String("test-secret-value"), + }, nil + }, + expectErr: false, + }, + { + name: "multiple_valid_arns", + arns: []string{ + "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret-1", + "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret-2", + }, + setupEnv: func() { + os.Setenv("SECRET_ARNS", "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret-1,arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret-2") + }, + cleanupEnv: func() { + os.Unsetenv("SECRET_ARNS") + }, + mockFn: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return &secretsmanager.GetSecretValueOutput{ + SecretString: aws.String("test-secret-value"), + }, nil + }, + expectErr: false, + }, + { + name: "invalid_arn_format", + arns: []string{"invalid:arn:format"}, + setupEnv: func() { + os.Setenv("SECRET_ARN", "invalid:arn:format") + }, + cleanupEnv: func() { + os.Unsetenv("SECRET_ARN") + }, + mockFn: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return nil, &types.ResourceNotFoundException{Message: aws.String("Secret not found")} + }, + expectErr: true, + errorContains: "Secret not found", + }, + { + name: "no_arns_configured", + arns: []string{}, + setupEnv: func() {}, + cleanupEnv: func() {}, + mockFn: func(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + return nil, &types.ResourceNotFoundException{Message: aws.String("Secret not found")} + }, + expectErr: true, + errorContains: "no secret ARNs found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Setup + tc.setupEnv() + defer tc.cleanupEnv() + + // Create mock client + mockClient := &mockSecretsManagerClient{ + getSecretValueFn: tc.mockFn, + } + + // Create test options with mock client + opts := &Options{ + PreloadARNs: true, + CacheDuration: time.Minute, + cache: make(map[string]*cachedValue), + AWS: &aws.Config{ + Region: "us-west-2", + }, + SecretsManager: mockClient, + } + + // Execute test + err := preloadSecretsFromARNs(context.Background(), opts) + + // Verify results + if tc.expectErr { + require.Error(t, err) + if tc.errorContains != "" { + assert.Contains(t, err.Error(), tc.errorContains) + } + } else { + require.NoError(t, err) + } + }) + } +} + +func TestGetSecretARNs(t *testing.T) { + // Setup test cases + testCases := []struct { + name string + setupEnv func() + cleanupEnv func() + expectedCount int + expectedARNs []string + }{ + { + name: "single_arn", + setupEnv: func() { + os.Setenv("SECRET_ARN", "arn:aws:secretsmanager:region:account:secret:name") + }, + cleanupEnv: func() { + os.Unsetenv("SECRET_ARN") + }, + expectedCount: 1, + expectedARNs: []string{"arn:aws:secretsmanager:region:account:secret:name"}, + }, + { + name: "multiple_arns", + setupEnv: func() { + os.Setenv("SECRET_ARNS", "arn:aws:secretsmanager:region:account:secret:name1,arn:aws:secretsmanager:region:account:secret:name2") + }, + cleanupEnv: func() { + os.Unsetenv("SECRET_ARNS") + }, + expectedCount: 2, + expectedARNs: []string{ + "arn:aws:secretsmanager:region:account:secret:name1", + "arn:aws:secretsmanager:region:account:secret:name2", + }, + }, + { + name: "no_arns", + setupEnv: func() {}, + cleanupEnv: func() {}, + expectedCount: 0, + expectedARNs: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Setup + tc.setupEnv() + defer tc.cleanupEnv() + + // Execute test + arns := getSecretARNs() + + // Verify results + assert.Equal(t, tc.expectedCount, len(arns)) + assert.ElementsMatch(t, tc.expectedARNs, arns) + }) + } +} + +func TestPreloadARNsIntegration(t *testing.T) { + // Skip if not running integration tests + if os.Getenv("INTEGRATION_TEST") != "true" { + t.Skip("Skipping integration test") + } + + // Setup test environment + testARN := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" + os.Setenv("SECRET_ARN", testARN) + defer os.Unsetenv("SECRET_ARN") + + // Create test options with AWS config + opts := &Options{ + PreloadARNs: true, + CacheDuration: time.Minute, + cache: make(map[string]*cachedValue), + } + + // Execute test + err := preloadSecretsFromARNs(context.Background(), opts) + require.NoError(t, err) + + // Verify the secret was cached + opts.cacheMu.RLock() + defer opts.cacheMu.RUnlock() + + // Check if the secret exists in the cache + cacheKey := "aws:" + testARN + cachedSecret, exists := opts.cache[cacheKey] + assert.True(t, exists) + assert.NotNil(t, cachedSecret) +} diff --git a/secret.go b/secret.go index 00392c3..813cb6f 100644 --- a/secret.go +++ b/secret.go @@ -6,7 +6,6 @@ package secretfetch import ( "context" "encoding/base64" - "encoding/json" "fmt" "os" "reflect" @@ -21,6 +20,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/secretsmanager" ) +// SecretsManagerClient defines the interface for AWS Secrets Manager operations +type SecretsManagerClient interface { + GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) +} + // Options holds configuration options for fetching secrets type Options struct { // AWS is the AWS configuration @@ -33,10 +37,15 @@ type Options struct { CacheDuration time.Duration // PreloadARNs indicates whether to preload secrets from ARNs PreloadARNs bool + // SecretsManager is the AWS Secrets Manager client + SecretsManager SecretsManagerClient cacheMu sync.RWMutex cache map[string]*cachedValue } +// OptionsKey is the key for storing Options in the context +var optionsKey = "secretfetch-options" + // ValidationFunc is a function type for custom validation type ValidationFunc func(string) error @@ -276,6 +285,9 @@ func Fetch(ctx context.Context, v interface{}, opts *Options) error { opts.cache = make(map[string]*cachedValue) } + // Store options in context + ctx = context.WithValue(ctx, optionsKey, opts) + // Preload secrets from ARNs if enabled if opts.PreloadARNs { if err := preloadSecretsFromARNs(ctx, opts); err != nil { @@ -470,69 +482,35 @@ func (s *secret) getFromAWS(ctx context.Context, awsConfig *aws.Config) (string, return "", nil } - // Try to get from cache first - secretsMu.RLock() - if value, ok := secretsCache[s.awsKey]; ok { - secretsMu.RUnlock() - return value, nil - } - secretsMu.RUnlock() - - // Initialize cache if needed - secretsOnce.Do(func() { - secretsCache = make(map[string]string) - }) - - // Load AWS config - cfg, err := config.LoadDefaultConfig(ctx, func(o *config.LoadOptions) error { - if awsConfig != nil { - o.Region = awsConfig.Region - o.Credentials = awsConfig.Credentials + var client SecretsManagerClient + if opts, ok := ctx.Value(optionsKey).(*Options); ok && opts.SecretsManager != nil { + client = opts.SecretsManager + } else { + cfg := awsConfig + if cfg == nil { + defaultCfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return "", fmt.Errorf("unable to load AWS config: %v", err) + } + cfg = &defaultCfg } - return nil - }) - if err != nil { - return "", fmt.Errorf("unable to load AWS config: %w", err) + client = secretsmanager.NewFromConfig(*cfg) } - // Create client and fetch secret - client := secretsmanager.NewFromConfig(cfg) input := &secretsmanager.GetSecretValueInput{ SecretId: aws.String(s.awsKey), } result, err := client.GetSecretValue(ctx, input) if err != nil { - return "", fmt.Errorf("failed to get AWS secret %s: %w", s.awsKey, err) - } - - if result.SecretString == nil { - return "", fmt.Errorf("no secret string found for %s", s.awsKey) + return "", fmt.Errorf("error fetching secret %s: %v", s.awsKey, err) } - // Try to parse as JSON first - var secretMap map[string]string - if err := json.Unmarshal([]byte(*result.SecretString), &secretMap); err == nil { - // If successful, cache all values - secretsMu.Lock() - for k, v := range secretMap { - secretsCache[k] = v - } - secretsMu.Unlock() - - // Return the specific key if it exists - if value, ok := secretMap[s.awsKey]; ok { - return value, nil - } - // If key not found in JSON, use the entire string + if result.SecretString != nil { + return *result.SecretString, nil } - // Cache and return the raw string - secretsMu.Lock() - secretsCache[s.awsKey] = *result.SecretString - secretsMu.Unlock() - - return *result.SecretString, nil + return "", fmt.Errorf("no secret string found for %s", s.awsKey) } // FetchAndValidate is an alias for Fetch to maintain backward compatibility @@ -542,60 +520,48 @@ func FetchAndValidate(ctx context.Context, v interface{}) error { // preloadSecretsFromARNs fetches secrets from AWS Secrets Manager and caches them func preloadSecretsFromARNs(ctx context.Context, opts *Options) error { - secretArns := getSecretARNs() - if len(secretArns) == 0 { + arns := getSecretARNs() + if len(arns) == 0 { return fmt.Errorf("no secret ARNs found in environment variables SECRET_ARNS or SECRET_ARN") } - if opts.AWS == nil { - // Load default AWS config if not provided - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return fmt.Errorf("unable to load AWS config: %w", err) - } - opts.AWS = &cfg - } - - client := secretsmanager.NewFromConfig(*opts.AWS) - - var wg sync.WaitGroup - errorsCh := make(chan error, len(secretArns)) - - for _, arn := range secretArns { - wg.Add(1) - go func(arn string) { - defer wg.Done() - output, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(arn), - }) + var client SecretsManagerClient + if opts.SecretsManager != nil { + client = opts.SecretsManager + } else { + cfg := opts.AWS + if cfg == nil { + defaultCfg, err := config.LoadDefaultConfig(ctx) if err != nil { - errorsCh <- fmt.Errorf("error fetching secret %s: %w", arn, err) - return + return fmt.Errorf("unable to load AWS config: %v", err) } + cfg = &defaultCfg + } + client = secretsmanager.NewFromConfig(*cfg) + } - var secretPairs map[string]string - if err := json.Unmarshal([]byte(aws.ToString(output.SecretString)), &secretPairs); err != nil { - // If it's not JSON, store the raw secret string - secretPairs = map[string]string{ - arn: aws.ToString(output.SecretString), - } - } + for _, arn := range arns { + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(arn), + } - // Cache the secrets - secretsMu.Lock() - for k, v := range secretPairs { - secretsCache[k] = v - } - secretsMu.Unlock() - }(arn) - } + result, err := client.GetSecretValue(ctx, input) + if err != nil { + return fmt.Errorf("error fetching secret %s: %v", arn, err) + } - wg.Wait() - close(errorsCh) + if result.SecretString == nil { + return fmt.Errorf("no secret string found for %s", arn) + } - // Check for errors - if len(errorsCh) > 0 { - return <-errorsCh // Return the first error encountered + // Cache the secret + cacheKey := "aws:" + arn + opts.cacheMu.Lock() + opts.cache[cacheKey] = &cachedValue{ + value: *result.SecretString, + expiration: time.Now().Add(opts.CacheDuration), + } + opts.cacheMu.Unlock() } return nil