diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..7238196 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +# Basic set up for three package managers +version: 2 +updates: + # Maintain dependencies for Composer + - package-ecosystem: "gomod" + directory: "./" + reviewers: + - "Matrix278" + - "KostLinux" + schedule: + interval: "weekly" + commit-message: + prefix: '[Go Modules]' + include: scope \ No newline at end of file diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..fb511a9 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,10 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +change-title-escapes: '\<*_&' +template: | + # What's Changed + + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION \ No newline at end of file diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..3548ade --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,76 @@ +name: Verify & Release +on: + pull_request: + types: + - opened + - synchronize + - edited + - closed + push: + branches: + - '*' + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +env: + GOPROXY: https://proxy.golang.org + +permissions: + contents: write + packages: read + statuses: write + pull-requests: write + +jobs: + verify_quality: + name: Verify Code Quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.56.2 + working-directory: ./ + only-new-issues: false + args: --concurrency=16 --timeout=5m --out-format=github-actions --issues-exit-code=1 + skip-cache: false + skip-pkg-cache: true + + verify_functionality: + name: Verify Code Functionality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Verify functionality + run: go test -v ./... + + publish_release: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + needs: + - verify_quality + - verify_functionality + runs-on: ubuntu-latest + steps: + - name: Set version env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - uses: release-drafter/release-drafter@v6 + with: + disable-autolabeler: true + name: ${{ env.RELEASE_VERSION }} + tag: ${{ env.RELEASE_VERSION }} + version: ${{ env.RELEASE_VERSION }} + publish: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index c0de26c..e69de29 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +0,0 @@ -benchmarks/ \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..9d48047 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,51 @@ +run: + concurrency: 4 + deadline: 5m + tests: true + +linters-settings: + gocyclo: + min-complexity: 20 + +linters: + disable-all: true + enable: + - asasalint + - asciicheck + - bidichk + - bodyclose + - dogsled + - durationcheck + - errcheck + - errchkjson + - forbidigo + - gocognit + - gocritic + - godox + - gofmt + - gofumpt + - goimports + - goprintffuncname + - gosimple + - govet + - ineffassign + - misspell + - nakedret + - nestif + - nilerr + - nonamedreturns + - prealloc + - predeclared + - revive + - staticcheck + - stylecheck + - tenv + - typecheck + - unconvert + - unparam + - unused + - usestdlibvars + - whitespace + +issues: + max-same-issues: 0 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d393517 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +run: + go run main.go + +mod-vendor: + go mod vendor + +linter: + @golangci-lint run + +gosec: + @gosec -quiet ./... + +validate: linter gosec diff --git a/README.md b/README.md index ef3da8d..c472760 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,9 @@ func main() { } } ``` +## Logs example at CloudWatch -## Benchmarks - - +![CloudWatch Logs](assets/cloudwatch.png) ## License diff --git a/assets/cloudwatch.png b/assets/cloudwatch.png new file mode 100644 index 0000000..2f04fad Binary files /dev/null and b/assets/cloudwatch.png differ diff --git a/cmd/Dockerfile b/cmd/Dockerfile index abe573e..7e7ed64 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -11,7 +11,7 @@ COPY . . RUN go mod download # Build the Go app -RUN go build -o main . +RUN GOOS=linux GOARCH=amd64 go build -o main . # Start a new stage from scratch FROM alpine:latest @@ -23,4 +23,4 @@ COPY --from=builder /app/main /app/main EXPOSE 8080 # Command to run the executable -ENTRYPOINT ["/app/main"] \ No newline at end of file +ENTRYPOINT ["/app/main"] diff --git a/cmd/build.sh b/cmd/build.sh new file mode 100644 index 0000000..b943841 --- /dev/null +++ b/cmd/build.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Source environment variables +source ~/dev/lookinlabs/terraform/platform-tf/.env.production + +# Define the image name and tag +IMAGE_NAME="public.ecr.aws/c2w5h6c4/go-logger-middleware" +IMAGE_TAG="latest" + +# Build the Docker image without using the cache +echo "Building Docker image without cache..." +docker build --no-cache -t ${IMAGE_NAME}:${IMAGE_TAG} . +if [ $? -ne 0 ]; then + echo "Failed to build Docker image" + exit 1 +fi + +# Reauthenticate with AWS ECR +echo "Authenticating with AWS ECR..." +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/c2w5h6c4 +if [ $? -ne 0 ]; then + echo "Failed to authenticate with AWS ECR" + exit 1 +fi + +# Push the Docker image +echo "Pushing Docker image..." +docker push ${IMAGE_NAME}:${IMAGE_TAG} +if [ $? -ne 0 ]; then + echo "Failed to push Docker image" + exit 1 +fi + +echo "Docker image pushed successfully" \ No newline at end of file diff --git a/cmd/go.mod b/cmd/go.mod index 9a987db..b19c81e 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -4,5 +4,7 @@ go 1.23.2 require ( github.com/go-chi/chi/v5 v5.1.0 - github.com/lookinlabs/go-logger-middleware v0.0.0-20241023154743-cacce64726f3 + github.com/lookinlabs/go-logger-middleware v0.0.0-00010101000000-000000000000 ) + +replace github.com/lookinlabs/go-logger-middleware => ../ diff --git a/cmd/go.sum b/cmd/go.sum index 62d4b86..823cdbb 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -1,4 +1,2 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/lookinlabs/go-logger-middleware v0.0.0-20241023154743-cacce64726f3 h1:FqaC99m8pfUtES08uMX6TWXq6ubo2B+0zHWDDMKvOxk= -github.com/lookinlabs/go-logger-middleware v0.0.0-20241023154743-cacce64726f3/go.mod h1:sLQMeTuFt+ZV367AHHYnGgGtbGwdqEB01Inrg/UH+BQ= diff --git a/cmd/main.go b/cmd/main.go index 6edaf6b..0c3388c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -4,56 +4,46 @@ import ( "encoding/json" "log" "net/http" - "os" - - "github.com/lookinlabs/go-logger-middleware" + "time" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) func main() { - // Initialize the logger middleware - sensitiveFields := []string{"password", "token"} - appLogger := log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) - loggerMiddleware := logger.NewLoggerMiddleware(sensitiveFields, appLogger) - - // Create a Chi router r := chi.NewRouter() + r.Use(middleware.Logger) - // Use the built-in Chi middleware - r.Use(middleware.RequestID) - r.Use(middleware.RealIP) - r.Use(middleware.Recoverer) - - // Use the custom logger middleware - r.Use(loggerMiddleware.Middleware) - - // Define a simple GET endpoint r.Get("/hello", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("Hello, World!")) - }) - - // Define a POST endpoint to test sanitization - r.Post("/test", func(w http.ResponseWriter, r *http.Request) { - var requestBody map[string]interface{} - if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + if _, err := w.Write([]byte("Hello, World!")); err != nil { + log.Printf("Failed to write response: %v", err) } + }) - responseBody, err := json.Marshal(requestBody) + r.Get("/json", func(w http.ResponseWriter, r *http.Request) { + response := map[string]string{"message": "Hello, JSON!"} + responseBody, err := json.Marshal(response) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "Failed to marshal JSON", http.StatusInternalServerError) return } - w.Header().Set("Content-Type", "application/json") - w.Write(responseBody) + if _, err := w.Write(responseBody); err != nil { + log.Printf("Failed to write response: %v", err) + } }) + // Create a custom server with timeouts + server := &http.Server{ + Addr: ":8080", + Handler: r, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 15 * time.Second, + } + // Start the server - if err := http.ListenAndServe(":8080", r); err != nil { + if err := server.ListenAndServe(); err != nil { log.Fatalf("Failed to run server: %v", err) } } diff --git a/json.go b/json.go index c7c4d01..3ae18bc 100644 --- a/json.go +++ b/json.go @@ -47,3 +47,12 @@ func Unmarshal(data []byte, v interface{}) error { } return nil } + +// MapToKeyValuePairs converts a map to a slice of KeyValuePair +func MapToKeyValuePairs(m map[string]interface{}) []KeyValuePair { + pairs := make([]KeyValuePair, 0, len(m)) + for k, v := range m { + pairs = append(pairs, KeyValuePair{Key: k, Value: v}) + } + return pairs +} diff --git a/logger.go b/logger.go index bc105c2..387167c 100644 --- a/logger.go +++ b/logger.go @@ -2,12 +2,13 @@ package logger import ( "bytes" - "encoding/json" + "crypto/rand" "fmt" "io" "log" - "math/rand" + "math/big" "net/http" + "sync" "time" ) @@ -34,52 +35,69 @@ func (resp *responseCapture) Header() http.Header { } // LoggerMiddleware is a struct that holds the configuration for the middleware. -type LoggerMiddleware struct { +type Middleware struct { sensitiveFields []string logger *log.Logger + bufferPool sync.Pool } -// NewLoggerMiddleware creates a new LoggerMiddleware with the given sensitive fields and logger. -func NewLoggerMiddleware(sensitiveFields []string, logger *log.Logger) *LoggerMiddleware { - return &LoggerMiddleware{sensitiveFields: sensitiveFields, logger: logger} +// NewMiddleware creates a new Middleware with the given sensitive fields and logger. +func NewLoggerMiddleware(sensitiveFields []string, logger *log.Logger) *Middleware { + return &Middleware{ + sensitiveFields: sensitiveFields, + logger: logger, + bufferPool: sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + }, + } } // generateRequestID generates a unique request ID. func generateRequestID() string { - return fmt.Sprintf("%d", rand.Int63()) + genNum := new(big.Int).SetBit(new(big.Int), 63, 1) + num, err := rand.Int(rand.Reader, genNum) + if err != nil { + // Handle error appropriately in your context + panic(err) + } + return fmt.Sprintf("%d", num) } // Middleware is the actual middleware function. -func (lm *LoggerMiddleware) Middleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Start timer - startTime := time.Now() - - // Generate a unique request ID - requestID := generateRequestID() - - // Read the request body - var requestBody []byte - if r.Body != nil { - requestBody, _ = io.ReadAll(r.Body) - r.Body = io.NopCloser(bytes.NewBuffer(requestBody)) - } - // Sanitize the request body - sanitizedRequestBody := lm.sanitizeBody(requestBody) - +func (lm *Middleware) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(response http.ResponseWriter, req *http.Request) { // Create a custom response writer - responseWriter := &responseCapture{body: bytes.NewBufferString(""), writer: w} + responseWriter := &responseCapture{body: lm.bufferPool.Get().(*bytes.Buffer), writer: response} + defer lm.bufferPool.Put(responseWriter.body) + responseWriter.body.Reset() + + var ( + startTime = time.Now() + requestID = generateRequestID() + requestBody []byte + bodySize = responseWriter.body.Len() + responseBody = responseWriter.body.Bytes() + + // Get request details + clientIP = req.RemoteAddr + method = req.Method + path = req.URL.Path + userAgent = req.UserAgent() + host = req.Host + + sanitizedRequestBody = lm.sanitizeBody(requestBody) + sanitizedResponseBody = lm.sanitizeBody(responseBody) + ) + + if req.Body != nil { + requestBody, _ = io.ReadAll(req.Body) + req.Body = io.NopCloser(bytes.NewBuffer(requestBody)) + } // Process request - next.ServeHTTP(responseWriter, r) - - // Get request details - clientIP := r.RemoteAddr - method := r.Method - path := r.URL.Path - userAgent := r.UserAgent() - referer := r.Referer() - host := r.Host + next.ServeHTTP(responseWriter, req) // Get response details statusCode := responseWriter.statusCode @@ -87,15 +105,11 @@ func (lm *LoggerMiddleware) Middleware(next http.Handler) http.Handler { statusCode = http.StatusOK } - bodySize := responseWriter.body.Len() - responseBody := responseWriter.body.String() - - // Sanitize the response body - sanitizedResponseBody := lm.sanitizeBody([]byte(responseBody)) - // Write the sanitized response body to the response writer - w.Header().Set("Content-Type", "application/json") - w.Write(sanitizedResponseBody) + responseWriter.WriteHeader(statusCode) + if _, err := response.Write(sanitizedResponseBody); err != nil { + lm.logger.Printf("Error writing response: %v", err) + } // Calculate latency in milliseconds latency := time.Since(startTime).Seconds() * 1000 @@ -110,13 +124,16 @@ func (lm *LoggerMiddleware) Middleware(next http.Handler) http.Handler { "response_body": string(sanitizedResponseBody), "path": path, "user_agent": userAgent, - "referer": referer, "request_id": requestID, "host": host, "latency_ms": fmt.Sprintf("%.4fms", latency), } - logDetailsJSON, err := json.Marshal(logDetails) + // Convert logDetails to a slice of KeyValuePair + logDetailsPairs := MapToKeyValuePairs(logDetails) + + // Marshal logDetails using the custom Marshal function + logDetailsJSON, err := Marshal(logDetailsPairs) if err != nil { lm.logger.Printf("Error marshalling log details: %v", err) } else { @@ -126,9 +143,9 @@ func (lm *LoggerMiddleware) Middleware(next http.Handler) http.Handler { } // sanitizeBody removes or masks sensitive fields from the body. -func (lm *LoggerMiddleware) sanitizeBody(body []byte) []byte { +func (lm *Middleware) sanitizeBody(body []byte) []byte { var data map[string]interface{} - if err := json.Unmarshal(body, &data); err != nil { + if err := Unmarshal(body, &data); err != nil { return body } @@ -138,7 +155,8 @@ func (lm *LoggerMiddleware) sanitizeBody(body []byte) []byte { } } - sanitizedBody, err := json.Marshal(data) + sanitizedBodyPairs := MapToKeyValuePairs(data) + sanitizedBody, err := Marshal(sanitizedBodyPairs) if err != nil { return body } diff --git a/logger_test.go b/logger_test.go index 69e8f98..165a313 100644 --- a/logger_test.go +++ b/logger_test.go @@ -19,16 +19,18 @@ func TestLoggerMiddleware(t *testing.T) { lm := NewLoggerMiddleware([]string{"password", "token"}, logger) // Create a test HTTP handler - testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"message": "success", "password": "secret", "token": "12345"}`)) + if _, err := w.Write([]byte(`{"message": "success", "password": "secret", "token": "12345"}`)); err != nil { + t.Fatalf("failed to write response: %v", err) + } }) // Wrap the test handler with the middleware wrappedHandler := lm.Middleware(testHandler) // Create a test HTTP request - req := httptest.NewRequest("POST", "http://example.com/foo", strings.NewReader(`{"username": "john", "password": "secret", "token": "12345"}`)) + req := httptest.NewRequest(http.MethodPost, "http://example.com/foo", strings.NewReader(`{"username": "john", "password": "secret", "token": "12345"}`)) req.Header.Set("Content-Type", "application/json") // Create a response recorder