diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..9be630d --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,20 @@ +name: CI +on: [push] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v1 + + - uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - run: go test -v ./... + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ap-southeast-2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f5c24f6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,20 @@ +name: Release +on: + release: + types: [created] +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + + - uses: actions/setup-go@v1 + with: + go-version: 1.13 + + - uses: goreleaser/goreleaser-action@v1 + with: + version: latest + args: release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..86306b4 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,18 @@ +builds: + - env: [CGO_ENABLED=0] + goos: [linux] + goarch: [amd64] +archives: + - name_template: lambdahttp + format: zip + files: [none*] # https://github.com/goreleaser/goreleaser/issues/602 +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..acdaf67 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,41 @@ +# lambdahttp + +Do you yearn for the days of listening on port 80(80) for web traffic? Does this +new serverless world scare you and make you wonder if you'll have to refactor +your existing code? Or maybe you have a crusty old binary lying around, have +lost the source code and are desperate to make it work on AWS Lambda? + +If so, `lambdahttp` is for **you**. + +## How do I use it? + +* Grab the latest zip from the [Releases][releases] + tab on GitHub. +* Add your app to the zip file. +* Upload the zip to S3. +* Create a Lambda function from that zip file. Configure it to use a "custom runtime" + and specify a shell command as the "handler", e.g. `./myprogram listen --port 8080`. +* Do the API Gateway / Application Load Balancer dance. +* Swim in the savings of serverless. + +If you app _doesn't_ listen on port 8080, you can specify a different one using +the `PORT` environment variable. + +Additionally, `lambdahttp` has no real way of knowing when your app is ready to +start serving traffic. For this reason, it will continuously make requests to +`/ping` until that endpoint returns a `200 OK` - this is how it knows you are +good to go. If that path doesn't work for you, specify a different one in the +`HEALTHCHECK_PATH` environment variable. + +## How does it work? + +Let me explain through the only kind of diagram I know: a sequence diagram. + +![seq-diag](/docs/seq-diag.png) + +`lambdahttp` communicates with the Lambda service using the [Lambda runtime interface][runtime] +and converts these requests into regular HTTP over TCP. It converts the responses back to the +format expected by Lambda and voilà. Nothing new you need to learn. + +[releases]: https://github.com/glassechidna/lambdahttp/releases +[runtime]: https://docs.aws.amazon.com/lambda/latest/dg/runtimes-api.html diff --git a/docs/seq-diag.png b/docs/seq-diag.png new file mode 100644 index 0000000..ccfddf2 Binary files /dev/null and b/docs/seq-diag.png differ diff --git a/docs/seq-diag.txt b/docs/seq-diag.txt new file mode 100644 index 0000000..9bf1cf3 --- /dev/null +++ b/docs/seq-diag.txt @@ -0,0 +1,16 @@ +@startuml +participant Browser +participant "API Gateway" as apigw +participant Lambda +participant lambdahttp +participant "Your app" as app + +Browser -> apigw: GET /index HTTP/1.1 +apigw -> Lambda: {"spooky": "json"...} +Lambda -> lambdahttp: (magic) +lambdahttp -> app: GET /index HTTP/1.1 +app -> lambdahttp: 200 OK ... +lambdahttp -> Lambda: (magic) +Lambda -> apigw: {"more": "aws json"...} +apigw -> Browser: 200 OK ... +@enduml diff --git a/go.mod b/go.mod index 2aedcbf..0ccfe2d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.13 require ( github.com/aws/aws-lambda-go v1.13.2 + github.com/aws/aws-sdk-go v1.25.6 + github.com/jarcoal/httpmock v1.0.4 github.com/pkg/errors v0.8.1 github.com/stretchr/testify v1.4.0 ) diff --git a/go.sum b/go.sum index 870f48b..13e6b58 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aws/aws-lambda-go v1.13.2 h1:8lYuRVn6rESoUNZXdbCmtGB4bBk4vcVYojiHjE4mMrM= github.com/aws/aws-lambda-go v1.13.2/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.25.6 h1:Rmg2pgKXoCfNe0KQb4LNSNmHqMdcgBjpMeXK9IjHWq8= +github.com/aws/aws-sdk-go v1.25.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= +github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..bc57601 --- /dev/null +++ b/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "fmt" + "github.com/glassechidna/lambdahttp/pkg/proxy" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +func main() { + port := port() + + ctx := context.Background() + cmdch := make(chan error) + go runCmd(ctx, cmdch) + + readych := make(chan error) + go waitForHealthy(ctx, readych, port) + + select { + case err := <-cmdch: + panic(fmt.Sprintf("%+v", err)) + case err := <-readych: + if err != nil { + panic(fmt.Sprintf("%+v", err)) + } + } + + go runProxy(ctx, port, cmdch) + for { + select { + case <-ctx.Done(): + panic("cancelled") + case err := <-cmdch: + panic(err) + } + } +} + +func waitForHealthy(ctx context.Context, readych chan error, port int) { + path := strings.TrimPrefix(os.Getenv("HEALTHCHECK_PATH"), "/") + if path == "" { + path = "ping" + } + + url := fmt.Sprintf("http://127.0.0.1:%d/%s", port, path) + + waitUntil(ctx, readych, func() bool { + resp, err := http.Get(url) + return err == nil && resp != nil && resp.StatusCode == 200 + }) +} + +func runProxy(ctx context.Context, port int, cmdch chan error) { + runtimeBaseUrl := os.ExpandEnv("http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01") + + proxy := proxy.New(runtimeBaseUrl, port, &http.Client{}, &http.Client{}) + for { + err := proxy.Next(ctx) + if err != nil { + cmdch <- err + } + } +} + +func port() int { + port, _ := strconv.Atoi(os.Getenv("PORT")) + if port == 0 { + port = 8080 + os.Setenv("PORT", "8080") + } + return port +} + +func runCmd(ctx context.Context, ch chan error) { + fmt.Println(os.Getwd()) + + subcmd := os.Getenv("_HANDLER") + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", subcmd) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + ch <- cmd.Run() +} + +func waitUntil(ctx context.Context, done chan error, condition func() bool) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + done <- ctx.Err() + case <-ticker.C: + if condition() { + done <- nil + } + } + } +} diff --git a/pkg/gowrap/request.go b/pkg/gowrap/request.go index 1ec4e3a..39aab97 100644 --- a/pkg/gowrap/request.go +++ b/pkg/gowrap/request.go @@ -62,7 +62,11 @@ func urlForRequest(request events.ALBTargetGroupRequest) *url.URL { query[k] = append(query[k], v) } - u, _ := url.Parse(fmt.Sprintf("%s://%s%s?%s", proto, host, path, query.Encode())) + u, err := url.Parse(fmt.Sprintf("%s://%s%s?%s", proto, host, path, query.Encode())) + if err != nil { + panic(err) + } + return u } diff --git a/pkg/proxy/e2e_test.go b/pkg/proxy/e2e_test.go new file mode 100644 index 0000000..84d7830 --- /dev/null +++ b/pkg/proxy/e2e_test.go @@ -0,0 +1,62 @@ +package proxy + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "io/ioutil" + "os" + "os/exec" + "regexp" + "strings" + "testing" +) + +func TestEndToEnd(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + cmd := exec.Command("testdata/e2e.sh") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + require.NoError(t, err) + + zip, err := ioutil.ReadFile("../../lambda.zip") + require.NoError(t, err) + + sess, err := session.NewSession(aws.NewConfig().WithRegion("ap-southeast-2")) + require.NoError(t, err) + + api := lambda.New(sess) + updateResp, err := api.UpdateFunctionCode(&lambda.UpdateFunctionCodeInput{ + FunctionName: aws.String("lambdahttptest"), + Publish: aws.Bool(true), + ZipFile: zip, + }) + require.NoError(t, err) + + payload, err := ioutil.ReadFile("testdata/alb_input.json") + require.NoError(t, err) + + invokeResp, err := api.Invoke(&lambda.InvokeInput{ + FunctionName: updateResp.FunctionArn, + Payload: payload, + }) + require.NoError(t, err) + + expected, err := ioutil.ReadFile("testdata/alb_expected_output.json") + assert.NoError(t, err) + assert.JSONEq(t, string(expected), normalizeDateInResponse(invokeResp.Payload)) +} + +func normalizeDateInResponse(payload []byte) string { + response := string(payload) + regex := regexp.MustCompile(`"Date":"([^"]+)"`) + matches := regex.FindStringSubmatch(response) + returnedDate := matches[1] + return strings.ReplaceAll(response, returnedDate, "Sun, 06 Oct 2019 06:53:36 GMT") +} diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go new file mode 100644 index 0000000..6df9a7b --- /dev/null +++ b/pkg/proxy/proxy.go @@ -0,0 +1,137 @@ +package proxy + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-lambda-go/events" + "github.com/glassechidna/lambdahttp/pkg/gowrap" + "github.com/pkg/errors" + "io/ioutil" + "net/http" + "strconv" + "strings" + "time" +) + +type Proxy struct { + runtimeBaseUrl string + localPort int + runtimeClient *http.Client + webClient *http.Client +} + +func New(runtimeBaseUrl string, localPort int, runtimeClient *http.Client, webClient *http.Client) *Proxy { + return &Proxy{ + runtimeBaseUrl: runtimeBaseUrl, + localPort: localPort, + runtimeClient: runtimeClient, + webClient: webClient, + } +} + +func (p *Proxy) Next(ctx context.Context) error { + albReq, requestId, err := p.getNextRequest(ctx) + if err != nil { + return err + } + + unixMillis, _ := strconv.ParseInt(albReq.Headers["Lambda-Runtime-Deadline-Ms"], 10, 64) + deadline := time.Unix(0, unixMillis * 1_000_000) + ctx, _ = context.WithDeadline(ctx, deadline) + + childReq := gowrap.NewHttpRequest(*albReq).WithContext(ctx) + childReq.URL.Host = fmt.Sprintf("127.0.0.1:%d", p.localPort) + childReq.URL.Scheme = "http" + + childResponse, err := p.webClient.Do(childReq) + if err != nil { + p.postLambdaError(ctx, err, requestId) + return err + } + + err = p.postLambdaResponse(ctx, childResponse, requestId) + if err != nil { + return err + } + + return nil +} + +func (p *Proxy) postLambdaError(ctx context.Context, err error, requestId string) { + respUrl := fmt.Sprintf("%s/runtime/invocation/%s/error", p.runtimeBaseUrl, requestId) + payload := strings.NewReader(fmt.Sprintf("%+v", err)) + + req, _ := http.NewRequestWithContext(ctx, "POST", respUrl, payload) + _, _ = p.runtimeClient.Do(req) +} + +func (p *Proxy) postLambdaResponse(ctx context.Context, childResponse *http.Response, requestId string) error { + respUrl := fmt.Sprintf("%s/runtime/invocation/%s/response", p.runtimeBaseUrl, requestId) + + albResp, err := gowrap.NewLambdaResponse(childResponse) + if err != nil { + return errors.WithStack(err) + } + + responseJson, err := json.Marshal(albResp) + if err != nil { + return errors.WithStack(err) + } + + runtimeReq, err := http.NewRequestWithContext(ctx, "POST", respUrl, bytes.NewReader(responseJson)) + if err != nil { + return errors.WithStack(err) + } + + _, err = p.runtimeClient.Do(runtimeReq) + return errors.WithStack(err) +} + +func (p *Proxy) getNextRequest(ctx context.Context) (*events.ALBTargetGroupRequest, string, error) { + nextUrl := fmt.Sprintf("%s/runtime/invocation/next", p.runtimeBaseUrl) + + runtimeReq, err := http.NewRequestWithContext(ctx, "GET", nextUrl, nil) + if err != nil { + return nil, "", errors.WithStack(err) + } + + runtimeResponse, err := p.runtimeClient.Do(runtimeReq) + if err != nil { + return nil, "", errors.WithStack(err) + } + + payload, err := ioutil.ReadAll(runtimeResponse.Body) + if err != nil { + return nil, "", errors.WithStack(err) + } + + albReq := events.ALBTargetGroupRequest{} + err = json.Unmarshal(payload, &albReq) + if err != nil { + return nil, "", errors.WithStack(err) + } + + for _, name := range LambdaHeaders { + val := runtimeResponse.Header.Get(name) + if albReq.Headers != nil { + albReq.Headers[name] = val + } + if albReq.MultiValueHeaders != nil { + albReq.MultiValueHeaders[name] = append(albReq.MultiValueHeaders[name], val) + } + } + + requestId := runtimeResponse.Header.Get("Lambda-Runtime-Aws-Request-Id") + return &albReq, requestId, nil +} + +var LambdaHeaders = []string{ + "Lambda-Runtime-Aws-Request-Id", + "Lambda-Runtime-Deadline-Ms", + "Lambda-Runtime-Invoked-Function-Arn", + "Lambda-Runtime-Trace-Id", + "Lambda-Runtime-Client-Context", + "Lambda-Runtime-Cognito-Identity", +} diff --git a/pkg/proxy/proxy_test.go b/pkg/proxy/proxy_test.go new file mode 100644 index 0000000..9c2a0d0 --- /dev/null +++ b/pkg/proxy/proxy_test.go @@ -0,0 +1,86 @@ +package proxy + +import ( + "bytes" + "context" + "fmt" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + "io/ioutil" + "net/http" + "net/http/httputil" + "strings" + "testing" + "time" +) + +func TestProxy(t *testing.T) { + runtimeMock := httpmock.NewMockTransport() + websiteMock := httpmock.NewMockTransport() + + proxy := New( + "http://runtime/base", + 3000, + &http.Client{Transport: runtimeMock}, + &http.Client{Transport: websiteMock}, + ) + + runtimeMock.RegisterResponder("GET", "http://runtime/base/runtime/invocation/next", func(req *http.Request) (*http.Response, error) { + resp := httpmock.NewStringResponse(200, ` + { + "requestContext": { + "elb": { + "targetGroupArn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/lambda-279XGJDqGZ5rsrHC2Fjr/49e9d65c45c6791a" + } + }, + "httpMethod": "GET", + "path": "/lambda", + "queryStringParameters": { + "query": "1234ABCD" + }, + "headers": { + "host": "lambda-alb-123578498.us-east-2.elb.amazonaws.com", + "x-forwarded-proto": "https" + }, + "body": "", + "isBase64Encoded": false + } + `) + + resp.Header.Set("Lambda-Runtime-Aws-Request-Id", "reqId") + resp.Header.Set("Lambda-Runtime-Deadline-Ms", fmt.Sprintf("%d", (time.Now().UnixNano() / 1_000_000) + 1_000)) + resp.Header.Set("Lambda-Runtime-Invoked-Function-Arn", "c") + resp.Header.Set("Lambda-Runtime-Trace-Id", "d") + resp.Header.Set("Lambda-Runtime-Client-Context", "e") + resp.Header.Set("Lambda-Runtime-Cognito-Identity", "f") + + return resp, nil + }) + + responseReturned := false + + runtimeMock.RegisterResponder("POST", "http://runtime/base/runtime/invocation/reqId/response", func(req *http.Request) (*http.Response, error) { + dump, _ := httputil.DumpRequest(req, true) + expected := ` +POST /base/runtime/invocation/reqId/response HTTP/1.1 +Host: runtime + +{"statusCode":200,"statusDescription":"200","headers":{},"multiValueHeaders":{},"body":"aGVsbG8gd29ybGQh","isBase64Encoded":true} +` + expected = strings.ReplaceAll(strings.TrimSpace(expected), "\n", "\r\n") + assert.Equal(t, expected, string(dump)) + + responseReturned = true + return &http.Response{ + StatusCode: 200, + Status: "200 OK", + Body: ioutil.NopCloser(&bytes.Buffer{}), + }, nil + }) + + websiteMock.RegisterResponder("GET", "http://127.0.0.1:3000/lambda?query=1234ABCD", httpmock.NewStringResponder(200, "hello world!")) + + err := proxy.Next(context.Background()) + assert.NoError(t, err) + assert.True(t, responseReturned) +} diff --git a/pkg/proxy/testdata/alb_expected_output.json b/pkg/proxy/testdata/alb_expected_output.json new file mode 100644 index 0000000..e4e944b --- /dev/null +++ b/pkg/proxy/testdata/alb_expected_output.json @@ -0,0 +1,22 @@ +{ + "statusCode": 200, + "statusDescription": "200 OK", + "headers": { + "Content-Length": "4", + "Content-Type": "text/plain; charset=utf-8", + "Date": "Sun, 06 Oct 2019 06:53:36 GMT" + }, + "multiValueHeaders": { + "Content-Length": [ + "4" + ], + "Content-Type": [ + "text/plain; charset=utf-8" + ], + "Date": [ + "Sun, 06 Oct 2019 06:53:36 GMT" + ] + }, + "body": "cG9uZw==", + "isBase64Encoded": true +} diff --git a/pkg/proxy/testdata/alb_input.json b/pkg/proxy/testdata/alb_input.json new file mode 100644 index 0000000..45bdde6 --- /dev/null +++ b/pkg/proxy/testdata/alb_input.json @@ -0,0 +1,25 @@ +{ + "httpMethod": "GET", + "path": "/ping", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-GB,en-US;q=0.9,en;q=0.8,pl;q=0.7", + "cache-control": "max-age=0", + "dnt": "1", + "host": "teamcityhub.biz.xero-support.com", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36", + "x-amzn-trace-id": "Root=1-5cdde3e8-16864980246acecc6d7fdd18", + "x-forwarded-for": "10.56.10.2", + "x-forwarded-port": "443", + "x-forwarded-proto": "https" + }, + "requestContext": { + "elb": { + "targetGroupArn": "arn:aws:elasticloadbalancing:ap-southeast-2:100000000000:targetgroup/teamc-Targe-SWF911AA0S7W/12487fb2099d8c01" + } + }, + "isBase64Encoded": true, + "body": "" +} diff --git a/pkg/proxy/testdata/e2e.sh b/pkg/proxy/testdata/e2e.sh new file mode 100755 index 0000000..09f57b8 --- /dev/null +++ b/pkg/proxy/testdata/e2e.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eux + +export GOOS=linux +export CGO_ENABLED=0 + +cd ../.. +go build -ldflags='-s -w' +mv lambdahttp bootstrap + +cd pkg/proxy/testdata +go build -ldflags='-s -w' +mv testdata ../../../hello.handler +cd - + +zip lambda.zip bootstrap hello.handler diff --git a/pkg/proxy/testdata/helloworld_http.go b/pkg/proxy/testdata/helloworld_http.go new file mode 100644 index 0000000..aff9c1d --- /dev/null +++ b/pkg/proxy/testdata/helloworld_http.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httputil" +) + +func main() { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + bytes, err := httputil.DumpRequest(r, true) + fmt.Println(string(bytes), err) + w.Write([]byte(`pong`)) + }) + + panic(http.ListenAndServe(":8080", nil)) +}