From b43ff91a14b0f30f656475989f6ef68368d75147 Mon Sep 17 00:00:00 2001 From: "Han Verstraete (OpenFaaS Ltd)" Date: Mon, 29 Jul 2024 18:03:06 +0200 Subject: [PATCH] Initial function builder package Signed-off-by: Han Verstraete (OpenFaaS Ltd) --- README.md | 69 ++++++++ builder/builder.go | 367 ++++++++++++++++++++++++++++++++++++++++ builder/builder_test.go | 1 + builder/copy.go | 108 ++++++++++++ builder/copy_test.go | 114 +++++++++++++ go.mod | 2 + go.sum | 2 + 7 files changed, 663 insertions(+) create mode 100644 builder/builder.go create mode 100644 builder/builder_test.go create mode 100644 builder/copy.go create mode 100644 builder/copy_test.go diff --git a/README.md b/README.md index 3f2ce3b..8232328 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,75 @@ client := sdk.NewClientWithOpts( ) ``` +## Build functions + +Use the OpenFaaS [OpenFaaS Function Builder API](https://docs.openfaas.com/openfaas-pro/builder/) to build functions from code. + +The Function Builder API provides a simple REST API to create your functions from source code. The API accepts a tar archive with the function build context and build configuration. The SDk provides methods to create this tar archive and invoke the build API. + +If your functions are using a language template you will need to make sure the required templates are available on the file system. How this is done is up to your implementation. Templates can be pulled from a git repository, copied from an S3 bucket, downloaded with an http call or fetched with the faas-cli. + +```go +functionName := "hello-world" +handler := "./hello-world" +lang := "node18" + +// Get the HMAC secret used for payload authentication with the builder API. +payloadSecret, err := os.ReadFile("payload.txt") +if err != nil { + log.Fatal(err) +} +payloadSecret := bytes.TrimSpace(payloadSecret) + +// Initialize a new builder client. +builderURL, _ := url.Parse("http://pro-builder.openfaas.svc.cluster.local") +b := builder.NewFunctionBuilder(builderURL, http.DefaultClient, builder.WithHmacAuth(string(payloadSecret))) + +// Create the function build context using the provided function handler and language template. +buildContext, err := builder.CreateBuildContext(functionName, handler, lang, []string{}) +if err != nil { + log.Fatalf("failed to create build context: %s", err) +} + +// Create a temporary file for the build tar. +tarFile, err := os.CreateTemp(os.TempDir(), "build-context-*.tar") +if err != nil { + log.Fatalf("failed to temporary file: %s", err) +} +tarFile.Close() + +tarPath := tarFile.Name() +defer os.Remove(tarPath) + +// Configuration for the build. +// Set the image name plus optional build arguments and target platforms for multi-arch images. +buildConfig := builder.BuildConfig{ + Image: image, + Platforms: []string{"linux/arm64"}, + BuildArgs: map[string]string{}, +} + +// Prepare a tar archive that contains the build config and build context. +// The function build context is a normal docker build context. Any valid folder with a Dockerfile will work. +if err := builder.MakeTar(tarPath, buildContext, &buildConfig); err != nil { + log.Fatal(err) +} + +// Invoke the function builder with the tar archive containing the build config and context +// to build and push the function image. +result, err := b.Build(tarPath) +if err != nil { + log.Fatal(err) +} + +// Print build logs +for _, logMsg := range result.Log { + fmt.Printf("%s\n", logMsg) +} +``` + +Take a look at the [function builder examples](https://github.com/openfaas/function-builder-examples) for a complete example. + ## License License: MIT diff --git a/builder/builder.go b/builder/builder.go new file mode 100644 index 0000000..6ec7e57 --- /dev/null +++ b/builder/builder.go @@ -0,0 +1,367 @@ +package builder + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/alexellis/hmac/v2" +) + +const BuilderConfigFileName = "com.openfaas.docker.config" + +type BuildConfig struct { + // Image reference. + Image string `json:"image"` + + // Extra build arguments for the Dockerfile. + BuildArgs map[string]string `json:"buildArgs,omitempty"` + + // Platforms for multi-arch builds. + Platforms []string `json:"platforms,omitempty"` +} + +type BuildResult struct { + Log []string `json:"log"` + Image string `json:"image"` + Status string `json:"status"` +} + +type FunctionBuilder struct { + // URL of the OpenFaaS Builder API. + URL *url.URL + + // Http client used for calls to the builder API. + client *http.Client + + // HMAC secret used for hashing request payloads. + hmacSecret string +} + +type BuilderOption func(*FunctionBuilder) + +// WithHmacAuth configures the HMAC secret used to sign request payloads to the builder API. +func WithHmacAuth(secret string) BuilderOption { + return func(b *FunctionBuilder) { + b.hmacSecret = secret + } +} + +// NewFunctionBuilder create a new builder for building OpenFaaS functions using the Function Builder API. +func NewFunctionBuilder(url *url.URL, client *http.Client, options ...BuilderOption) *FunctionBuilder { + b := &FunctionBuilder{ + URL: url, + + client: client, + } + + for _, option := range options { + option(b) + } + + return b +} + +// Build invokes the function builder API with the provided tar archive containing the build config and context +// to build and push a function image. +func (b *FunctionBuilder) Build(tarPath string) (BuildResult, error) { + tarFile, err := os.Open(tarPath) + if err != nil { + return BuildResult{}, err + } + defer tarFile.Close() + + tarFileBytes, err := io.ReadAll(tarFile) + if err != nil { + return BuildResult{}, err + } + + u := b.URL.JoinPath("/build") + + digest := hmac.Sign(tarFileBytes, []byte(b.hmacSecret), sha256.New) + req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewReader(tarFileBytes)) + if err != nil { + return BuildResult{}, err + } + + req.Header.Set("X-Build-Signature", "sha256="+hex.EncodeToString(digest)) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("User-Agent", "openfaas-go-sdk") + + res, err := b.client.Do(req) + if err != nil { + return BuildResult{}, err + } + + result := BuildResult{} + if res.Body != nil { + defer res.Body.Close() + + data, err := io.ReadAll(res.Body) + if err != nil { + return BuildResult{}, err + } + if err := json.Unmarshal(data, &result); err != nil { + return BuildResult{}, err + } + } + + if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted { + return result, fmt.Errorf("failed to build function, builder responded with status code %d, build status: %s", res.StatusCode, result.Status) + } + + return result, nil +} + +// MakeTar create a tar archive that contains the build config and build context. +func MakeTar(tarPath string, context string, buildConfig *BuildConfig) error { + tarFile, err := os.Create(tarPath) + if err != nil { + return err + } + + tarWriter := tar.NewWriter(tarFile) + defer tarWriter.Close() + + if err = filepath.Walk(context, func(path string, f os.FileInfo, pathErr error) error { + if pathErr != nil { + return pathErr + } + + targetFile, err := os.Open(path) + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(f, f.Name()) + if err != nil { + return err + } + + header.Name = filepath.Join("context", strings.TrimPrefix(path, context)) + header.Name = strings.TrimPrefix(header.Name, "/") + + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + if f.Mode().IsDir() { + return nil + } + + _, err = io.Copy(tarWriter, targetFile) + return err + }); err != nil { + return err + } + + configBytes, err := json.Marshal(buildConfig) + if err != nil { + return err + } + + configHeader := &tar.Header{ + Name: BuilderConfigFileName, + Mode: 0664, + Size: int64(len(configBytes)), + } + + if err := tarWriter.WriteHeader(configHeader); err != nil { + return err + } + + if _, err := tarWriter.Write(configBytes); err != nil { + return err + } + + return err +} + +const ( + DefaultTemplateDir = "./template" + DefaultTemplateHandler = "function" + DefaultBuildDir = "./build" +) + +type BuildContextOption func(*BuildContextConfig) + +type BuildContextConfig struct { + // Directory where the build context will be created. + BuildDir string + + // Directory used to lookup templates + TemplateDir string + + // Path where the function handler should be overlayed + // in the selected template + TemplateHandlerOverlay string +} + +// WithBuildDir is an option to configure the directory the build context is created in. +// If this options is not set a default path `./build` is used. +func WithBuildDir(path string) BuildContextOption { + return func(c *BuildContextConfig) { + c.BuildDir = path + } +} + +// WithTemplateDir is an option to configure the directory where the build +// template is looked up. +// If this option is not set a default path `./template` is used. +func WithTemplateDir(path string) BuildContextOption { + return func(c *BuildContextConfig) { + c.TemplateDir = path + } +} + +// WithHandlerOverlay is an option to configure the path where the function handler needs to be +// overlayed in the template. +// If this option is not set a default overlay path `function` is used. +func WithHandlerOverlay(path string) BuildContextOption { + return func(c *BuildContextConfig) { + c.TemplateHandlerOverlay = path + } +} + +// CreateBuildContext create a Docker build context using the provided function handler and language template. +// +// Parameters: +// - functionName: name of the function. +// - handler: path to the function handler. +// - language: name of the language template to use. +// - copyExtraPaths: additional paths to copy into the function handler folder. Paths should be relative to the current directory. +// Any paths outside if this directory will be skipped. +// +// By default templates are looked up in the `./template` directory. The path the the template +// directory can be overridden by setting the `builder.WithTemplateDir` option. +// CreateBuildContext overlays the function handler in the `function` folder of the template by default. +// This setting can be overridden by setting the `builder.WithHandlerOverlay` option. +// +// The function returns the path to the build context, `./build/` by default. +// The build directory can be overridden by setting the `builder.WithBuildDir` option. +// An error is returned if creating the build context fails. +func CreateBuildContext(functionName string, handler string, language string, copyExtraPaths []string, options ...BuildContextOption) (string, error) { + c := &BuildContextConfig{ + BuildDir: DefaultBuildDir, + TemplateHandlerOverlay: DefaultTemplateHandler, + TemplateDir: DefaultTemplateDir, + } + + for _, option := range options { + option(c) + } + + contextPath := path.Join(c.BuildDir, functionName) + + if err := os.RemoveAll(contextPath); err != nil { + return contextPath, fmt.Errorf("unable to clear context folder: %s", contextPath) + } + + handlerDst := contextPath + if language != "dockerfile" { + handlerDst = path.Join(contextPath, c.TemplateHandlerOverlay) + } + + permissions := defaultDirPermissions + if isRunningInCI() { + permissions = 0777 + } + + err := os.MkdirAll(handlerDst, permissions) + if err != nil { + return contextPath, fmt.Errorf("error creating function handler path %s: %w", handlerDst, err) + } + + if language != "dockerfile" { + templateSrc := path.Join(c.TemplateDir, language) + if err := copyFiles(templateSrc, contextPath); err != nil { + return contextPath, fmt.Errorf("error copying template %s: %w", language, err) + } + } + + // Overlay function handler in template. + handlerSrc := handler + infos, err := os.ReadDir(handlerSrc) + if err != nil { + return contextPath, fmt.Errorf("error reading function handler %s: %w", handlerSrc, err) + } + + for _, info := range infos { + switch info.Name() { + case "build", "template": + continue + default: + if err := copyFiles( + filepath.Clean(path.Join(handlerSrc, info.Name())), + filepath.Clean(path.Join(handlerDst, info.Name())), + ); err != nil { + return contextPath, err + } + } + } + + for _, extraPath := range copyExtraPaths { + extraPathAbs, err := pathInScope(extraPath, ".") + if err != nil { + return contextPath, err + } + // Note that if template is nil or the language is `dockerfile`, then + // handlerDest == contextPath, the docker build context, not the handler folder + // inside the docker build context. + if err := copyFiles( + extraPathAbs, + filepath.Clean(path.Join(handlerDst, extraPath)), + ); err != nil { + return contextPath, fmt.Errorf("error copying extra paths: %w", err) + } + } + + return contextPath, nil +} + +// pathInScope returns the absolute path to `path` and ensures that it is located within the +// provided scope. An error will be returned, if the path is outside of the provided scope. +func pathInScope(path string, scope string) (string, error) { + scope, err := filepath.Abs(filepath.FromSlash(scope)) + if err != nil { + return "", err + } + + abs, err := filepath.Abs(filepath.FromSlash(path)) + if err != nil { + return "", err + } + + if abs == scope { + return "", fmt.Errorf("forbidden path appears to equal the entire project: %s (%s)", path, abs) + } + + if strings.HasPrefix(abs, scope) { + return abs, nil + } + + // default return is an error + return "", fmt.Errorf("forbidden path appears to be outside of the build context: %s (%s)", path, abs) +} + +const defaultDirPermissions os.FileMode = 0700 + +// isRunningInCI checks the ENV var CI and returns true if it's set to true or 1 +func isRunningInCI() bool { + if env, ok := os.LookupEnv("CI"); ok { + if env == "true" || env == "1" { + return true + } + } + return false +} diff --git a/builder/builder_test.go b/builder/builder_test.go new file mode 100644 index 0000000..e647fb2 --- /dev/null +++ b/builder/builder_test.go @@ -0,0 +1 @@ +package builder diff --git a/builder/copy.go b/builder/copy.go new file mode 100644 index 0000000..18bb0f7 --- /dev/null +++ b/builder/copy.go @@ -0,0 +1,108 @@ +package builder + +// Copy "recursivelies copy a file object from source to dest while perserving +import ( + "fmt" + "io" + "os" + "path" + "path/filepath" +) + +// copyFiles copies files from src to destination. +func copyFiles(src, dest string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + + if info.IsDir() { + debugPrint(fmt.Sprintf("Creating directory: %s at %s", info.Name(), dest)) + return copyDir(src, dest) + } + + debugPrint(fmt.Sprintf("cp - %s %s", src, dest)) + return copyFile(src, dest) +} + +// copyDir will recursively copy a directory to dest +func copyDir(src, dest string) error { + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("error reading dest stats: %s", err.Error()) + } + + if err := os.MkdirAll(dest, info.Mode()); err != nil { + return fmt.Errorf("error creating path: %s - %s", dest, err.Error()) + } + + infos, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, info := range infos { + if err := copyFiles( + filepath.Join(src, info.Name()), + filepath.Join(dest, info.Name()), + ); err != nil { + return err + } + } + + return nil +} + +// copyFile will copy a file with the same mode as the src file +func copyFile(src, dest string) error { + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("error reading src file stats: %s", err.Error()) + } + + err = ensureBaseDir(dest) + if err != nil { + return fmt.Errorf("error creating dest base directory: %s", err.Error()) + } + + f, err := os.Create(dest) + if err != nil { + return fmt.Errorf("error creating dest file: %s", err.Error()) + } + defer f.Close() + + if err = os.Chmod(f.Name(), info.Mode()); err != nil { + return fmt.Errorf("error setting dest file mode: %s", err.Error()) + } + + s, err := os.Open(src) + if err != nil { + return fmt.Errorf("error opening src file: %s", err.Error()) + } + defer s.Close() + + _, err = io.Copy(f, s) + if err != nil { + return fmt.Errorf("Error copying dest file: %s\n" + err.Error()) + } + + return nil +} + +// ensureBaseDir creates the base directory of the given file path, if needed. +// For example, if fpath is 'build/extras/dictionary.txt`, ensureBaseDir will +// make sure that the directory `buid/extras/` is created. +func ensureBaseDir(fpath string) error { + baseDir := path.Dir(fpath) + info, err := os.Stat(baseDir) + if err == nil && info.IsDir() { + return nil + } + return os.MkdirAll(baseDir, 0755) +} + +func debugPrint(message string) { + if val, exists := os.LookupEnv("debug"); exists && (val == "1" || val == "true") { + fmt.Println(message) + } +} diff --git a/builder/copy_test.go b/builder/copy_test.go new file mode 100644 index 0000000..17215ff --- /dev/null +++ b/builder/copy_test.go @@ -0,0 +1,114 @@ +package builder + +import ( + "errors" + "fmt" + "log" + "os" + "testing" +) + +func Test_CopyFiles(t *testing.T) { + fileModes := []int{0600, 0640, 0644, 0700, 0755} + + dir := os.TempDir() + for _, mode := range fileModes { + // set up a source folder with 2 file + srcDir, srcDirErr := setupSourceFolder(2, mode) + if srcDirErr != nil { + log.Fatal("Error creating source folder") + } + defer os.RemoveAll(srcDir) + + // create a destination folder to copy the files to + destDir, destDirErr := os.MkdirTemp(dir, "openfaas-test-destination-") + if destDirErr != nil { + t.Fatalf("Error creating destination folder\n%v", destDirErr) + } + defer os.RemoveAll(destDir) + + err := copyFiles(srcDir, destDir+"/") + if err != nil { + t.Fatalf("Unexpected copy error\n%v", err) + } + + err = checkDestinationFiles(destDir, 2, mode) + if err != nil { + t.Fatalf("Destination file mode differs from source file mode\n%v", err) + } + } +} + +func Test_CopyFiles_ToDestinationWithIntermediateFolder(t *testing.T) { + dir := os.TempDir() + data := []byte("open faas") + + // create a folder for source files + srcDir, dirError := os.MkdirTemp(dir, "openfaas-test-source-") + if dirError != nil { + t.Fatalf("Error creating source folder\n%v", dirError) + } + defer os.RemoveAll(srcDir) + + // create a file inside the created folder + mode := 0600 + srcFile := fmt.Sprintf("%s/test-file-1", srcDir) + fileErr := os.WriteFile(srcFile, data, os.FileMode(mode)) + if fileErr != nil { + t.Fatalf("Error creating source file\n%v", dirError) + } + + // create a destination folder to copy the files to + destDir, destDirErr := os.MkdirTemp(dir, "openfaas-test-destination-") + if destDirErr != nil { + t.Fatalf("Error creating destination folder\n%v", destDirErr) + } + defer os.RemoveAll(destDir) + + err := copyFiles(srcFile, destDir+"/intermediate/test-file-1") + if err != nil { + t.Fatalf("Unexpected copy error\n%v", err) + } + + err = checkDestinationFiles(destDir+"/intermediate/", 1, mode) + if err != nil { + t.Fatalf("Destination file mode differs from source file mode\n%v", err) + } +} + +func setupSourceFolder(numberOfFiles, mode int) (string, error) { + dir := os.TempDir() + data := []byte("open faas") + + // create a folder for source files + srcDir, dirError := os.MkdirTemp(dir, "openfaas-test-source-") + if dirError != nil { + return "", dirError + } + + // create n files inside the created folder + for i := 1; i <= numberOfFiles; i++ { + srcFile := fmt.Sprintf("%s/test-file-%d", srcDir, i) + fileErr := os.WriteFile(srcFile, data, os.FileMode(mode)) + if fileErr != nil { + return "", fileErr + } + } + + return srcDir, nil +} + +func checkDestinationFiles(dir string, numberOfFiles, mode int) error { + // Check each file inside the destination folder + for i := 1; i <= numberOfFiles; i++ { + fileStat, err := os.Stat(fmt.Sprintf("%s/test-file-%d", dir, i)) + if os.IsNotExist(err) { + return err + } + if fileStat.Mode() != os.FileMode(mode) { + return errors.New("expected mode did not match") + } + } + + return nil +} diff --git a/go.mod b/go.mod index 0d89969..a89548b 100644 --- a/go.mod +++ b/go.mod @@ -5,3 +5,5 @@ go 1.21 require github.com/openfaas/faas-provider v0.25.4 require github.com/google/go-cmp v0.6.0 + +require github.com/alexellis/hmac/v2 v2.0.0 diff --git a/go.sum b/go.sum index cdd15bc..6dc2096 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alexellis/hmac/v2 v2.0.0 h1:/sH/UJxDXPpJorUeg2DudeKSeUrWPF32Yamw2TiDoOQ= +github.com/alexellis/hmac/v2 v2.0.0/go.mod h1:O7hZZgTfh5fp5+vAamzodZPlbw+aQK+nnrrJNHsEvL0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/openfaas/faas-provider v0.25.4 h1:Cly/M8/Q+OOn8qFxxeaZGyC5B2x4f+RSU28hej+1WcM=