From 9f723f6a2db7eea0a97b3b94b98318d33019b3df Mon Sep 17 00:00:00 2001 From: gotwarlost Date: Wed, 9 Sep 2020 14:30:29 -0700 Subject: [PATCH] add support for importing a glob of files with either import or importstr semantics This change introduces the ability to import a bag of data or libsonnet files. It provides the following import syntax: import 'glob-import:*.json' and import 'glob-importstr:*.yaml' The first form looks for all files that match the wildcard and returns an object keyed by relative file name with values importing the actual files. i.e. something like: { 'a.json': import 'a.json', 'b.json': import 'b.json', } The second form is very similar except that it uses 'importstr' for inner imports. The returned code looks like: { 'a.yaml': importstr 'a.yaml', 'b.yaml': importstr 'b.yaml', } This allows for loading optional overlay files, user data files and such without having to write out an import statement in the calling code for each file imported. This importer is enabled by default in qbec. --- .../{base.libsonnet => _.libsonnet} | 0 examples/test-app/environments/dev.libsonnet | 2 +- examples/test-app/environments/prod.libsonnet | 2 +- examples/test-app/lib/globutil.libsonnet | 61 ++++++ examples/test-app/params.libsonnet | 13 +- internal/vm/importers/api.go | 10 + internal/vm/importers/composite.go | 28 +++ internal/vm/importers/composite_test.go | 36 ++++ internal/vm/importers/file.go | 18 ++ internal/vm/importers/glob.go | 180 ++++++++++++++++++ internal/vm/importers/glob_test.go | 151 +++++++++++++++ .../vm/importers/testdata/example1/a.json | 3 + .../vm/importers/testdata/example1/b.json | 3 + .../vm/importers/testdata/example1/z.json | 3 + .../importers/testdata/example2/inc1/a.json | 3 + .../importers/testdata/example2/inc2/a.json | 3 + internal/vm/vm.go | 13 +- 17 files changed, 515 insertions(+), 14 deletions(-) rename examples/test-app/environments/{base.libsonnet => _.libsonnet} (100%) create mode 100644 examples/test-app/lib/globutil.libsonnet create mode 100644 internal/vm/importers/api.go create mode 100644 internal/vm/importers/composite.go create mode 100644 internal/vm/importers/composite_test.go create mode 100644 internal/vm/importers/file.go create mode 100644 internal/vm/importers/glob.go create mode 100644 internal/vm/importers/glob_test.go create mode 100644 internal/vm/importers/testdata/example1/a.json create mode 100644 internal/vm/importers/testdata/example1/b.json create mode 100644 internal/vm/importers/testdata/example1/z.json create mode 100644 internal/vm/importers/testdata/example2/inc1/a.json create mode 100644 internal/vm/importers/testdata/example2/inc2/a.json diff --git a/examples/test-app/environments/base.libsonnet b/examples/test-app/environments/_.libsonnet similarity index 100% rename from examples/test-app/environments/base.libsonnet rename to examples/test-app/environments/_.libsonnet diff --git a/examples/test-app/environments/dev.libsonnet b/examples/test-app/environments/dev.libsonnet index 19d51695..d3f8c88b 100644 --- a/examples/test-app/environments/dev.libsonnet +++ b/examples/test-app/environments/dev.libsonnet @@ -1,4 +1,4 @@ -local base = import './base.libsonnet'; +local base = import '_.libsonnet'; base { components +: { diff --git a/examples/test-app/environments/prod.libsonnet b/examples/test-app/environments/prod.libsonnet index 8ea8ed7c..12ab5f0c 100644 --- a/examples/test-app/environments/prod.libsonnet +++ b/examples/test-app/environments/prod.libsonnet @@ -1,4 +1,4 @@ -local base = import './base.libsonnet'; +local base = import '_.libsonnet'; base { components +: { diff --git a/examples/test-app/lib/globutil.libsonnet b/examples/test-app/lib/globutil.libsonnet new file mode 100644 index 00000000..cca87cb7 --- /dev/null +++ b/examples/test-app/lib/globutil.libsonnet @@ -0,0 +1,61 @@ +local len = std.length; +local split = std.split; +local join = std.join; + +// keepDirs returns a key mapping function for the number of directories to be retained +local keepDirs = function(num=0) function(s) ( + if num < 0 + then + s + else ( + local elems = split(s, '/'); + local preserveRight = num + 1; + if len(elems) <= preserveRight + then + s + else ( + local remove = len(elems) - preserveRight; + join('/', elems[remove:]) + ) + ) +); + +// stripExtension is a key mapping function that strips the file extension from the key +local stripExtension = function(s) ( + local parts = split(s, '/'); + local dirs = parts[:len(parts) - 1]; + local file = parts[len(parts) - 1]; + local fileParts = split(file, '.'); + local fixed = if len(fileParts) == 1 then file else join('.', fileParts[:len(fileParts) - 1]); + join('/', dirs + [fixed]) +); + +// compose composes an array of map functions by applying them in sequence +local compose = function(arr) function(s) std.foldl(function(prev, fn) fn(prev), arr, s); + +// transform transforms an object, mapping keys using the key mapper and values using the valueMapper. +// It ensures that the key mapping does not produce duplicate keys. +local transform = function(globObject, keyMapper=function(s) s, valueMapper=function(o) o) ( + local keys = std.objectFields(globObject); + std.foldl(function(obj, key) ( + local mKey = keyMapper(key); + local val = globObject[key]; + if std.objectHas(obj, mKey) + then + error 'multiple keys map to the same value: %s' % [mKey] + else + obj { [mKey]: valueMapper(val) } + ), keys, {}) +); + +// nameOnly is a key mapper that removes all directories and strips extensions from file names, +// syntax sugar for the common case. +local nameOnly = compose([keepDirs(0), stripExtension]); + +{ + transform:: transform, + keepDirs:: keepDirs, + stripExtension:: stripExtension, + compose:: compose, + nameOnly:: nameOnly, +} diff --git a/examples/test-app/params.libsonnet b/examples/test-app/params.libsonnet index c5afcdfa..ef9f0569 100644 --- a/examples/test-app/params.libsonnet +++ b/examples/test-app/params.libsonnet @@ -1,10 +1,5 @@ -local p = { - _: import './environments/base.libsonnet', - dev: import './environments/dev.libsonnet', - prod: import './environments/prod.libsonnet', -}; - -local env = std.extVar('qbec.io/env'); - -if std.objectHas(p, env) then p[env] else error 'Environment ' + env + ' not defined in ' + std.thisFile +local globutil = import 'globutil.libsonnet'; +local p = globutil.transform(import 'glob-import:environments/*.libsonnet', globutil.nameOnly); +local key = std.extVar('qbec.io/env'); +if std.objectHas(p, key) then p[key] else error 'Environment ' + key + ' not defined in environments/' diff --git a/internal/vm/importers/api.go b/internal/vm/importers/api.go new file mode 100644 index 00000000..68aad8aa --- /dev/null +++ b/internal/vm/importers/api.go @@ -0,0 +1,10 @@ +package importers + +import "github.com/google/go-jsonnet" + +// ExtendedImporter extends the jsonnet importer interface to add a new method that can determine whether +// an importer can be used for a path. +type ExtendedImporter interface { + jsonnet.Importer + CanProcess(path string) bool +} diff --git a/internal/vm/importers/composite.go b/internal/vm/importers/composite.go new file mode 100644 index 00000000..95834415 --- /dev/null +++ b/internal/vm/importers/composite.go @@ -0,0 +1,28 @@ +package importers + +import ( + "fmt" + + "github.com/google/go-jsonnet" +) + +// CompositeImporter tries multiple extended importers in sequence for a given path +type CompositeImporter struct { + importers []ExtendedImporter +} + +// NewCompositeImporter creates a composite importer with the supplied extended importers. Note that if +// two importers could match the same path, the first one will be used so order is important. +func NewCompositeImporter(importers ...ExtendedImporter) *CompositeImporter { + return &CompositeImporter{importers: importers} +} + +// Import implements the interface method by delegating to installed importers in sequence +func (c *CompositeImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { + for _, importer := range c.importers { + if importer.CanProcess(importedPath) { + return importer.Import(importedFrom, importedPath) + } + } + return contents, foundAt, fmt.Errorf("no importer for path %s", importedPath) +} diff --git a/internal/vm/importers/composite_test.go b/internal/vm/importers/composite_test.go new file mode 100644 index 00000000..55183818 --- /dev/null +++ b/internal/vm/importers/composite_test.go @@ -0,0 +1,36 @@ +package importers + +import ( + "testing" + + "github.com/google/go-jsonnet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCompositeImporter(t *testing.T) { + vm := jsonnet.MakeVM() + vm.Importer( + NewCompositeImporter( + NewGlobImporter("import"), + NewGlobImporter("importstr"), + NewFileImporter(&jsonnet.FileImporter{}), + ), + ) + _, err := vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import '../a.json'`) + require.NoError(t, err) + + _, err = vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import 'glob-import:../*.json'`) + require.NoError(t, err) + + vm = jsonnet.MakeVM() + vm.Importer( + NewCompositeImporter( + NewGlobImporter("import"), + NewGlobImporter("importstr"), + ), + ) + _, err = vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import '../bag-of-files/a.json'`) + require.Error(t, err) + assert.Contains(t, err.Error(), "RUNTIME ERROR: no importer for path ../bag-of-files/a.json") +} diff --git a/internal/vm/importers/file.go b/internal/vm/importers/file.go new file mode 100644 index 00000000..a518e4ce --- /dev/null +++ b/internal/vm/importers/file.go @@ -0,0 +1,18 @@ +package importers + +import "github.com/google/go-jsonnet" + +// NewFileImporter creates an extended file importer, wrapping the one supplied. +func NewFileImporter(jfi *jsonnet.FileImporter) *ExtendedFileImporter { + return &ExtendedFileImporter{FileImporter: jfi} +} + +// ExtendedFileImporter wraps a file importer and declares that it can import any path. +type ExtendedFileImporter struct { + *jsonnet.FileImporter +} + +// CanProcess implements the interface method. +func (e *ExtendedFileImporter) CanProcess(_ string) bool { + return true +} diff --git a/internal/vm/importers/glob.go b/internal/vm/importers/glob.go new file mode 100644 index 00000000..b7d77d36 --- /dev/null +++ b/internal/vm/importers/glob.go @@ -0,0 +1,180 @@ +package importers + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "fmt" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/google/go-jsonnet" +) + +// globCacheKey is the key to use for the importer cache. Two entries are equivalent if +// they resolve to the same set of files, have the same relative path to access them, and +// the same inner verb for access. +type globCacheKey struct { + verb string // the inner verb used for import + resolved string // the glob pattern as resolved from the current working directory + relative string // the glob pattern as specified by the user +} + +var separator = []byte{0} + +// file returns a virtual file name as represented by this cache key, relative to the supplied base directory. +func (c globCacheKey) file(baseDir string) string { + h := sha256.New() + h.Write([]byte(c.verb)) + h.Write(separator) + h.Write([]byte(c.resolved)) + h.Write(separator) + h.Write([]byte(c.relative)) + baseName := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + fileName := fmt.Sprintf("%s-%s.glob", baseName, c.verb) + return filepath.Join(baseDir, fileName) +} + +// globEntry is a cache entry for a glob path resolved from the current working directory. +type globEntry struct { + contents jsonnet.Contents + foundAt string + err error +} + +// GlobImporter provides facilities to import a bag of files using a glob pattern. Note that it will NOT +// honor any library paths and must be exactly resolved from the caller's location. It is initialized with +// a verb that configures how the inner imports are done (i.e. `import` or `importstr`) and it processes +// paths that start with `glob-{verb}:` +// +// After the marker prefix is stripped, the input is treated as a file pattern that is resolved using Go's glob functionality. +// The return value is an object that is keyed by file names relative to the import location with values +// importing the contents of the file. +// +// That is, given the following directory structure: +// +// lib +// - a.json +// - b.json +// caller +// - c.libsonnet +// +// where c.libsonnet has the following contents +// +// import 'glob-import:../lib/*.json' +// +// evaluating `c.libsonnet` will return jsonnet code of the following form: +// +// { +// '../lib/a.json': import '../lib/a.json', +// '../lib/b.json': import '../lib/b.json', +// } +// +type GlobImporter struct { + innerVerb string + prefix string + cache map[globCacheKey]*globEntry +} + +// NewGlobImporter creates a glob importer. +func NewGlobImporter(innerVerb string) *GlobImporter { + if !(innerVerb == "import" || innerVerb == "importstr") { + panic("invalid inner verb " + innerVerb + " for glob importer") + } + return &GlobImporter{ + innerVerb: innerVerb, + prefix: fmt.Sprintf("glob-%s:", innerVerb), + cache: map[globCacheKey]*globEntry{}, + } +} + +func (g *GlobImporter) cacheKey(resolved, relative string) globCacheKey { + return globCacheKey{ + verb: g.innerVerb, + resolved: resolved, + relative: relative, + } +} + +// getEntry returns an entry from the cache or nil, if not found. +func (g *GlobImporter) getEntry(resolved, relative string) *globEntry { + ret := g.cache[g.cacheKey(resolved, relative)] + return ret +} + +// setEntry sets the cache entry for the supplied path. +func (g *GlobImporter) setEntry(resolved, relative string, e *globEntry) { + g.cache[g.cacheKey(resolved, relative)] = e +} + +// CanProcess implements the interface method, returning true for paths that start with the string "glob:" +func (g *GlobImporter) CanProcess(path string) bool { + return strings.HasPrefix(path, g.prefix) +} + +// Import implements the interface method. +func (g *GlobImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { + // baseDir is the directory from which things are relatively imported + baseDir, _ := path.Split(importedFrom) + + relativeGlob := strings.TrimPrefix(importedPath, g.prefix) + + if strings.HasPrefix(relativeGlob, "/") { + return contents, foundAt, fmt.Errorf("invalid glob pattern '%s', cannot be absolute", relativeGlob) + } + + baseDir = filepath.FromSlash(baseDir) + relativeGlob = filepath.FromSlash(relativeGlob) + + // globPath is the glob path relative to the working directory + globPath := filepath.Clean(filepath.Join(baseDir, relativeGlob)) + r := g.getEntry(globPath, relativeGlob) + if r != nil { + return r.contents, r.foundAt, r.err + } + + // once we have successfully gotten a glob path, we can store results in the cache + defer func() { + g.setEntry(globPath, relativeGlob, &globEntry{ + contents: contents, + foundAt: foundAt, + err: err, + }) + }() + + matches, err := filepath.Glob(globPath) + if err != nil { + return contents, foundAt, fmt.Errorf("unable to expand glob %q, %v", globPath, err) + } + + // convert matches to be relative to our baseDir + var relativeMatches []string + for _, m := range matches { + rel, err := filepath.Rel(baseDir, m) + if err != nil { + return contents, globPath, fmt.Errorf("could not resolve %s from %s", m, importedFrom) + } + relativeMatches = append(relativeMatches, rel) + } + + // ensure consistent order (not strictly required, makes it human friendly) + sort.Strings(relativeMatches) + + var out bytes.Buffer + out.WriteString("{\n") + for _, file := range relativeMatches { + file = filepath.ToSlash(file) + out.WriteString("\t") + _, _ = fmt.Fprintf(&out, `'%s': %s '%s',`, file, g.innerVerb, file) + out.WriteString("\n") + } + out.WriteString("}") + k := g.cacheKey(globPath, relativeGlob) + output := out.String() + + return jsonnet.MakeContents(output), + filepath.ToSlash(k.file(baseDir)), + nil +} diff --git a/internal/vm/importers/glob_test.go b/internal/vm/importers/glob_test.go new file mode 100644 index 00000000..3bf40f4a --- /dev/null +++ b/internal/vm/importers/glob_test.go @@ -0,0 +1,151 @@ +package importers + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-jsonnet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func makeVM() (*jsonnet.VM, *GlobImporter) { + vm := jsonnet.MakeVM() + g1 := NewGlobImporter("import") + g2 := NewGlobImporter("importstr") + vm.Importer( + NewCompositeImporter( + g1, + g2, + NewFileImporter(&jsonnet.FileImporter{}), + ), + ) + return vm, g1 +} + +type outputData map[string]interface{} + +func evaluateVirtual(t *testing.T, vm *jsonnet.VM, virtFile string, code string) outputData { + jsonStr, err := vm.EvaluateSnippet(virtFile, code) + require.NoError(t, err) + t.Logf("input from '%s'\n%s\noutput:%s\n", virtFile, code, jsonStr) + var data outputData + err = json.Unmarshal([]byte(jsonStr), &data) + require.NoError(t, err) + return data +} + +func evaluateVirtualErr(t *testing.T, virtFile string, code string) error { + vm, _ := makeVM() + _, err := vm.EvaluateSnippet(virtFile, code) + require.Error(t, err) + return err +} + +func TestGlobSimple(t *testing.T) { + vm, _ := makeVM() + data := evaluateVirtual(t, vm, "testdata/caller.jsonnet", `import 'glob-import:example1/*.json'`) + for _, k := range []string{"a", "b", "z"} { + relFile := fmt.Sprintf("example1/%s.json", k) + val, ok := data[relFile] + require.True(t, ok) + mVal, ok := val.(map[string]interface{}) + require.True(t, ok) + _, ok = mVal[k] + assert.True(t, ok) + } +} + +func TestDuplicateFileName(t *testing.T) { + vm, _ := makeVM() + data := evaluateVirtual(t, vm, "testdata/example2/caller.jsonnet", `import 'glob-import:inc?/*.json'`) + _, firstOk := data["inc1/a.json"] + require.True(t, firstOk) + _, secondOk := data["inc2/a.json"] + require.True(t, secondOk) +} + +func TestGlobNoMatch(t *testing.T) { + vm, _ := makeVM() + data := evaluateVirtual(t, vm, "testdata/example1/caller/no-match.jsonnet", `import 'glob-import:*.json'`) + require.Equal(t, 0, len(data)) +} + +func TestGlobImportStr(t *testing.T) { + vm, _ := makeVM() + data, err := vm.EvaluateSnippet("testdata/example1/caller/synthesized.jsonnet", `importstr 'glob-import:../*.json'`) + require.NoError(t, err) + var str string + err = json.Unmarshal([]byte(data), &str) + require.NoError(t, err) + assert.Equal(t, `{ + '../a.json': import '../a.json', + '../b.json': import '../b.json', + '../z.json': import '../z.json', +}`, str) +} + +func TestGlobImportStrVerb(t *testing.T) { + vm, _ := makeVM() + data := evaluateVirtual(t, vm, "testdata/example1/caller/synthesized.jsonnet", `import 'glob-importstr:../*.json'`) + for _, k := range []string{"a", "b", "z"} { + val, ok := data[fmt.Sprintf("../%s.json", k)] + require.True(t, ok) + mVal, ok := val.(string) + require.True(t, ok) + assert.Contains(t, mVal, fmt.Sprintf("%q", k)) + } +} + +func TestGlobInternalCaching(t *testing.T) { + a := assert.New(t) + vm, gi := makeVM() + _ = evaluateVirtual(t, vm, "testdata/example1/caller/synthesized.jsonnet", `import 'glob-import:../*.json'`) + a.Equal(1, len(gi.cache)) + _ = evaluateVirtual(t, vm, "testdata/example1/caller/synthesized2.jsonnet", `import 'glob-import:../*.json'`) + a.Equal(1, len(gi.cache)) + _ = evaluateVirtual(t, vm, "testdata/example1/caller2/synthesized.jsonnet", `import 'glob-import:../*.json'`) + a.Equal(1, len(gi.cache)) + _ = evaluateVirtual(t, vm, "testdata/example1/caller/inner/synthesized.jsonnet", `import 'glob-import:../../*.json'`) + a.Equal(2, len(gi.cache)) + _ = evaluateVirtual(t, vm, "testdata/example1/caller/inner/synthesized.jsonnet", `import 'glob-import:../../[a,b].json'`) + a.Equal(3, len(gi.cache)) +} + +func TestGlobNegativeCases(t *testing.T) { + checkMsg := func(m string) func(t *testing.T, err error) { + return func(t *testing.T, err error) { + assert.Contains(t, err.Error(), m) + } + } + tests := []struct { + name string + expr string + asserter func(t *testing.T, err error) + }{ + { + name: "bad path", + expr: `import 'glob-import:/bag-of-files/*.json'`, + asserter: checkMsg(`RUNTIME ERROR: invalid glob pattern '/bag-of-files/*.json', cannot be absolute`), + }, + { + name: "bad pattern", + expr: `import 'glob-import:../[.json'`, + asserter: func(t *testing.T, err error) { + assert.Contains(t, err.Error(), `RUNTIME ERROR: unable to expand glob`) + assert.Contains(t, err.Error(), `[.json", syntax error in pattern`) + }, + }, + } + file := "testdata/example1/caller/synthesized.jsonnet" + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.asserter(t, evaluateVirtualErr(t, file, test.expr)) + }) + } +} + +func TestGlobInit(t *testing.T) { + require.Panics(t, func() { _ = NewGlobImporter("foobar") }) +} diff --git a/internal/vm/importers/testdata/example1/a.json b/internal/vm/importers/testdata/example1/a.json new file mode 100644 index 00000000..09d92d0c --- /dev/null +++ b/internal/vm/importers/testdata/example1/a.json @@ -0,0 +1,3 @@ +{ + "a": "the first letter of the alphabet" +} diff --git a/internal/vm/importers/testdata/example1/b.json b/internal/vm/importers/testdata/example1/b.json new file mode 100644 index 00000000..3d373b08 --- /dev/null +++ b/internal/vm/importers/testdata/example1/b.json @@ -0,0 +1,3 @@ +{ + "b": "the second letter of the alphabet" +} diff --git a/internal/vm/importers/testdata/example1/z.json b/internal/vm/importers/testdata/example1/z.json new file mode 100644 index 00000000..1079a143 --- /dev/null +++ b/internal/vm/importers/testdata/example1/z.json @@ -0,0 +1,3 @@ +{ + "z": "the last letter of the alphabet" +} diff --git a/internal/vm/importers/testdata/example2/inc1/a.json b/internal/vm/importers/testdata/example2/inc1/a.json new file mode 100644 index 00000000..97f7e168 --- /dev/null +++ b/internal/vm/importers/testdata/example2/inc1/a.json @@ -0,0 +1,3 @@ +{ + "a": "a" +} diff --git a/internal/vm/importers/testdata/example2/inc2/a.json b/internal/vm/importers/testdata/example2/inc2/a.json new file mode 100644 index 00000000..b42d34e3 --- /dev/null +++ b/internal/vm/importers/testdata/example2/inc2/a.json @@ -0,0 +1,3 @@ +{ + "a": "long form a" +} diff --git a/internal/vm/vm.go b/internal/vm/vm.go index e00548e2..2cd3795e 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -28,6 +28,7 @@ import ( "github.com/google/go-jsonnet" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/splunk/qbec/internal/vm/importers" ) // Config is the desired configuration of the Jsonnet VM. @@ -353,9 +354,15 @@ func New(config Config) *VM { if config.importer != nil { vm.Importer(config.importer) } else { - vm.Importer(&jsonnet.FileImporter{ - JPaths: config.libPaths, - }) + vm.Importer( + importers.NewCompositeImporter( + importers.NewGlobImporter("import"), + importers.NewGlobImporter("importstr"), + importers.NewFileImporter(&jsonnet.FileImporter{ + JPaths: config.libPaths, + }), + ), + ) } return &VM{VM: vm, config: config} }