add support for importing a glob of files with either import or importstr semantics

examples/test-app/environments/dev.libsonnet

@@ -1,4 +1,4 @@
-local base = import './base.libsonnet';
+local base = import '_.libsonnet';
 
 base {
     components +: {

examples/test-app/environments/prod.libsonnet

@@ -1,4 +1,4 @@
-local base = import './base.libsonnet';
+local base = import '_.libsonnet';
 
 base {
     components +: {

examples/test-app/lib/globutil.libsonnet

@@ -0,0 +1,61 @@
+local len = std.length;
+local split = std.split;
+local join = std.join;
+
+// keepDirs returns a key mapping function for the number of directories to be retained
+local keepDirs = function(num=0) function(s) (
+  if num < 0
+  then
+    s
+  else (
+    local elems = split(s, '/');
+    local preserveRight = num + 1;
+    if len(elems) <= preserveRight
+    then
+      s
+    else (
+      local remove = len(elems) - preserveRight;
+      join('/', elems[remove:])
+    )
+  )
+);
+
+// stripExtension is a key mapping function that strips the file extension from the key
+local stripExtension = function(s) (
+  local parts = split(s, '/');
+  local dirs = parts[:len(parts) - 1];
+  local file = parts[len(parts) - 1];
+  local fileParts = split(file, '.');
+  local fixed = if len(fileParts) == 1 then file else join('.', fileParts[:len(fileParts) - 1]);
+  join('/', dirs + [fixed])
+);
+
+// compose composes an array of map functions by applying them in sequence
+local compose = function(arr) function(s) std.foldl(function(prev, fn) fn(prev), arr, s);
+
+// transform transforms an object, mapping keys using the key mapper and values using the valueMapper.
+// It ensures that the key mapping does not produce duplicate keys.
+local transform = function(globObject, keyMapper=function(s) s, valueMapper=function(o) o) (
+  local keys = std.objectFields(globObject);
+  std.foldl(function(obj, key) (
+    local mKey = keyMapper(key);
+    local val = globObject[key];
+    if std.objectHas(obj, mKey)
+    then
+      error 'multiple keys map to the same value: %s' % [mKey]
+    else
+      obj { [mKey]: valueMapper(val) }
+  ), keys, {})
+);
+
+// nameOnly is a key mapper that removes all directories and strips extensions from file names,
+// syntax sugar for the common case.
+local nameOnly = compose([keepDirs(0), stripExtension]);
+
+{
+  transform:: transform,
+  keepDirs:: keepDirs,
+  stripExtension:: stripExtension,
+  compose:: compose,
+  nameOnly:: nameOnly,
+}

examples/test-app/params.libsonnet

@@ -1,10 +1,5 @@
-local p = {
-    _: import './environments/base.libsonnet',
-    dev: import './environments/dev.libsonnet',
-    prod: import './environments/prod.libsonnet',
-};
-
-local env = std.extVar('qbec.io/env');
-
-if std.objectHas(p, env) then p[env] else error 'Environment ' + env + ' not defined in ' + std.thisFile
+local globutil = import 'globutil.libsonnet';
+local p = globutil.transform(import 'glob-import:environments/*.libsonnet', globutil.nameOnly);
 
+local key = std.extVar('qbec.io/env');
+if std.objectHas(p, key) then p[key] else error 'Environment ' + key + ' not defined in environments/'

internal/vm/importers/api.go

@@ -0,0 +1,10 @@
+package importers
+
+import "github.com/google/go-jsonnet"
+
+// ExtendedImporter extends the jsonnet importer interface to add a new method that can determine whether
+// an importer can be used for a path.
+type ExtendedImporter interface {
+	jsonnet.Importer
+	CanProcess(path string) bool
+}

internal/vm/importers/composite.go

@@ -0,0 +1,28 @@
+package importers
+
+import (
+	"fmt"
+
+	"github.com/google/go-jsonnet"
+)
+
+// CompositeImporter tries multiple extended importers in sequence for a given path
+type CompositeImporter struct {
+	importers []ExtendedImporter
+}
+
+// NewCompositeImporter creates a composite importer with the supplied extended importers. Note that if
+// two importers could match the same path, the first one will be used so order is important.
+func NewCompositeImporter(importers ...ExtendedImporter) *CompositeImporter {
+	return &CompositeImporter{importers: importers}
+}
+
+// Import implements the interface method by delegating to installed importers in sequence
+func (c *CompositeImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) {
+	for _, importer := range c.importers {
+		if importer.CanProcess(importedPath) {
+			return importer.Import(importedFrom, importedPath)
+		}
+	}
+	return contents, foundAt, fmt.Errorf("no importer for path %s", importedPath)
+}

internal/vm/importers/composite_test.go

@@ -0,0 +1,36 @@
+package importers
+
+import (
+	"testing"
+
+	"github.com/google/go-jsonnet"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompositeImporter(t *testing.T) {
+	vm := jsonnet.MakeVM()
+	vm.Importer(
+		NewCompositeImporter(
+			NewGlobImporter("import"),
+			NewGlobImporter("importstr"),
+			NewFileImporter(&jsonnet.FileImporter{}),
+		),
+	)
+	_, err := vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import '../a.json'`)
+	require.NoError(t, err)
+
+	_, err = vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import 'glob-import:../*.json'`)
+	require.NoError(t, err)
+
+	vm = jsonnet.MakeVM()
+	vm.Importer(
+		NewCompositeImporter(
+			NewGlobImporter("import"),
+			NewGlobImporter("importstr"),
+		),
+	)
+	_, err = vm.EvaluateSnippet("testdata/example1/caller/caller.json", `import '../bag-of-files/a.json'`)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "RUNTIME ERROR: no importer for path ../bag-of-files/a.json")
+}

internal/vm/importers/file.go

@@ -0,0 +1,18 @@
+package importers
+
+import "github.com/google/go-jsonnet"
+
+// NewFileImporter creates an extended file importer, wrapping the one supplied.
+func NewFileImporter(jfi *jsonnet.FileImporter) *ExtendedFileImporter {
+	return &ExtendedFileImporter{FileImporter: jfi}
+}
+
+// ExtendedFileImporter wraps a file importer and declares that it can import any path.
+type ExtendedFileImporter struct {
+	*jsonnet.FileImporter
+}
+
+// CanProcess implements the interface method.
+func (e *ExtendedFileImporter) CanProcess(_ string) bool {
+	return true
+}

internal/vm/importers/glob.go

@@ -0,0 +1,180 @@
+package importers
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/google/go-jsonnet"
+)
+
+// globCacheKey is the key to use for the importer cache. Two entries are equivalent if
+// they resolve to the same set of files, have the same relative path to access them, and
+// the same inner verb for access.
+type globCacheKey struct {
+	verb     string // the inner verb used for import
+	resolved string // the glob pattern as resolved from the current working directory
+	relative string // the glob pattern as specified by the user
+}
+
+var separator = []byte{0}
+
+// file returns a virtual file name as represented by this cache key, relative to the supplied base directory.
+func (c globCacheKey) file(baseDir string) string {
+	h := sha256.New()
+	h.Write([]byte(c.verb))
+	h.Write(separator)
+	h.Write([]byte(c.resolved))
+	h.Write(separator)
+	h.Write([]byte(c.relative))
+	baseName := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
+	fileName := fmt.Sprintf("%s-%s.glob", baseName, c.verb)
+	return filepath.Join(baseDir, fileName)
+}
+
+// globEntry is a cache entry for a glob path resolved from the current working directory.
+type globEntry struct {
+	contents jsonnet.Contents
+	foundAt  string
+	err      error
+}
+
+// GlobImporter provides facilities to import a bag of files using a glob pattern. Note that it will NOT
+// honor any library paths and must be exactly resolved from the caller's location. It is initialized with
+// a verb that configures how the inner imports are done (i.e. `import` or `importstr`) and it processes
+// paths that start with `glob-{verb}:`
+//
+// After the marker prefix is stripped, the input is treated as a file pattern that is resolved using Go's glob functionality.
+// The return value is an object that is keyed by file names relative to the import location with values
+// importing the contents of the file.
+//
+// That is, given the following directory structure:
+//
+// 		lib
+//			- a.json
+//			- b.json
+//		caller
+//			- c.libsonnet
+//
+// where c.libsonnet has the following contents
+//
+//		import 'glob-import:../lib/*.json'
+//
+// evaluating `c.libsonnet` will return jsonnet code of the following form:
+//
+//		{
+//			'../lib/a.json': import '../lib/a.json',
+//			'../lib/b.json': import '../lib/b.json',
+//		}
+//
+type GlobImporter struct {
+	innerVerb string
+	prefix    string
+	cache     map[globCacheKey]*globEntry
+}
+
+// NewGlobImporter creates a glob importer.
+func NewGlobImporter(innerVerb string) *GlobImporter {
+	if !(innerVerb == "import" || innerVerb == "importstr") {
+		panic("invalid inner verb " + innerVerb + " for glob importer")
+	}
+	return &GlobImporter{
+		innerVerb: innerVerb,
+		prefix:    fmt.Sprintf("glob-%s:", innerVerb),
+		cache:     map[globCacheKey]*globEntry{},
+	}
+}
+
+func (g *GlobImporter) cacheKey(resolved, relative string) globCacheKey {
+	return globCacheKey{
+		verb:     g.innerVerb,
+		resolved: resolved,
+		relative: relative,
+	}
+}
+
+// getEntry returns an entry from the cache or nil, if not found.
+func (g *GlobImporter) getEntry(resolved, relative string) *globEntry {
+	ret := g.cache[g.cacheKey(resolved, relative)]
+	return ret
+}
+
+// setEntry sets the cache entry for the supplied path.
+func (g *GlobImporter) setEntry(resolved, relative string, e *globEntry) {
+	g.cache[g.cacheKey(resolved, relative)] = e
+}
+
+// CanProcess implements the interface method, returning true for paths that start with the string "glob:"
+func (g *GlobImporter) CanProcess(path string) bool {
+	return strings.HasPrefix(path, g.prefix)
+}
+
+// Import implements the interface method.
+func (g *GlobImporter) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) {
+	// baseDir is the directory from which things are relatively imported
+	baseDir, _ := path.Split(importedFrom)
+
+	relativeGlob := strings.TrimPrefix(importedPath, g.prefix)
+
+	if strings.HasPrefix(relativeGlob, "/") {
+		return contents, foundAt, fmt.Errorf("invalid glob pattern '%s', cannot be absolute", relativeGlob)
+	}
+
+	baseDir = filepath.FromSlash(baseDir)
+	relativeGlob = filepath.FromSlash(relativeGlob)
+
+	// globPath is the glob path relative to the working directory
+	globPath := filepath.Clean(filepath.Join(baseDir, relativeGlob))
+	r := g.getEntry(globPath, relativeGlob)
+	if r != nil {
+		return r.contents, r.foundAt, r.err
+	}
+
+	// once we have successfully gotten a glob path, we can store results in the cache
+	defer func() {
+		g.setEntry(globPath, relativeGlob, &globEntry{
+			contents: contents,
+			foundAt:  foundAt,
+			err:      err,
+		})
+	}()
+
+	matches, err := filepath.Glob(globPath)
+	if err != nil {
+		return contents, foundAt, fmt.Errorf("unable to expand glob %q, %v", globPath, err)
+	}
+
+	// convert matches to be relative to our baseDir
+	var relativeMatches []string
+	for _, m := range matches {
+		rel, err := filepath.Rel(baseDir, m)
+		if err != nil {
+			return contents, globPath, fmt.Errorf("could not resolve %s from %s", m, importedFrom)
+		}
+		relativeMatches = append(relativeMatches, rel)
+	}
+
+	// ensure consistent order (not strictly required, makes it human friendly)
+	sort.Strings(relativeMatches)
+
+	var out bytes.Buffer
+	out.WriteString("{\n")
+	for _, file := range relativeMatches {
+		file = filepath.ToSlash(file)
+		out.WriteString("\t")
+		_, _ = fmt.Fprintf(&out, `'%s': %s '%s',`, file, g.innerVerb, file)
+		out.WriteString("\n")
+	}
+	out.WriteString("}")
+	k := g.cacheKey(globPath, relativeGlob)
+	output := out.String()
+
+	return jsonnet.MakeContents(output),
+		filepath.ToSlash(k.file(baseDir)),
+		nil
+}