Implement pixlet bundle for directories

WIP, tests have not been updated.

bundle/bundle.go

@@ -7,10 +7,14 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 
+	"github.com/quay/claircore/pkg/tarfs"
+
 	"tidbyt.dev/pixlet/manifest"
+	"tidbyt.dev/pixlet/runtime"
 )
 
 const (
@@ -26,16 +30,12 @@ const (
 
 // AppBundle represents the unpacked bundle in our system.
 type AppBundle struct {
-	Source   []byte
 	Manifest *manifest.Manifest
+	Source   fs.FS
 }
 
-// InitFromPath translates a directory containing an app manifest and source
-// into an AppBundle.
-func InitFromPath(dir string) (*AppBundle, error) {
-	// Load manifest
-	path := filepath.Join(dir, manifest.ManifestFileName)
-	m, err := os.Open(path)
+func fromFS(fs fs.FS) (*AppBundle, error) {
+	m, err := fs.Open(manifest.ManifestFileName)
 	if err != nil {
 		return nil, fmt.Errorf("could not open manifest: %w", err)
 	}
@@ -46,80 +46,39 @@ func InitFromPath(dir string) (*AppBundle, error) {
 		return nil, fmt.Errorf("could not load manifest: %w", err)
 	}
 
-	// Load source
-	path = filepath.Join(dir, man.FileName)
-	s, err := os.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("could not open app source: %w", err)
-	}
-	defer s.Close()
-
-	src, err := io.ReadAll(s)
-	if err != nil {
-		return nil, fmt.Errorf("could not read app source: %w", err)
-	}
-
 	// Create app bundle struct
 	return &AppBundle{
 		Manifest: man,
-		Source:   src,
+		Source:   fs,
 	}, nil
 }
 
+// InitFromPath translates a directory containing an app manifest and source
+// into an AppBundle.
+func InitFromPath(dir string) (*AppBundle, error) {
+	return fromFS(os.DirFS(dir))
+}
+
 // LoadBundle loads a compressed archive into an AppBundle.
 func LoadBundle(in io.Reader) (*AppBundle, error) {
 	gzr, err := gzip.NewReader(in)
 	if err != nil {
-		return nil, fmt.Errorf("could not create gzip reader: %w", err)
+		return nil, fmt.Errorf("creating gzip reader: %w", err)
 	}
 	defer gzr.Close()
 
-	tr := tar.NewReader(gzr)
-	ab := &AppBundle{}
-
-	for {
-		header, err := tr.Next()
-
-		switch {
-		case err == io.EOF:
-			// If there are no more files in the bundle, validate and return it.
-			if ab.Manifest == nil {
-				return nil, fmt.Errorf("could not find manifest in archive")
-			}
-			if ab.Source == nil {
-				return nil, fmt.Errorf("could not find source in archive")
-			}
-			return ab, nil
-		case err != nil:
-			// If there is an error, return immediately.
-			return nil, fmt.Errorf("could not read archive: %w", err)
-		case header == nil:
-			// If for some reason we end up with a blank header, continue to the
-			// next one.
-			continue
-		case header.Name == AppSourceName:
-			// Load the app source.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read source from archive: %w", err)
-			}
-			ab.Source = buff
-		case header.Name == manifest.ManifestFileName:
-			// Load the app manifest.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read manifest from archive: %w", err)
-			}
+	// read the entire tarball into memory so that we can seek
+	// around it, and so that the underlying reader can be closed.
+	var b bytes.Buffer
+	io.Copy(&b, gzr)
 
-			man, err := manifest.LoadManifest(bytes.NewReader(buff))
-			if err != nil {
-				return nil, fmt.Errorf("could not load manifest: %w", err)
-			}
-			ab.Manifest = man
-		}
+	r := bytes.NewReader(b.Bytes())
+	fs, err := tarfs.New(r)
+	if err != nil {
+		return nil, fmt.Errorf("creating tarfs: %w", err)
 	}
+
+	return fromFS(fs)
 }
 
 // WriteBundleToPath is a helper to be able to write the bundle to a provided
@@ -137,6 +96,16 @@ func (b *AppBundle) WriteBundleToPath(dir string) error {
 
 // WriteBundle writes a compressed archive to the provided writer.
 func (ab *AppBundle) WriteBundle(out io.Writer) error {
+	// we don't want to naively write the entire source FS to the tarball,
+	// since it could contain a lot of extraneous files. instead, run the
+	// applet and interrogate it for the files it needs to include in the
+	// bundle.
+	app, err := runtime.NewAppletFromFS(ab.Manifest.ID, ab.Source, runtime.WithPrintDisabled())
+	if err != nil {
+		return fmt.Errorf("loading applet for bundling: %w", err)
+	}
+	bundleFiles := app.PathsForBundle()
+
 	// Setup writers.
 	gzw := gzip.NewWriter(out)
 	defer gzw.Close()
@@ -146,7 +115,7 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 
 	// Write manifest.
 	buff := &bytes.Buffer{}
-	err := ab.Manifest.WriteManifest(buff)
+	err = ab.Manifest.WriteManifest(buff)
 	if err != nil {
 		return fmt.Errorf("could not write manifest to buffer: %w", err)
 	}
@@ -166,19 +135,37 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 		return fmt.Errorf("could not write manifest to archive: %w", err)
 	}
 
-	// Write source.
-	hdr = &tar.Header{
-		Name: AppSourceName,
-		Mode: 0600,
-		Size: int64(len(ab.Source)),
-	}
-	err = tw.WriteHeader(hdr)
-	if err != nil {
-		return fmt.Errorf("could not write source header: %w", err)
-	}
-	_, err = tw.Write(ab.Source)
-	if err != nil {
-		return fmt.Errorf("could not write source to archive: %w", err)
+	// write sources.
+	for _, path := range bundleFiles {
+		stat, err := fs.Stat(ab.Source, path)
+		if err != nil {
+			return fmt.Errorf("could not stat %s: %w", path, err)
+		}
+
+		hdr, err := tar.FileInfoHeader(stat, "")
+		if err != nil {
+			return fmt.Errorf("creating header for %s: %w", path, err)
+		}
+		hdr.Name = filepath.ToSlash(path)
+
+		err = tw.WriteHeader(hdr)
+		if err != nil {
+			return fmt.Errorf("writing header for %s: %w", path, err)
+		}
+
+		if !stat.IsDir() {
+			file, err := ab.Source.Open(path)
+			if err != nil {
+				return fmt.Errorf("opening file %s: %w", path, err)
+			}
+
+			written, err := io.Copy(tw, file)
+			if err != nil {
+				return fmt.Errorf("writing file %s: %w", path, err)
+			} else if written != stat.Size() {
+				return fmt.Errorf("did not write entire file %s: %w", path, err)
+			}
+		}
 	}
 
 	return nil

bundle/testdata/testapp/manifest.yaml

@@ -4,5 +4,3 @@ name: Test App
 summary: For Testing
 desc: It's an app for testing.
 author: Test Dev
-fileName: test_app.star
-packageName: testapp

bundle/testdata/testapp/test_app.star

@@ -7,9 +7,12 @@ Author: Test Dev
 
 load("render.star", "render")
 load("schema.star", "schema")
+load("test.txt", test_txt = "file")
 
 DEFAULT_WHO = "world"
 
+TEST_TXT_CONTENT = test_txt.readall()
+
 def main(config):
     who = config.str("who", DEFAULT_WHO)
     message = "Hello, {}!".format(who)

cmd/community/manifestprompt.go

@@ -51,12 +51,10 @@ func ManifestPrompt() (*manifest.Manifest, error) {
 	}
 
 	return &manifest.Manifest{
-		ID:          manifest.GenerateID(name),
-		Name:        name,
-		Summary:     summary,
-		Desc:        desc,
-		Author:      author,
-		FileName:    manifest.GenerateFileName(name),
-		PackageName: manifest.GeneratePackageName(name),
+		ID:      manifest.GenerateID(name),
+		Name:    name,
+		Summary: summary,
+		Desc:    desc,
+		Author:  author,
 	}, nil
 }

cmd/community/validatemanifest.go

@@ -12,7 +12,6 @@ import (
 var ValidateManifestAppFileName string
 
 func init() {
-	ValidateManifestCmd.Flags().StringVarP(&ValidateManifestAppFileName, "app-file-name", "a", "", "ensures the app file name is the same as the manifest")
 }
 
 var ValidateManifestCmd = &cobra.Command{
@@ -47,9 +46,5 @@ func ValidateManifest(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("couldn't validate manifest: %w", err)
 	}
 
-	if ValidateManifestAppFileName != "" && m.FileName != ValidateManifestAppFileName {
-		return fmt.Errorf("app name doesn't match: %s != %s", ValidateManifestAppFileName, m.FileName)
-	}
-
 	return nil
 }

go.mod

@@ -84,6 +84,7 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/quay/claircore v1.5.26 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -100,10 +101,10 @@ require (
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -275,6 +275,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/qri-io/starlib v0.5.1-0.20220611014110-7fb7ff9ec804 h1:uiSBjMqewVGbxBDsF5UOR7NARfhcSgpihRNvH9NiroA=
 github.com/qri-io/starlib v0.5.1-0.20220611014110-7fb7ff9ec804/go.mod h1:Geq0MWa2oq+Ki/05aXaKoJAguFzlCZQd9Fx3hTsAEPU=
+github.com/quay/claircore v1.5.26 h1:gkZeMDJV1XU0fSw9trfzr0g+/OtQiQFU7INdGDsg66Y=
+github.com/quay/claircore v1.5.26/go.mod h1:SiN/daNJW3l0tvGkSTOCWO8MXmwlbP2AdsDCGx51aLw=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -394,6 +396,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
 golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -485,6 +489,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

manifest/manifest.go

@@ -16,21 +16,22 @@ type Manifest struct {
 	// ID is the unique identifier of this app. It has to be globally unique,
 	// which means it cannot conflict with any of our private apps.
 	ID string `json:"id" yaml:"id"`
+
 	// Name is the name of the applet. Ex. "Fuzzy Clock"
 	Name string `json:"name" yaml:"name"`
+
 	// Summary is the short form of what this applet does. Ex. "Human readable
 	// time".
 	Summary string `json:"summary" yaml:"summary"`
+
 	// Desc is the long form of what this applet does. Ex. "Display the time in
 	// a groovy, human-readable way."
 	Desc string `json:"desc" yaml:"desc"`
+
 	// Author is the person or organization who contributed this applet. Ex,
 	// "Max Timkovich"
 	Author string `json:"author" yaml:"author"`
-	// FileName is the name of the starlark source file.
-	FileName string `json:"fileName" yaml:"fileName"`
-	// PackageName is the name of the go package where this app lives.
-	PackageName string `json:"packageName" yaml:"packageName"`
+
 	// Source is the starlark source code for this applet using the go `embed`
 	// module.
 	Source []byte `json:"-" yaml:"-"`
@@ -100,24 +101,14 @@ func (m Manifest) Validate() error {
 		return err
 	}
 
-	err = ValidateFileName(m.FileName)
-	if err != nil {
-		return err
-	}
-
-	err = ValidatePackageName(m.PackageName)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
-// GeneratePackageName creates a suitable go package name from an app name.
-func GeneratePackageName(name string) string {
-	packageName := strings.ReplaceAll(name, "-", "")
-	packageName = strings.ReplaceAll(packageName, "_", "")
-	return strings.ToLower(strings.Join(strings.Fields(packageName), ""))
+// GenerateDirName creates a suitable directory name from an app name.
+func GenerateDirName(name string) string {
+	dir := strings.ReplaceAll(name, "-", "")
+	dir = strings.ReplaceAll(dir, "_", "")
+	return strings.ToLower(strings.Join(strings.Fields(dir), ""))
 }
 
 // GenerateID creates a suitable ID from an app name.