rename zed/zq to super (#5346)

This commit changes the command structure from "zed" to "super".
The zq command is now "super query" and the zed command is now
"super db".  "super query" no longer attempts to differentiate
a single argument file from a single argument query and instead
has a flag "-c" for specifying the query text.  "super db query"
does not have a "-c" flag since there are no command-line file args.

The ZED_LAKE environement is now SUPER_DB_LAKE.

There is still work to transition from zed terminology both in the
source code as well as documentation but here we are getting all
of the automated and docs tests passing.

We will rename the file formats in a subsequent PR.

Some of us will miss typing zq in place of "super query" but
shell aliases are your friend, e.g., sup='super query'

CHANGELOG.md

@@ -220,7 +220,7 @@
 * Number `-split` flag files sequentially for `zed` and `zq`  (#3937)
 * Remove index support for relative comparisons (#3932)
 * Support `...` spread operator in [array expressions](docs/language/expressions.md#array-expressions) and [set expressions](docs/language/expressions.md#set-expressions) (#3910)
-* [Python client](python/zed): rework `zed.Client.query_raw()` to handle any format (#3911)
+* [Python client](python/superdb): rework `zed.Client.query_raw()` to handle any format (#3911)
 * Add support for Parquet API responses (#3909)
 * Support the indexing operation on sets (#3908)
 * Add `nullsMax` argument to [`compare() function`](docs/language/functions/compare.md) (#3898)
@@ -253,13 +253,13 @@
 * `zapi`: Rename the `ZED_LAKE_HOST` environment variable to `ZED_LAKE` and rename the `-host` flag to `-lake` (#3280)
 * `zq`: Improve ZNG read performance when the command line includes multiple input files (#3282)
 * `zed lake serve`: Add the `-rootcontentfile` flag  (#3283)
-* [Python client](python/zed): Improve error messages (#3279)
-* [Python client](python/zed): Fix Zed `bytes` decoding (#3278)
+* [Python client](python/superdb): Improve error messages (#3279)
+* [Python client](python/superdb): Fix Zed `bytes` decoding (#3278)
 * Detect CSV input (#3277)
 * `zed lake serve`: Fix an issue where `POST /pool/{}/branch/{}` format detection errors caused a 500 response (#3272)
 * Fix an issue where the ZSON parser failed to normalize maps and sets (#3273)
-* [Python client](python/zed): Add authentication (#3270)
-* [Python client](python/zed): Handle query errors  (#3269)
+* [Python client](python/superdb): Add authentication (#3270)
+* [Python client](python/superdb): Handle query errors  (#3269)
 * Remove support for the TZNG format (#3263)
 * `zapi`, `zed lake serve`: Add authentication with Auth0 (#3266)
 * Fix an issue preventing casting from `ip` to `ip` (#3259)

Makefile

@@ -2,7 +2,7 @@ export GO111MODULE=on
 
 VERSION = $(shell git describe --tags --dirty --always)
 LDFLAGS = -s -X github.com/brimdata/super/cli.version=$(VERSION)
-BUILD_COMMANDS = ./cmd/zed ./cmd/zq
+BUILD_COMMANDS = ./cmd/super
 
 ifeq "$(filter-out 386 arm mips mipsle, $(shell go env GOARCH))" ""
 $(error 32-bit architectures are unsupported; see https://github.com/brimdata/super/issues/4044)

cli/lakeflags/flags.go

@@ -37,10 +37,10 @@ func (l *Flags) SetFlags(fs *flag.FlagSet) {
 		dir = filepath.Join(dir, ".zed")
 	}
 	fs.StringVar(&l.ConfigDir, "configdir", dir, "configuration and credentials directory")
-	if s, ok := os.LookupEnv("ZED_LAKE"); ok {
+	if s, ok := os.LookupEnv("SUPER_DB_LAKE"); ok {
 		l.Lake, l.lakeSpecified = s, true
 	}
-	fs.Func("lake", fmt.Sprintf("lake location (env ZED_LAKE) (default %s)", l.Lake), func(s string) error {
+	fs.Func("lake", fmt.Sprintf("lake location (env SUPER_DB_LAKE) (default %s)", l.Lake), func(s string) error {
 		l.Lake, l.lakeSpecified = s, true
 		return nil
 	})
@@ -90,7 +90,7 @@ func (l *Flags) URI() (*storage.URI, error) {
 		lk = getDefaultDataDir()
 	}
 	if lk == "" {
-		return nil, errors.New("lake location must be set (either with the -lake flag or ZED_LAKE environment variable)")
+		return nil, errors.New("lake location must be set (either with the -lake flag or SUPER_DB_LAKE environment variable)")
 	}
 	u, err := storage.ParseURI(lk)
 	if err != nil {

cli/queryflags/flags.go

@@ -8,9 +8,7 @@ import (
 	"net/url"
 	"os"
 	"slices"
-	"strings"
 
-	"github.com/brimdata/super/cli"
 	"github.com/brimdata/super/compiler"
 	"github.com/brimdata/super/compiler/ast"
 	"github.com/brimdata/super/compiler/data"
@@ -32,36 +30,33 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) {
 	fs.Var(&f.Includes, "I", "source file containing Zed query text (may be used multiple times)")
 }
 
-func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, *parser.SourceSet, bool, error) {
-	var src string
-	if len(paths) != 0 && !cli.FileExists(paths[0]) && !isURLWithKnownScheme(paths[0], "http", "https", "s3") {
-		src = paths[0]
-		paths = paths[1:]
-		if len(paths) == 0 {
-			// Consider a lone argument to be a query if it compiles
-			// and appears to start with a from or yield operator.
-			// Otherwise, consider it a file.
-			query, sset, err := compiler.Parse(src, f.Includes...)
-			if err != nil {
-				return nil, nil, nil, false, singleArgError(src, err)
-			}
-			s, err := semantic.Analyze(context.Background(), query, data.NewSource(storage.NewLocalEngine(), nil), nil)
-			if err != nil {
-				if list, ok := err.(parser.ErrorList); ok {
-					list.SetSourceSet(sset)
-				}
-				return nil, nil, nil, false, err
-			}
-			if semantic.HasSource(s) {
-				return nil, query, sset, false, nil
-			}
-			if semantic.StartsWithYield(s) {
-				return nil, query, sset, true, nil
+func (f *Flags) ParseSourcesAndInputs(src string, paths []string) ([]string, ast.Seq, *parser.SourceSet, bool, error) {
+	if len(paths) == 0 && src != "" {
+		// Consider a lone argument to be a query if it compiles
+		// and appears to start with a from or yield operator.
+		// Otherwise, consider it a file.
+		query, sset, err := compiler.Parse(src, f.Includes...)
+		if err != nil {
+			return nil, nil, nil, false, err
+		}
+		s, err := semantic.Analyze(context.Background(), query, data.NewSource(storage.NewLocalEngine(), nil), nil)
+		if err != nil {
+			if list, ok := err.(parser.ErrorList); ok {
+				list.SetSourceSet(sset)
 			}
-			return nil, nil, nil, false, singleArgError(src, nil)
+			return nil, nil, nil, false, err
+		}
+		//XXX we should simplify this logic, e.g., by inserting a null source
+		// if no source is given (this is how sql "select count(*)" works with no from)
+		if semantic.HasSource(s) {
+			return nil, query, sset, false, nil
 		}
+		if semantic.StartsWithYield(s) {
+			return nil, query, sset, true, nil
+		}
+		return nil, nil, nil, false, errors.New("no data source found")
 	}
-	query, sset, err := parser.ParseZed(f.Includes, src)
+	query, sset, err := parser.ParseSuperPipe(f.Includes, src)
 	if err != nil {
 		return nil, nil, nil, false, err
 	}
@@ -85,23 +80,3 @@ func (f *Flags) PrintStats(stats zbuf.Progress) {
 		fmt.Fprintln(os.Stderr, out)
 	}
 }
-
-func singleArgError(src string, err error) error {
-	var b strings.Builder
-	b.WriteString("could not invoke zq with a single argument because:")
-	if len(src) > 20 {
-		src = src[:20] + "..."
-	}
-	fmt.Fprintf(&b, "\n - a file could not be found with the name %q", src)
-	if list := (parser.ErrorList)(nil); errors.As(err, &list) {
-		b.WriteString("\n - the argument could not be compiled as a valid Zed query:")
-		for _, line := range strings.Split(list.Error(), "\n") {
-			fmt.Fprintf(&b, "\n   %s", line)
-		}
-	} else if err == nil {
-		b.WriteString("\n - the argument is a valid Zed query but does not begin with a source (e.g., \"file input.zson\") or yield operator")
-	} else {
-		b.WriteString("\n - the argument did not parse as a valid Zed query")
-	}
-	return errors.New(b.String())
-}

cmd/mockzui/main.go

@@ -25,9 +25,9 @@ var (
 )
 
 func init() {
-	flag.StringVar(&portfile, "portfile", "", "location to write zed lake serve port")
-	flag.StringVar(&pidfile, "pidfile", "", "location to write zed lake serve pid")
-	flag.StringVar(&lakeroot, "lake", "", "Zed lake location")
+	flag.StringVar(&portfile, "portfile", "", "location to write SuperDB lake serve port")
+	flag.StringVar(&pidfile, "pidfile", "", "location to write SuperDB lake serve pid")
+	flag.StringVar(&lakeroot, "lake", "", "SuperDB lake location")
 	flag.Parse()
 }
 
@@ -44,6 +44,7 @@ func main() {
 		os.Exit(1)
 	}
 	args := []string{
+		"db",
 		"serve",
 		"-l=localhost:0",
 		"-lake=" + lakeroot,
@@ -52,7 +53,7 @@ func main() {
 		fmt.Sprintf("-brimfd=%d", r.Fd()),
 	}
 	stderr := bytes.NewBuffer(nil)
-	cmd := exec.Command("zed", args...)
+	cmd := exec.Command("super", args...)
 	cmd.Stderr = stderr
 	cmd.ExtraFiles = []*os.File{r}
 

cmd/zed/auth/command.go → cmd/super/db/auth/command.go

@@ -3,11 +3,11 @@ package auth
 import (
 	"flag"
 
-	"github.com/brimdata/super/cmd/zed/root"
+	"github.com/brimdata/super/cmd/super/db"
 	"github.com/brimdata/super/pkg/charm"
 )
 
-var Cmd = &charm.Spec{
+var spec = &charm.Spec{
 	Name:  "auth",
 	Usage: "auth [subcommand]",
 	Short: "authentication and authorization commands",
@@ -16,19 +16,20 @@ var Cmd = &charm.Spec{
 }
 
 func init() {
-	Cmd.Add(Login)
-	Cmd.Add(Logout)
-	Cmd.Add(Method)
-	Cmd.Add(Store)
-	Cmd.Add(Verify)
+	spec.Add(Login)
+	spec.Add(Logout)
+	spec.Add(Method)
+	spec.Add(Store)
+	spec.Add(Verify)
+	db.Spec.Add(spec)
 }
 
 type Command struct {
-	*root.Command
+	*db.Command
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	return &Command{Command: parent.(*root.Command)}, nil
+	return &Command{Command: parent.(*db.Command)}, nil
 }
 
 func (c *Command) Run(args []string) error {

cmd/zed/branch/command.go → cmd/super/db/branch/command.go

@@ -8,15 +8,15 @@ import (
 
 	"github.com/brimdata/super/cli/outputflags"
 	"github.com/brimdata/super/cli/poolflags"
-	"github.com/brimdata/super/cmd/zed/root"
+	"github.com/brimdata/super/cmd/super/db"
 	"github.com/brimdata/super/lake/api"
 	"github.com/brimdata/super/lakeparse"
 	"github.com/brimdata/super/pkg/charm"
 	"github.com/brimdata/super/pkg/storage"
 	"github.com/brimdata/super/zbuf"
 )
 
-var Cmd = &charm.Spec{
+var spec = &charm.Spec{
 	Name:  "branch",
 	Usage: "branch new-branch [base]",
 	Short: "create a new branch",
@@ -38,14 +38,18 @@ supplied to specify the desired pool for the new branch.
 }
 
 type Command struct {
-	*root.Command
+	*db.Command
 	delete      bool
 	outputFlags outputflags.Flags
 	poolFlags   poolflags.Flags
 }
 
+func init() {
+	db.Spec.Add(spec)
+}
+
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	c := &Command{Command: parent.(*root.Command)}
+	c := &Command{Command: parent.(*db.Command)}
 	f.BoolVar(&c.delete, "d", false, "delete the branch instead of creating it")
 	c.outputFlags.DefaultFormat = "lake"
 	c.outputFlags.SetFlags(f)

cmd/super/db/command.go

@@ -0,0 +1,47 @@
+package db
+
+import (
+	"flag"
+
+	"github.com/brimdata/super/cli/lakeflags"
+	"github.com/brimdata/super/cmd/super/root"
+	"github.com/brimdata/super/pkg/charm"
+)
+
+var Spec = &charm.Spec{
+	Name:  "db",
+	Usage: "db <sub-command> [options] [arguments...]",
+	Short: "run SuperDB data lake commands",
+	Long: `
+XXX db is a command-line tool for creating, configuring, ingesting into,
+querying, and orchestrating Zed data lakes.`,
+	New: New,
+}
+
+func init() {
+	root.Super.Add(Spec)
+}
+
+type Command struct {
+	*root.Command
+	LakeFlags lakeflags.Flags
+}
+
+func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
+	c := &Command{Command: parent.(*root.Command)}
+	c.LakeFlags.SetFlags(f)
+	return c, nil
+}
+
+func (c *Command) Run(args []string) error {
+	//XXX
+	_, cancel, err := c.Init()
+	if err != nil {
+		return err
+	}
+	defer cancel()
+	if len(args) == 0 {
+		return charm.NeedHelp
+	}
+	return charm.ErrNoRun
+}

cmd/zed/compact/command.go → cmd/super/db/compact/command.go

@@ -6,12 +6,12 @@ import (
 
 	"github.com/brimdata/super/cli/commitflags"
 	"github.com/brimdata/super/cli/poolflags"
-	"github.com/brimdata/super/cmd/zed/root"
+	"github.com/brimdata/super/cmd/super/db"
 	"github.com/brimdata/super/lakeparse"
 	"github.com/brimdata/super/pkg/charm"
 )
 
-var Cmd = &charm.Spec{
+var spec = &charm.Spec{
 	Name:  "compact",
 	Usage: "compact id id [id ...]",
 	Short: "compact data objects on a pool branch",
@@ -23,14 +23,18 @@ a commit on HEAD replacing the old objects with the new ones.`,
 }
 
 type Command struct {
-	*root.Command
+	*db.Command
 	commitFlags  commitflags.Flags
 	poolFlags    poolflags.Flags
 	writeVectors bool
 }
 
+func init() {
+	db.Spec.Add(spec)
+}
+
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	c := &Command{Command: parent.(*root.Command)}
+	c := &Command{Command: parent.(*db.Command)}
 	c.commitFlags.SetFlags(f)
 	c.poolFlags.SetFlags(f)
 	f.BoolVar(&c.writeVectors, "vectors", false, "write vectors for compacted objects")

cmd/zed/create/command.go → cmd/super/db/create/command.go

@@ -6,14 +6,14 @@ import (
 	"fmt"
 
 	"github.com/brimdata/super/cli/poolflags"
-	"github.com/brimdata/super/cmd/zed/root"
+	"github.com/brimdata/super/cmd/super/db"
 	"github.com/brimdata/super/lake/data"
 	"github.com/brimdata/super/order"
 	"github.com/brimdata/super/pkg/charm"
 	"github.com/brimdata/super/pkg/units"
 )
 
-var Cmd = &charm.Spec{
+var spec = &charm.Spec{
 	Name:  "create",
 	Usage: "create [-orderby key[:asc|:desc]] name",
 	Short: "create a new data pool",
@@ -37,16 +37,20 @@ By default, a branch called "main" is initialized in the newly created pool.
 }
 
 type Command struct {
-	*root.Command
+	*db.Command
 	sortKey    string
 	thresh     units.Bytes
 	seekStride units.Bytes
 	use        bool
 }
 
+func init() {
+	db.Spec.Add(spec)
+}
+
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 	c := &Command{
-		Command:    parent.(*root.Command),
+		Command:    parent.(*db.Command),
 		seekStride: units.Bytes(data.DefaultSeekStride),
 	}
 	f.Var(&c.seekStride, "seekstride", "size of seek-index unit for ZNG data, as '32KB', '1MB', etc.")