change zed formats to super formats (#5384)

This commit changes the formats zson,zng, and vng to jsup, bsup,
and csup, repectively along with all the file extensions in the docs
and tests.  We did not here address -Z, -z, parse_zson, or zjson.

This is primarily a user visible change.  There are still lots of
places in the source code that use the Zed terminology.

---------

Co-authored-by: Noah Treuhaft <[email protected]>

api/client/connection.go

@@ -61,8 +61,8 @@ func NewConnection() *Connection {
 // and a base URL derived from the hostURL argument.
 func NewConnectionTo(hostURL string) *Connection {
 	defaultHeader := http.Header{
-		"Accept":       []string{api.MediaTypeZNG},
-		"Content-Type": []string{api.MediaTypeZNG},
+		"Accept":       []string{api.MediaTypeBSUP},
+		"Content-Type": []string{api.MediaTypeBSUP},
 	}
 	return &Connection{
 		client:        &http.Client{},
@@ -409,7 +409,7 @@ func (c *Connection) doVector(ctx context.Context, pool, revision string, object
 
 func (c *Connection) SubscribeEvents(ctx context.Context) (*EventsClient, error) {
 	req := c.NewRequest(ctx, http.MethodGet, "/events", nil)
-	req.Header.Set("Accept", api.MediaTypeZSON)
+	req.Header.Set("Accept", api.MediaTypeJSUP)
 	resp, err := c.Do(req)
 	if err != nil {
 		return nil, err

api/mime.go

@@ -10,17 +10,17 @@ import (
 const (
 	MediaTypeAny         = "*/*"
 	MediaTypeArrowStream = "application/vnd.apache.arrow.stream"
+	MediaTypeBSUP        = "application/x-bsup"
+	MediaTypeCSUP        = "application/x-csup"
 	MediaTypeCSV         = "text/csv"
 	MediaTypeJSON        = "application/json"
+	MediaTypeJSUP        = "application/x-jsup"
 	MediaTypeLine        = "application/x-line"
 	MediaTypeNDJSON      = "application/x-ndjson"
 	MediaTypeParquet     = "application/x-parquet"
 	MediaTypeTSV         = "text/tab-separated-values"
-	MediaTypeVNG         = "application/x-vng"
 	MediaTypeZeek        = "application/x-zeek"
 	MediaTypeZJSON       = "application/x-zjson"
-	MediaTypeZNG         = "application/x-zng"
-	MediaTypeZSON        = "application/x-zson"
 )
 
 type ErrUnsupportedMimeType struct {
@@ -46,10 +46,16 @@ func MediaTypeToFormat(s string, dflt string) (string, error) {
 		return dflt, nil
 	case MediaTypeArrowStream:
 		return "arrows", nil
+	case MediaTypeBSUP:
+		return "bsup", nil
+	case MediaTypeCSUP:
+		return "csup", nil
 	case MediaTypeCSV:
 		return "csv", nil
 	case MediaTypeJSON:
 		return "json", nil
+	case MediaTypeJSUP:
+		return "jsup", nil
 	case MediaTypeLine:
 		return "line", nil
 	case MediaTypeNDJSON:
@@ -58,16 +64,10 @@ func MediaTypeToFormat(s string, dflt string) (string, error) {
 		return "parquet", nil
 	case MediaTypeTSV:
 		return "tsv", nil
-	case MediaTypeVNG:
-		return "vng", nil
 	case MediaTypeZeek:
 		return "zeek", nil
 	case MediaTypeZJSON:
 		return "zjson", nil
-	case MediaTypeZNG:
-		return "zng", nil
-	case MediaTypeZSON:
-		return "zson", nil
 	}
 	return "", &ErrUnsupportedMimeType{typ}
 }
@@ -76,10 +76,16 @@ func FormatToMediaType(format string) (string, error) {
 	switch format {
 	case "arrows":
 		return MediaTypeArrowStream, nil
+	case "bsup":
+		return MediaTypeBSUP, nil
+	case "csup":
+		return MediaTypeCSUP, nil
 	case "csv":
 		return MediaTypeCSV, nil
 	case "json":
 		return MediaTypeJSON, nil
+	case "jsup":
+		return MediaTypeJSUP, nil
 	case "line":
 		return MediaTypeLine, nil
 	case "ndjson":
@@ -88,16 +94,10 @@ func FormatToMediaType(format string) (string, error) {
 		return MediaTypeParquet, nil
 	case "tsv":
 		return MediaTypeTSV, nil
-	case "vng":
-		return MediaTypeVNG, nil
 	case "zeek":
 		return MediaTypeZeek, nil
 	case "zjson":
 		return MediaTypeZJSON, nil
-	case "zng":
-		return MediaTypeZNG, nil
-	case "zson":
-		return MediaTypeZSON, nil
 	default:
 		return "", fmt.Errorf("unknown format type: %s", format)
 	}

api/queryio/writer.go

@@ -33,7 +33,7 @@ func NewWriter(w io.WriteCloser, format string, flusher http.Flusher, ctrl bool)
 	}
 	var err error
 	switch format {
-	case "zng":
+	case "bsup":
 		d.writer = NewZNGWriter(w)
 	case "zjson":
 		d.writer = NewZJSONWriter(w)

cli/inputflags/flags.go

@@ -28,7 +28,7 @@ func (f *Flags) Options() anyio.ReaderOpts {
 }
 
 func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) {
-	fs.StringVar(&f.Format, "i", "auto", "format of input data [auto,arrows,csv,json,line,parquet,tsv,vng,zeek,zjson,zng,zson]")
+	fs.StringVar(&f.Format, "i", "auto", "format of input data [auto,arrows,bsup,csup,csv,json,jsup,line,parquet,tsv,zeek,zjson]")
 	f.CSV.Delim = ','
 	fs.Func("csv.delim", `CSV field delimiter (default ",")`, func(s string) error {
 		if len(s) != 1 {
@@ -38,12 +38,12 @@ func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) {
 		return nil
 
 	})
-	fs.BoolVar(&f.ZNG.Validate, "zng.validate", validate, "validate format when reading ZNG")
-	fs.IntVar(&f.ZNG.Threads, "zng.threads", 0, "number of ZNG read threads (0=GOMAXPROCS)")
+	fs.BoolVar(&f.ZNG.Validate, "bsup.validate", validate, "validate format when reading Super Binary")
+	fs.IntVar(&f.ZNG.Threads, "bsup.threads", 0, "number of Super Binary read threads (0=GOMAXPROCS)")
 	f.ReadMax = auto.NewBytes(zngio.MaxSize)
-	fs.Var(&f.ReadMax, "zng.readmax", "maximum ZNG read buffer size in MiB, MB, etc.")
+	fs.Var(&f.ReadMax, "bsup.readmax", "maximum Super Binary read buffer size in MiB, MB, etc.")
 	f.ReadSize = auto.NewBytes(zngio.ReadSize)
-	fs.Var(&f.ReadSize, "zng.readsize", "target ZNG read buffer size in MiB, MB, etc.")
+	fs.Var(&f.ReadSize, "bsup.readsize", "target Super Binary read buffer size in MiB, MB, etc.")
 }
 
 // Init is called after flags have been parsed.

cli/outputflags/flags.go

@@ -22,17 +22,17 @@ import (
 type Flags struct {
 	anyio.WriterOpts
 	DefaultFormat string
-	split         string
-	splitSize     auto.Bytes
-	outputFile    string
+	color         bool
 	forceBinary   bool
-	jsonShortcut  bool
 	jsonPretty    bool
-	zsonShortcut  bool
-	zsonPretty    bool
-	zsonPersist   string
-	color         bool
+	jsonShortcut  bool
+	jsupPersist   string
+	jsupPretty    bool
+	jsupShortcut  bool
+	outputFile    string
 	pretty        int
+	split         string
+	splitSize     auto.Bytes
 	unbuffered    bool
 }
 
@@ -41,24 +41,21 @@ func (f *Flags) Options() anyio.WriterOpts {
 }
 
 func (f *Flags) setFlags(fs *flag.FlagSet) {
-	// zio stuff
-	fs.BoolVar(&f.color, "color", true, "enable/disable color formatting for -Z and lake text output")
 	f.ZNG = &zngio.WriterOpts{}
-	fs.BoolVar(&f.ZNG.Compress, "zng.compress", true, "compress ZNG frames")
-	fs.IntVar(&f.ZNG.FrameThresh, "zng.framethresh", zngio.DefaultFrameThresh,
-		"minimum ZNG frame size in uncompressed bytes")
-	fs.IntVar(&f.pretty, "pretty", 4,
-		"tab size to pretty print JSON/ZSON output (0 for newline-delimited JSON/ZSON")
-	fs.StringVar(&f.zsonPersist, "persist", "",
+	fs.BoolVar(&f.ZNG.Compress, "bsup.compress", true, "compress Super Binary frames")
+	fs.IntVar(&f.ZNG.FrameThresh, "bsup.framethresh", zngio.DefaultFrameThresh,
+		"minimum Super Binary frame size in uncompressed bytes")
+	fs.BoolVar(&f.color, "color", true, "enable/disable color formatting for -Z and lake text output")
+	fs.StringVar(&f.jsupPersist, "persist", "",
 		"regular expression to persist type definitions across the stream")
-
-	// emitter stuff
+	fs.IntVar(&f.pretty, "pretty", 4,
+		"tab size to pretty print JSON and Super JSON output (0 for newline-delimited output")
+	fs.StringVar(&f.outputFile, "o", "", "write data to output file")
 	fs.StringVar(&f.split, "split", "",
 		"split output into one file per data type in this directory (but see -splitsize)")
 	fs.Var(&f.splitSize, "splitsize",
 		"if >0 and -split is set, split into files at least this big rather than by data type")
 	fs.BoolVar(&f.unbuffered, "unbuffered", false, "disable output buffering")
-	fs.StringVar(&f.outputFile, "o", "", "write data to output file")
 }
 
 func (f *Flags) SetFlags(fs *flag.FlagSet) {
@@ -73,48 +70,48 @@ func (f *Flags) SetFlagsWithFormat(fs *flag.FlagSet, format string) {
 
 func (f *Flags) SetFormatFlags(fs *flag.FlagSet) {
 	if f.DefaultFormat == "" {
-		f.DefaultFormat = "zng"
+		f.DefaultFormat = "bsup"
 	}
-	fs.StringVar(&f.Format, "f", f.DefaultFormat, "format for output data [arrows,csv,json,lake,parquet,table,text,tsv,vng,zeek,zjson,zng,zson]")
-	fs.BoolVar(&f.jsonShortcut, "j", false, "use line-oriented JSON output independent of -f option")
+	fs.StringVar(&f.Format, "f", f.DefaultFormat, "format for output data [arrows,bsup,csup,csv,json,jsup,lake,parquet,table,text,tsv,zeek,zjson]")
+	fs.BoolVar(&f.forceBinary, "B", false, "allow Super Binary to be sent to a terminal output")
 	fs.BoolVar(&f.jsonPretty, "J", false, "use formatted JSON output independent of -f option")
-	fs.BoolVar(&f.zsonShortcut, "z", false, "use line-oriented ZSON output independent of -f option")
-	fs.BoolVar(&f.zsonPretty, "Z", false, "use formatted ZSON output independent of -f option")
-	fs.BoolVar(&f.forceBinary, "B", false, "allow binary zng be sent to a terminal output")
+	fs.BoolVar(&f.jsonShortcut, "j", false, "use line-oriented JSON output independent of -f option")
+	fs.BoolVar(&f.jsupPretty, "Z", false, "use formatted Super JSON output independent of -f option")
+	fs.BoolVar(&f.jsupShortcut, "z", false, "use line-oriented Super JSON output independent of -f option")
 }
 
 func (f *Flags) Init() error {
 	f.JSON.Pretty, f.ZSON.Pretty = f.pretty, f.pretty
-	if f.zsonPersist != "" {
-		re, err := regexp.Compile(f.zsonPersist)
+	if f.jsupPersist != "" {
+		re, err := regexp.Compile(f.jsupPersist)
 		if err != nil {
 			return err
 		}
 		f.ZSON.Persist = re
 	}
 	if f.jsonShortcut || f.jsonPretty {
-		if f.Format != f.DefaultFormat || f.zsonShortcut || f.zsonPretty {
+		if f.Format != f.DefaultFormat || f.jsupShortcut || f.jsupPretty {
 			return errors.New("cannot use -j or -J with -f, -z, or -Z")
 		}
 		f.Format = "json"
 		if !f.jsonPretty {
 			f.JSON.Pretty = 0
 		}
-	} else if f.zsonShortcut || f.zsonPretty {
+	} else if f.jsupShortcut || f.jsupPretty {
 		if f.Format != f.DefaultFormat {
 			return errors.New("cannot use -z or -Z with -f")
 		}
-		f.Format = "zson"
-		if !f.zsonPretty {
+		f.Format = "jsup"
+		if !f.jsupPretty {
 			f.ZSON.Pretty = 0
 		}
 	}
 	if f.outputFile == "-" {
 		f.outputFile = ""
 	}
-	if f.outputFile == "" && f.split == "" && f.Format == "zng" && !f.forceBinary &&
+	if f.outputFile == "" && f.split == "" && f.Format == "bsup" && !f.forceBinary &&
 		terminal.IsTerminalFile(os.Stdout) {
-		f.Format = "zson"
+		f.Format = "jsup"
 		f.ZSON.Pretty = 0
 	}
 	if f.unbuffered {

cmd/super/db/manage/ztests/config.yaml

@@ -6,7 +6,7 @@ script: |
   super db create -q test3
   super db branch -use test2 -q live
   super db manage -config=inherit.yaml -log.path=inherit.log
-  super -Z -c 'msg == "updating pool" | cut name, branch | sort name' inherit.log > inherit.zson
+  super -Z -c 'msg == "updating pool" | cut name, branch | sort name' inherit.log > inherit.jsup
 
 inputs:
   - name: inherit.yaml
@@ -18,7 +18,7 @@ inputs:
           branch: "live"
 
 outputs:
-  - name: inherit.zson
+  - name: inherit.jsup
     data: | 
       {
           name: "test1",

cmd/super/db/serve/command.go

@@ -76,7 +76,7 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 		c.conf.CORSAllowedOrigins = append(c.conf.CORSAllowedOrigins, s)
 		return nil
 	})
-	f.StringVar(&c.conf.DefaultResponseFormat, "defaultfmt", service.DefaultZedFormat, "default response format")
+	f.StringVar(&c.conf.DefaultResponseFormat, "defaultfmt", service.DefaultFormat, "default response format")
 	f.StringVar(&c.listenAddr, "l", ":9867", "[addr]:port to listen on")
 	f.DurationVar(&c.manage, "manage", 0, "when positive, run lake maintenance tasks at this interval")
 	f.StringVar(&c.portFile, "portfile", "", "write listen port to file")

cmd/super/db/ztests/query-stats.yaml

@@ -2,12 +2,12 @@ script: |
   export SUPER_DB_LAKE=test
   super db init -q
   super db create -q test
-  super db load -q -use test babble.zson
+  super db load -q -use test babble.jsup
   super db query -s -z "from test | count()"
 
 inputs:
-  - name: babble.zson
-    source: ../../../../testdata/babble.zson
+  - name: babble.jsup
+    source: ../../../../testdata/babble.jsup
 
 outputs:
   - name: stdout

cmd/super/dev/dig/ztests/frames.yaml

@@ -1,13 +1,13 @@
 script: |
-  super a.zson > c.zng
-  super b.zson >> c.zng
-  super dev dig frames -z c.zng
+  super a.jsup > c.bsup
+  super b.jsup >> c.bsup
+  super dev dig frames -z c.bsup
 
 inputs:
-  - name: a.zson
+  - name: a.jsup
     data: |
       {a:1}
-  - name: b.zson
+  - name: b.jsup
     data: |
       {b:2}
 

cmd/super/dev/vng/command.go

@@ -20,7 +20,7 @@ import (
 )
 
 var spec = &charm.Spec{
-	Name:  "vng",
+	Name:  "csup",
 	Usage: "vng uri",
 	Short: "dump VNG metadata",
 	Long: `

cmd/super/root/command.go

@@ -41,8 +41,9 @@ See https://github.com/brimdata/super/tree/main/docs/language
 for details.
 
 Supported input formats include CSV, JSON, NDJSON, Parquet,
-VNG, ZNG, ZSON, and Zeek TSV.  Supported output formats include
-all the input formats along with a SQL-like table format.
+Super JSON, Super Binary, Super Columnar,  and Zeek TSV. 
+Supported output formats include all the input formats along with
+a SQL-like table format.
 
 "zq" must be run with at least one input.  Input files can
 be file system paths; "-" for standard input; or HTTP, HTTPS, or S3 URLs.
@@ -61,7 +62,7 @@ flag to indicate a destination directory for separate output files for each
 output type.  This flag may be used in combination with -o, which
 provides the prefix for the file path, e.g.,
 
-  zq -f parquet -split out -o example-output input.zng
+  zq -f parquet -split out -o example-output input.bsup
 
 When writing to stdout and stdout is a terminal, the default output format is ZSON.
 Otherwise, the default format is binary ZNG.  In either case, the default
@@ -85,8 +86,8 @@ The "zq" engine processes data natively in Zed so if you intend to run
 many queries over the same data, you will see substantial performance gains
 by converting your data to the efficient binary form of Zed called ZNG, e.g.,
 
-  zq -f zng input.json > fast.zng
-  zq <query> fast.zng
+  zq -f zng input.json > fast.bsup
+  zq <query> fast.bsup
   ...
 
 Please see https://github.com/brimdata/super for more information.

cmd/super/ztests/aggmem.yaml

@@ -1,9 +1,9 @@
 script: |
-  super -aggmem 1B -z -c 'collect(this)' a.zson
-  ! super -aggmem 0 a.zson
+  super -aggmem 1B -z -c 'collect(this)' a.jsup
+  ! super -aggmem 0 a.jsup
 
 inputs:
-  - name: a.zson
+  - name: a.jsup
     data: |
       {a:1}
 

cmd/super/ztests/call-user-op-with-src.yaml

@@ -5,9 +5,9 @@ inputs:
   - name: countfile.zed
     data: |
       op countfile(): (
-        file test.zson | count()
+        file test.jsup | count()
       )
-  - name: test.zson
+  - name: test.jsup
     data: '{} {} {} {}'
 
 outputs: