compiler: Add participle-based protobuf parser

Makefile

@@ -18,7 +18,7 @@ clean::  ## Remove generated files
 .PHONY: all ci clean
 
 # --- Build --------------------------------------------------------------------
-CMDS = ./cmd/pb
+CMDS = ./cmd/pb ./cmd/protog
 
 build: | $(O)  ## Build reflect binaries
 	go build -o $(O) $(CMDS)

cmd/protog/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"foxygo.at/protog/compiler/parser"
+	"github.com/alecthomas/kong"
+)
+
+var (
+	// version vars set by goreleaser
+	version = "tip"
+	commit  = "HEAD"
+	date    = "now"
+
+	description = `protog compiles .proto files`
+
+	cli struct {
+		ProtogConfig
+		Version kong.VersionFlag `help:"Show version"`
+	}
+)
+
+type ProtogConfig struct {
+	Filename []string `arg:"" help:"filenames of .proto file to compile"`
+}
+
+func main() {
+	kctx := kong.Parse(&cli,
+		kong.Description(description),
+		kong.Vars{"version": fmt.Sprintf("%s (%s on %s)", version, commit, date)},
+	)
+	if err := kctx.Run(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func (cfg *ProtogConfig) Run() error {
+	for _, filename := range cfg.Filename {
+		f, err := os.Open(filename)
+		if err != nil {
+			return err
+		}
+		p, err := parser.Parse(filename, f)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("%+v\n", p)
+	}
+	return nil
+}

compiler/parser/lexer.go

@@ -0,0 +1,131 @@
+// This code is derived from https://github.com/alecthomas/protobuf
+
+package parser
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+var lex = lexer.MustSimple([]lexer.SimpleRule{
+	{Name: "String", Pattern: `"(\\"|[^"])*"|'(\\'|[^'])*'`},
+	{Name: "Ident", Pattern: `[a-zA-Z_]([a-zA-Z_0-9])*`},
+	{Name: "Float", Pattern: `[-+]?(\d*\.\d+([eE][-+]?\d+)?|\d+[eE][-+]?\d+|inf)`},
+	{Name: "Int", Pattern: `[-+]?(0[xX][0-9A-Fa-f]+)|([-+]?\d+)`},
+	{Name: "Whitespace", Pattern: `[ \t\n\r\s]+`},
+	{Name: "Comment", Pattern: `(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//(.*)[^\n]*\n)`},
+	{Name: "Symbols", Pattern: `[/={}\[\]()<>.,;:]`},
+})
+
+var escapeTable = map[rune]rune{
+	'0':  '\000',
+	'a':  '\x07',
+	'b':  '\x08',
+	'e':  '\x1B',
+	'f':  '\x0C',
+	'n':  '\x0A',
+	'r':  '\x0D',
+	't':  '\x09',
+	'v':  '\x0B',
+	'\\': '\x5C',
+	'\'': '\x27',
+	'"':  '\x22',
+	'?':  '\x3F',
+}
+
+var stateBase = map[unquoteState]int{
+	uqHex: 16, uqOctal: 8,
+}
+
+type unquoteState int
+
+const (
+	uqDefault unquoteState = iota
+	uqEscape
+	uqHex
+	uqOctal
+)
+
+// C-style unquoting - supports octal \DDD and hex \xDDD
+func unquote(token lexer.Token) (lexer.Token, error) {
+	kind := token.Value[0] // nolint: ifshort
+	token.Value = token.Value[1 : len(token.Value)-1]
+	// Single quoted, no escaping.
+	if kind == '\'' {
+		return token, nil
+	}
+	out := strings.Builder{}
+	state := uqDefault
+	// Digits being collected in hex/octal modes
+	var digits string
+	for _, rn := range token.Value {
+		switch state {
+		case uqEscape:
+			if rn == 'x' { // nolint: gocritic
+				state = uqHex
+			} else if unicode.Is(unicode.Digit, rn) {
+				state = uqOctal
+				digits = string(rn)
+			} else {
+				escaped, ok := escapeTable[rn]
+				if !ok {
+					return token, fmt.Errorf("%s: %q: unknown escape sequence \"\\%c\"", token.Pos, token.Value, rn)
+				}
+				out.WriteRune(escaped)
+				state = uqDefault
+			}
+			continue
+		case uqHex:
+			if unicode.Is(unicode.ASCII_Hex_Digit, rn) && len(digits) < 2 {
+				digits += string(rn)
+				continue
+			}
+			if err := flushDigits(digits, 16, &out); err != nil {
+				return token, fmt.Errorf("%s: %w", token.Pos, err)
+			}
+			state = uqDefault
+			digits = ""
+		case uqOctal:
+			if unicode.IsDigit(rn) && len(digits) < 3 {
+				digits += string(rn)
+				continue
+			}
+			if err := flushDigits(digits, 8, &out); err != nil {
+				return token, fmt.Errorf("%s: %w", token.Pos, err)
+			}
+			state = uqDefault
+			digits = ""
+		case uqDefault:
+		default:
+			panic(state)
+		}
+		if rn == '\\' {
+			state = uqEscape
+		} else {
+			out.WriteRune(rn)
+		}
+	}
+	if digits != "" {
+		if err := flushDigits(digits, stateBase[state], &out); err != nil {
+			return token, fmt.Errorf("%s: %w", token.Pos, err)
+		}
+	}
+	token.Value = out.String()
+	return token, nil
+}
+
+func flushDigits(digits string, base int, out *strings.Builder) error {
+	n, err := strconv.ParseUint(digits, base, 32)
+	if err != nil {
+		return fmt.Errorf("invalid base %d numeric value %s", base, digits)
+	}
+	if n > 255 {
+		return fmt.Errorf("base %d value %s larger than 255", base, digits)
+	}
+	out.WriteByte(byte(n))
+	return nil
+}

compiler/parser/lexer_test.go

@@ -0,0 +1,32 @@
+// This code is derived from https://github.com/alecthomas/protobuf
+
+package parser
+
+import (
+	"testing"
+
+	"github.com/alecthomas/participle/v2/lexer"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnquote(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{`"\n\027"`, "\n\027"},
+		{`"\?"`, "\x3f"},
+		{`'\n\027'`, `\n\027`},
+		{`"\n\x17"`, "\n\027"},
+		{`"hello\0world"`, "hello\000world"},
+		{`"hello\x0world"`, "hello\000world"},
+		{`"\0001"`, "\0001"},
+		{`"\x001"`, "\x001"},
+		{`"\341\210\264"`, "ሴ"},
+	}
+	for _, test := range tests {
+		actual, err := unquote(lexer.Token{Value: test.input})
+		require.NoError(t, err)
+		require.Equal(t, test.expected, actual.Value)
+	}
+}