diff --git a/app/pkg/agent/agent.go b/app/pkg/agent/agent.go index 44354ce..bcde365 100644 --- a/app/pkg/agent/agent.go +++ b/app/pkg/agent/agent.go @@ -6,6 +6,8 @@ import ( "strings" "sync" + "github.com/jlewi/foyle/app/pkg/runme/ulid" + "github.com/jlewi/foyle/protos/go/foyle/v1alpha1/v1alpha1connect" "google.golang.org/protobuf/encoding/protojson" @@ -114,6 +116,8 @@ func (a *Agent) Generate(ctx context.Context, req *v1alpha1.GenerateRequest) (*v return nil, err } + log.Info(logs.Level1Assertion, "assertion", logs.BuildAssertion(v1alpha1.Assertion_AT_LEAST_ONE_BLOCK_POST_PROCESSED, len(postProcessed) > 0)) + // Attach block ids to any blocks generated. // N.B. This is kind of a last resort to make sure all blocks have an ID set. In general, we want to set blockIds // earlier in the processing pipeline so that any log messages involving blocks has block ids set. BlockIDs @@ -148,8 +152,9 @@ func (a *Agent) completeWithRetries(ctx context.Context, req *v1alpha1.GenerateR }) } for try := 0; try < maxTries; try++ { + docText := t.Text() args := promptArgs{ - Document: t.Text(), + Document: docText, Examples: exampleArgs, } @@ -172,6 +177,29 @@ func (a *Agent) completeWithRetries(ctx context.Context, req *v1alpha1.GenerateR return nil, errors.Wrapf(err, "CreateChatCompletion failed") } + // Level1 assertion that docText is a non-empty string + assertion := &v1alpha1.Assertion{ + Name: v1alpha1.Assertion_NON_EMPTY_DOC, + Result: v1alpha1.AssertResult_PASSED, + Id: ulid.GenerateID(), + } + + if len(strings.TrimSpace(docText)) == 0 { + assertion.Result = v1alpha1.AssertResult_FAILED + } + + log.Info(logs.Level1Assertion, "assertion", assertion) + + assertBlocks := &v1alpha1.Assertion{ + Name: v1alpha1.Assertion_AT_LEAST_ONE_BLOCK, + Result: v1alpha1.AssertResult_PASSED, + Id: ulid.GenerateID(), + } + + if len(blocks) == 0 { + assertBlocks.Result = v1alpha1.AssertResult_FAILED + } + log.Info(logs.Level1Assertion, "assertion", assertion) return blocks, nil } err := errors.Errorf("Failed to generate a chat completion after %d tries", maxTries) diff --git a/app/pkg/analyze/fsql/eval_query.sql b/app/pkg/analyze/fsql/eval_query.sql index 49200b3..9f1085d 100644 --- a/app/pkg/analyze/fsql/eval_query.sql +++ b/app/pkg/analyze/fsql/eval_query.sql @@ -16,4 +16,14 @@ WHERE id = ?; SELECT * FROM results WHERE (:cursor = '' OR time < :cursor) ORDER BY time DESC - LIMIT :page_size; \ No newline at end of file + LIMIT :page_size; + +-- name: CountResults :one +-- Count the total number of results +SELECT COUNT(*) FROM results; + +-- name: CountErrors :one +SELECT COUNT(*) FROM results WHERE json_extract(proto_json, '$.error') IS NOT NULL; + +-- name: CountByCellsMatchResult :many +SELECT json_extract(proto_json, '$.cellsMatchResult') as match_result, COUNT(*) as count FROM results GROUP BY match_result; \ No newline at end of file diff --git a/app/pkg/analyze/fsql/eval_query.sql.go b/app/pkg/analyze/fsql/eval_query.sql.go index af3e775..af38684 100644 --- a/app/pkg/analyze/fsql/eval_query.sql.go +++ b/app/pkg/analyze/fsql/eval_query.sql.go @@ -10,6 +10,61 @@ import ( "time" ) +const countByCellsMatchResult = `-- name: CountByCellsMatchResult :many +SELECT json_extract(proto_json, '$.cellsMatchResult') as match_result, COUNT(*) as count FROM results GROUP BY match_result +` + +type CountByCellsMatchResultRow struct { + MatchResult interface{} + Count int64 +} + +func (q *Queries) CountByCellsMatchResult(ctx context.Context) ([]CountByCellsMatchResultRow, error) { + rows, err := q.db.QueryContext(ctx, countByCellsMatchResult) + if err != nil { + return nil, err + } + defer rows.Close() + var items []CountByCellsMatchResultRow + for rows.Next() { + var i CountByCellsMatchResultRow + if err := rows.Scan(&i.MatchResult, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const countErrors = `-- name: CountErrors :one +SELECT COUNT(*) FROM results WHERE json_extract(proto_json, '$.error') IS NOT NULL +` + +func (q *Queries) CountErrors(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countErrors) + var count int64 + err := row.Scan(&count) + return count, err +} + +const countResults = `-- name: CountResults :one +SELECT COUNT(*) FROM results +` + +// Count the total number of results +func (q *Queries) CountResults(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countResults) + var count int64 + err := row.Scan(&count) + return count, err +} + const getResult = `-- name: GetResult :one SELECT id, time, proto_json FROM results WHERE id = ? diff --git a/app/pkg/eval/assert.go b/app/pkg/eval/assert.go index a4000cc..a58c4e7 100644 --- a/app/pkg/eval/assert.go +++ b/app/pkg/eval/assert.go @@ -6,6 +6,8 @@ import ( "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" ) +// TODO(jeremy): A lot of this code is probably obsolete now that we are using protos. + // Assertion is an interface for evaluating AI generations. type Assertion interface { Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) @@ -14,7 +16,3 @@ type Assertion interface { } type AssertResult string - -const AssertPassed AssertResult = "passed" -const AssertFailed AssertResult = "failed" -const AssertSkipped AssertResult = "skipped" diff --git a/app/pkg/eval/assertions.go b/app/pkg/eval/assertions.go index e86a9ee..ebef665 100644 --- a/app/pkg/eval/assertions.go +++ b/app/pkg/eval/assertions.go @@ -20,7 +20,7 @@ type AssertCodeAfterMarkdown struct { func (a *AssertCodeAfterMarkdown) Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) { assertion := &v1alpha1.Assertion{ - Name: a.Name(), + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, } if len(doc.Blocks) == 0 { @@ -62,7 +62,7 @@ type AssertOneCodeCell struct { func (a *AssertOneCodeCell) Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) { assertion := &v1alpha1.Assertion{ - Name: a.Name(), + Name: v1alpha1.Assertion_ONE_CODE_CELL, } if len(doc.Blocks) == 0 { @@ -99,7 +99,7 @@ type AssertEndsWithCodeCell struct { func (a *AssertEndsWithCodeCell) Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) { assertion := &v1alpha1.Assertion{ - Name: a.Name(), + Name: v1alpha1.Assertion_ENDS_WITH_CODE_CELL, } if len(doc.Blocks) == 0 { diff --git a/app/pkg/eval/assertions_test.go b/app/pkg/eval/assertions_test.go index dde3b27..daa3b8d 100644 --- a/app/pkg/eval/assertions_test.go +++ b/app/pkg/eval/assertions_test.go @@ -23,7 +23,7 @@ func TestAssertCodeAfterMarkdown(t *testing.T) { examples: []*v1alpha1.Example{}, answer: []*v1alpha1.Block{}, expected: &v1alpha1.Assertion{ - Name: "AssertCodeAfterMarkdown", + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_SKIPPED, }, }, @@ -43,7 +43,7 @@ func TestAssertCodeAfterMarkdown(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: "AssertCodeAfterMarkdown", + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_PASSED, }, }, @@ -63,7 +63,7 @@ func TestAssertCodeAfterMarkdown(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: "AssertCodeAfterMarkdown", + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_FAILED, }, }, @@ -91,7 +91,7 @@ func TestAssertOneCodeCell(t *testing.T) { examples: []*v1alpha1.Example{}, answer: []*v1alpha1.Block{}, expected: &v1alpha1.Assertion{ - Name: "AssertCodeAfterMarkdown", + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_SKIPPED, }, }, @@ -111,7 +111,7 @@ func TestAssertOneCodeCell(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: CodeAfterMarkdownName, + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_PASSED, }, }, @@ -131,7 +131,7 @@ func TestAssertOneCodeCell(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: OneCodeCellName, + Name: v1alpha1.Assertion_ONE_CODE_CELL, Result: v1alpha1.AssertResult_FAILED, }, }, @@ -159,7 +159,7 @@ func TestAssertEndsWithCodeCell(t *testing.T) { examples: []*v1alpha1.Example{}, answer: []*v1alpha1.Block{}, expected: &v1alpha1.Assertion{ - Name: EndsWithCodeCellName, + Name: v1alpha1.Assertion_ENDS_WITH_CODE_CELL, Result: v1alpha1.AssertResult_SKIPPED, }, }, @@ -179,7 +179,7 @@ func TestAssertEndsWithCodeCell(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: EndsWithCodeCellName, + Name: v1alpha1.Assertion_ENDS_WITH_CODE_CELL, Result: v1alpha1.AssertResult_PASSED, }, }, @@ -202,7 +202,7 @@ func TestAssertEndsWithCodeCell(t *testing.T) { }, }, expected: &v1alpha1.Assertion{ - Name: EndsWithCodeCellName, + Name: v1alpha1.Assertion_ENDS_WITH_CODE_CELL, Result: v1alpha1.AssertResult_FAILED, }, }, diff --git a/app/pkg/eval/evaluator.go b/app/pkg/eval/evaluator.go index 5396788..2f2458b 100644 --- a/app/pkg/eval/evaluator.go +++ b/app/pkg/eval/evaluator.go @@ -7,6 +7,8 @@ import ( "sort" "time" + "google.golang.org/protobuf/encoding/protojson" + "github.com/go-logr/logr" "connectrpc.com/connect" @@ -144,6 +146,34 @@ func (e *Evaluator) Reconcile(ctx context.Context, experiment api.Experiment) er } log.Info("Successfully processed examples") + + report, err := e.buildExperimentReport(ctx, experiment.Metadata.Name, manager, logsClient) + + if err != nil { + return err + } + + log.Info("Successfully reported results") + + outputDir := filepath.Dir(experiment.Spec.OutputDB) + + reportFile := filepath.Join(outputDir, "report.json") + + opts := protojson.MarshalOptions{ + Indent: " ", + EmitDefaultValues: true, + } + reportJson, err := opts.Marshal(report) + if err != nil { + return errors.Wrapf(err, "Failed to marshal report") + } + + if err := os.WriteFile(reportFile, reportJson, 0644); err != nil { + return errors.Wrapf(err, "Failed to write report to file %s", reportFile) + } + + log.Info("Successfully wrote report", "file", reportFile) + return nil } @@ -541,6 +571,129 @@ func (e *Evaluator) reconcileBestRAGResult(ctx context.Context, evalResult *v1al return nil } +// buildExperimentReport generates a report of the experiment results. These are aggregate statistics for the +// experiment +func (e *Evaluator) buildExperimentReport(ctx context.Context, name string, manager *ResultsManager, logsClient logspbconnect.LogsServiceClient) (*v1alpha1.ExperimentReport, error) { + log := logs.FromContext(ctx) + r := &v1alpha1.ExperimentReport{ + Name: name, + } + + numExamples, err := manager.queries.CountResults(ctx) + if err != nil { + return r, errors.Wrapf(err, "Failed to count number of examples") + } + + r.NumExamples = numExamples + + errCount, err := manager.queries.CountErrors(ctx) + if err != nil { + return r, errors.Wrapf(err, "Failed to count errors") + } + + r.NumErrors = errCount + + // Get the counts based on cellsMatchResult + counts, err := manager.queries.CountByCellsMatchResult(ctx) + if err != nil { + return r, errors.Wrapf(err, "Failed to count cellsMatchResult") + } + + r.CellsMatchCounts = make(map[string]int32) + + for _, c := range counts { + s, ok := c.MatchResult.(string) + if !ok { + return r, errors.Wrapf(err, "Failed to convert cellsMatchResult to string") + } + r.CellsMatchCounts[s] = int32(c.Count) + } + + // Compute the 90th, 95th, 99th Percentile of generate time + // And assertionstats + assertionStats := make(map[v1alpha1.Assertion_Name]*v1alpha1.AssertionCounts) + generateTimes := make([]int, 0, numExamples) + var cursor *time.Time + for { + var listErr error + var results []*v1alpha1.EvalResult + results, cursor, listErr = manager.ListResults(ctx, cursor, 100) + if listErr != nil { + return r, errors.Wrapf(listErr, "Failed to list results") + } + + if len(results) == 0 { + break + } + for _, result := range results { + generateTimes = append(generateTimes, int(result.GenerateTimeMs)) + + // Get the Level1 assertions for this trace + if result.GetGenTraceId() != "" { + assertions, err := getAssertions(ctx, result.GetGenTraceId(), logsClient) + if err != nil { + log.Error(err, "Failed to get assertions", "targetTraceId", result.GetGenTraceId(), "exampleId", result.GetExample().GetId()) + continue + } + + accumulateAssertionCounts(assertionStats, assertions) + } + } + } + + percentiles, err := computePercentilesOfInts(generateTimes, []float64{.9, .95}) + if err != nil { + return r, errors.Wrapf(err, "Failed to compute percentiles") + } + + r.GenerateLatencyStats = percentiles + + // Add the assertions in sorted order based on key + statKeys := make([]string, 0, len(assertionStats)) + for k := range assertionStats { + statKeys = append(statKeys, k.String()) + } + + r.AssertionCounts = make([]*v1alpha1.AssertionCounts, 0, len(assertionStats)) + for _, key := range statKeys { + stat := assertionStats[v1alpha1.Assertion_Name(v1alpha1.Assertion_Name_value[key])] + r.AssertionCounts = append(r.AssertionCounts, stat) + } + return r, nil +} + +func accumulateAssertionCounts(stats map[v1alpha1.Assertion_Name]*v1alpha1.AssertionCounts, assertions []*v1alpha1.Assertion) { + for _, assertion := range assertions { + if _, ok := stats[assertion.GetName()]; !ok { + stats[assertion.GetName()] = &v1alpha1.AssertionCounts{} + } + + switch assertion.GetResult() { + case v1alpha1.AssertResult_PASSED: + stats[assertion.GetName()].Passed++ + case v1alpha1.AssertResult_FAILED: + stats[assertion.GetName()].Failed++ + case v1alpha1.AssertResult_UNKNOWN_AssertResult: + stats[assertion.GetName()].Unknown++ + case v1alpha1.AssertResult_SKIPPED: + stats[assertion.GetName()].Skipped++ + } + } +} + +func getAssertions(ctx context.Context, traceId string, client logspbconnect.LogsServiceClient) ([]*v1alpha1.Assertion, error) { + resp, err := client.GetTrace(ctx, connect.NewRequest(&logspb.GetTraceRequest{ + Id: traceId, + })) + + if err != nil { + return nil, errors.Wrapf(err, "Failed to get trace %s", traceId) + } + + trace := resp.Msg.GetTrace() + return trace.Assertions, nil +} + // isSortedByTimeDescending checks if the slice is sorted by Time in descending order func isSortedByTimeDescending(slice []*v1alpha1.EvalResult) bool { for i := 1; i < len(slice); i++ { diff --git a/app/pkg/eval/evaluator_test.go b/app/pkg/eval/evaluator_test.go index c06694e..ee854cf 100644 --- a/app/pkg/eval/evaluator_test.go +++ b/app/pkg/eval/evaluator_test.go @@ -6,6 +6,10 @@ import ( "path/filepath" "testing" + "github.com/jlewi/foyle/protos/go/foyle/logs/logspbconnect" + "google.golang.org/protobuf/encoding/protojson" + "gopkg.in/yaml.v3" + "connectrpc.com/connect" "github.com/go-logr/zapr" "github.com/jlewi/foyle/app/pkg/agent" @@ -222,3 +226,49 @@ func experimentForTesting() (*api.Experiment, error) { }, }, nil } + +func Test_buildExperimentReport(t *testing.T) { + // N.B. This is an integration test because it depends on an actual set of experiment results. + // It also potentially needs a running agent that we can use to access the traces of + if os.Getenv("GITHUB_ACTIONS") != "" { + t.Skipf("Test is skipped in GitHub actions") + } + + experimentFile := "/Users/jlewi/foyle_experiments/20241014-timing/experiment.yaml" + experimentBytes, err := os.ReadFile(experimentFile) + if err != nil { + t.Fatalf("Error reading experiment file; %v", err) + } + + experiment := &api.Experiment{} + if err := yaml.Unmarshal(experimentBytes, experiment); err != nil { + t.Fatalf("Error unmarshalling experiment; %v", err) + } + + resultsManager, err := openResultsManager(experiment.Spec.OutputDB) + + if err != nil { + t.Fatalf("Error opening results manager; %v", err) + } + + logsClient := logspbconnect.NewLogsServiceClient( + newHTTPClient(), + experiment.Spec.AgentAddress, + ) + + e := &Evaluator{} + report, err := e.buildExperimentReport(context.Background(), "testexperiment", resultsManager, logsClient) + if err != nil { + t.Fatalf("Error building report; %v", err) + } + + opts := protojson.MarshalOptions{ + Indent: " ", + EmitDefaultValues: true, + } + reportJson, err := opts.Marshal(report) + if err != nil { + t.Fatalf("Error marshalling report; %v", err) + } + t.Logf("Report: %v", string(reportJson)) +} diff --git a/app/pkg/eval/report_test.go b/app/pkg/eval/report_test.go index a6e7fb1..54eb4c1 100644 --- a/app/pkg/eval/report_test.go +++ b/app/pkg/eval/report_test.go @@ -26,7 +26,7 @@ func Test_buildReport(t *testing.T) { ExpectedCells: []*parserv1.Cell{ { Kind: parserv1.CellKind_CELL_KIND_CODE, - Value: "expected cell value", + Value: "expected cell Value", }, }, }, diff --git a/app/pkg/eval/results_manager.go b/app/pkg/eval/results_manager.go index 80512c5..f35541b 100644 --- a/app/pkg/eval/results_manager.go +++ b/app/pkg/eval/results_manager.go @@ -90,7 +90,7 @@ func (m *ResultsManager) Get(ctx context.Context, id string) (*v1alpha1.EvalResu // If the updateFunc returns an error then the example is not updated. // If the given id doesn't exist then an empty Session is passed to updateFunc and the result will be // inserted if the updateFunc returns nil. If the session result exists then the result is passed to updateFunc -// and the updated value is then written to the database +// and the updated Value is then written to the database // // TODO(jeremy): How should the update function signal an error that shouldn't block the update and should be reported // by Update. For example, when processing a result; we might have an error processing an example (e.g. generating @@ -176,6 +176,7 @@ func (m *ResultsManager) Update(ctx context.Context, id string, updateFunc EvalR // ListResults lists the results in the database if cursor is nil then the first page is returned. // If cursor is non-nil then the next page is returned. // The cursor is the time. +// Returns empty list of results when no more results. func (m *ResultsManager) ListResults(ctx context.Context, cursor *time.Time, pageSize int) ([]*v1alpha1.EvalResult, *time.Time, error) { params := fsql.ListResultsParams{ PageSize: int64(pageSize), diff --git a/app/pkg/eval/service.go b/app/pkg/eval/service.go index 70467a1..0dadeda 100644 --- a/app/pkg/eval/service.go +++ b/app/pkg/eval/service.go @@ -76,13 +76,13 @@ func (s *EvalServer) List( value, err := iter.ValueAndErr() if err != nil { - log.Error(err, "Failed to read value for key", "key", string(key)) + log.Error(err, "Failed to read Value for key", "key", string(key)) continue } result := &v1alpha1.EvalResult{} if err := proto.Unmarshal(value, result); err != nil { - log.Error(err, "Failed to unmarshal value for", "key", string(key)) + log.Error(err, "Failed to unmarshal Value for", "key", string(key)) continue } results.Items = append(results.Items, result) @@ -194,13 +194,13 @@ func (s *EvalServer) AssertionTable( value, err := iter.ValueAndErr() if err != nil { - log.Error(err, "Failed to read value for key", "key", string(key)) + log.Error(err, "Failed to read Value for key", "key", string(key)) continue } result := &v1alpha1.EvalResult{} if err := proto.Unmarshal(value, result); err != nil { - log.Error(err, "Failed to unmarshal value for", "key", string(key)) + log.Error(err, "Failed to unmarshal Value for", "key", string(key)) continue } @@ -263,11 +263,11 @@ func toAssertionRow(result *v1alpha1.EvalResult) (*v1alpha1.AssertionRow, error) for _, a := range result.GetAssertions() { switch a.Name { - case CodeAfterMarkdownName: + case v1alpha1.Assertion_CODE_AFTER_MARKDOWN: row.CodeAfterMarkdown = a.GetResult() - case OneCodeCellName: + case v1alpha1.Assertion_ONE_CODE_CELL: row.OneCodeCell = a.GetResult() - case EndsWithCodeCellName: + case v1alpha1.Assertion_ENDS_WITH_CODE_CELL: row.EndsWithCodeCell = a.GetResult() default: log.Info("Unknown assertion", "name", a.Name) diff --git a/app/pkg/eval/service_test.go b/app/pkg/eval/service_test.go index 355b91d..f720c89 100644 --- a/app/pkg/eval/service_test.go +++ b/app/pkg/eval/service_test.go @@ -39,7 +39,7 @@ func Test_ToAssertRow(t *testing.T) { }, Assertions: []*v1alpha1.Assertion{ { - Name: "AssertCodeAfterMarkdown", + Name: v1alpha1.Assertion_CODE_AFTER_MARKDOWN, Result: v1alpha1.AssertResult_PASSED, }, }, diff --git a/app/pkg/eval/stats.go b/app/pkg/eval/stats.go new file mode 100644 index 0000000..78c4f0a --- /dev/null +++ b/app/pkg/eval/stats.go @@ -0,0 +1,49 @@ +package eval + +import ( + "math" + "sort" + + "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" + "github.com/pkg/errors" +) + +// computePercentilesOfInts computes the percentiles of a slice of integers. +// p is a slice of percentiles to compute. Values should be between > 0 and <1. +func computePercentilesOfInts(data []int, p []float64) ([]*v1alpha1.PercentileStat, error) { + sort.Ints(data) + + indexes := map[int]bool{} + for _, p := range p { + if p < 0 || p >= 1 { + return nil, errors.Errorf("Percentile %v is not between 0 and 1", p) + } + + actual := p*float64(len(data)) - 1 + index := int(math.Floor(actual)) + indexes[index] = true + if actual != math.Floor(actual) && index+1 < len(data) { + indexes[index+1] = true + } + } + + // Extract keys from the map + var keys []int + for k := range indexes { + keys = append(keys, k) + } + + // Sort the keys + sort.Ints(keys) + + percentiles := make([]*v1alpha1.PercentileStat, 0, len(p)) + for _, k := range keys { + // Compute the Value at the kth index + percentiles = append(percentiles, &v1alpha1.PercentileStat{ + Percentile: float64(k+1) / float64(len(data)), + Value: float64(data[k]), + }) + } + + return percentiles, nil +} diff --git a/app/pkg/eval/stats_test.go b/app/pkg/eval/stats_test.go new file mode 100644 index 0000000..c35e82b --- /dev/null +++ b/app/pkg/eval/stats_test.go @@ -0,0 +1,73 @@ +package eval + +import ( + "testing" + + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/google/go-cmp/cmp" + "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" +) + +func Test_computePercentilesInts(t *testing.T) { + type testCase struct { + name string + data []int + percentiles []float64 + expected []*v1alpha1.PercentileStat + } + + cases := []testCase{ + { + name: "Basic", + data: []int{2, 1, 4, 3, 5}, + percentiles: []float64{0.5, .8, .99}, + expected: []*v1alpha1.PercentileStat{ + { + Percentile: 0.4, + Value: 2, + }, + { + Percentile: 0.6, + Value: 3, + }, + { + Percentile: 0.8, + Value: 4, + }, + { + Percentile: 1, + Value: 5, + }, + }, + }, + { + name: "up and down", + data: []int{2, 1, 4, 3}, + percentiles: []float64{0.60}, + expected: []*v1alpha1.PercentileStat{ + { + Percentile: 0.5, + Value: 2, + }, + { + Percentile: 0.75, + Value: 3, + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + actual, err := computePercentilesOfInts(c.data, c.percentiles) + if err != nil { + t.Fatalf("Error computing percentiles: %v", err) + } + + if d := cmp.Diff(c.expected, actual, cmpopts.IgnoreUnexported(v1alpha1.PercentileStat{})); d != "" { + t.Fatalf("Unexpected diff between expected and actual percentiles:\n%+v", d) + } + }) + } +} diff --git a/app/pkg/logs/logs.go b/app/pkg/logs/logs.go index 861d38a..6b1d570 100644 --- a/app/pkg/logs/logs.go +++ b/app/pkg/logs/logs.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" + "github.com/jlewi/foyle/app/pkg/runme/ulid" + "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" + "github.com/jlewi/foyle/app/api" "google.golang.org/protobuf/encoding/protojson" @@ -21,6 +24,9 @@ const ( // Debug is for debug verbosity level Debug = 1 + + // Level1Assertion Message denoting a level1 assertion + Level1Assertion = "Level1Assert" ) // FromContext returns a logr.Logger from the context or an instance of the global logger @@ -70,3 +76,17 @@ func LogLLMUsage(ctx context.Context, usage api.LLMUsage) { log := FromContext(ctx) log.Info("LLM usage", "usage", usage) } + +// BuildAssertion creates an assertion based on the name. +// N.B. We don't put this in the eval package because that would create a circular dependency. +func BuildAssertion(name v1alpha1.Assertion_Name, passed bool) *v1alpha1.Assertion { + result := v1alpha1.AssertResult_FAILED + if passed { + result = v1alpha1.AssertResult_PASSED + } + return &v1alpha1.Assertion{ + Name: name, + Result: result, + Id: ulid.GenerateID(), + } +} diff --git a/protos/foyle/logs/traces.proto b/protos/foyle/logs/traces.proto index 821ba98..c1132ef 100644 --- a/protos/foyle/logs/traces.proto +++ b/protos/foyle/logs/traces.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "foyle/logs/logs.proto"; import "foyle/v1alpha1/agent.proto"; +import "foyle/v1alpha1/eval.proto"; import "foyle/v1alpha1/providers.proto"; import "foyle/v1alpha1/trainer.proto"; import "foyle/logs/blocks.proto"; @@ -30,6 +31,10 @@ message Trace { repeated Span spans = 8; + // Assertions about this trace. + // Should these be properties of the spans? We'll cross that bridge if we come to it. + repeated Assertion assertions = 9; + reserved 5,7; } diff --git a/protos/foyle/v1alpha1/eval.proto b/protos/foyle/v1alpha1/eval.proto index e4aadbb..4869dd7 100644 --- a/protos/foyle/v1alpha1/eval.proto +++ b/protos/foyle/v1alpha1/eval.proto @@ -82,11 +82,29 @@ enum BlockLogStatus { } message Assertion { + + enum Name { + UNKNOWN = 0; + CODE_AFTER_MARKDOWN = 1; + ONE_CODE_CELL = 2; + ENDS_WITH_CODE_CELL = 3; + // NON_EMPTY_DOC asserts that the document sent to the LLM is non empty + NON_EMPTY_DOC = 4; + // AT_LEAST_ONE_BLOCK asserts that at at least one block was generated from the response before post processing + AT_LEAST_ONE_BLOCK = 5; + // AT_LEAST_ONE_POST_PROCESSED asserts that at at least one block is returned after post-processing + AT_LEAST_ONE_BLOCK_POST_PROCESSED = 6; + } // Name of the assertion - string name = 1; + Name name = 1; AssertResult result = 2; // Human readable detail of the assertion. If there was an error this should contain the error message. string detail = 3; + + // id is a unique id of the assertion. This is needed for real time processing of the logs. Since our log + // processing guarantees at least once semantics, we may end up processing the same log entry about an assertion + // multiple times. By assigning a unique id to each assertion we can dedupe them. + string id = 4; } message EvalResultListRequest { @@ -159,3 +177,38 @@ message GetEvalResultRequest { message GetEvalResultResponse { string reportHTML = 1; } + +// ExperimentReport is a report for an experiment +message ExperimentReport { + string name = 1; + + // num_examples is the number of examples evaluated in the experiment + int64 num_examples = 2; + + // num_errors is the number of examples that resulted in an error + int64 num_errors = 3; + + // Map from string representation of CellsMatchResult to the number of counts + map cells_match_counts = 4; + + repeated AssertionCounts assertion_counts = 5; + + repeated PercentileStat generate_latency_stats = 6; +} + +message AssertionCounts { + Assertion.Name name = 1; + int32 passed = 2; + int32 failed = 3; + int32 unknown = 4; + int32 skipped = 5; +} + +// PercentileStat represents a percentile value +message PercentileStat { + // The percentile a value 0 to 1 + double percentile = 1; + + // The value of the percentile + double value = 2; +} diff --git a/protos/go/foyle/logs/conversion.zap.go b/protos/go/foyle/logs/conversion.zap.go index 8e925a3..8aaaf86 100644 --- a/protos/go/foyle/logs/conversion.zap.go +++ b/protos/go/foyle/logs/conversion.zap.go @@ -7,9 +7,9 @@ import ( fmt "fmt" math "math" proto "github.com/golang/protobuf/proto" + _ "google.golang.org/protobuf/types/known/timestamppb" _ "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" _ "google.golang.org/protobuf/types/known/structpb" - _ "google.golang.org/protobuf/types/known/timestamppb" go_uber_org_zap_zapcore "go.uber.org/zap/zapcore" ) diff --git a/protos/go/foyle/logs/traces.pb.go b/protos/go/foyle/logs/traces.pb.go index ae66074..bee7ae3 100644 --- a/protos/go/foyle/logs/traces.pb.go +++ b/protos/go/foyle/logs/traces.pb.go @@ -41,6 +41,9 @@ type Trace struct { // Eval mode is true if the trace was generated in eval mode. EvalMode bool `protobuf:"varint,6,opt,name=eval_mode,json=evalMode,proto3" json:"eval_mode,omitempty"` Spans []*Span `protobuf:"bytes,8,rep,name=spans,proto3" json:"spans,omitempty"` + // Assertions about this trace. + // Should these be properties of the spans? We'll cross that bridge if we come to it. + Assertions []*v1alpha1.Assertion `protobuf:"bytes,9,rep,name=assertions,proto3" json:"assertions,omitempty"` } func (x *Trace) Reset() { @@ -124,6 +127,13 @@ func (x *Trace) GetSpans() []*Span { return nil } +func (x *Trace) GetAssertions() []*v1alpha1.Assertion { + if x != nil { + return x.Assertions + } + return nil +} + type isTrace_Data interface { isTrace_Data() } @@ -780,128 +790,133 @@ var file_foyle_logs_traces_proto_rawDesc = []byte{ 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x1a, 0x15, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x6c, 0x6f, 0x67, 0x73, 0x2f, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1a, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x6c, 0x6f, - 0x67, 0x73, 0x2f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x1c, 0x72, 0x75, 0x6e, 0x6d, 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x2f, 0x76, 0x31, - 0x2f, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x72, 0x75, - 0x6e, 0x6d, 0x65, 0x2f, 0x70, 0x61, 0x72, 0x73, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, - 0x72, 0x73, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x9b, 0x02, 0x0a, 0x05, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x08, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, - 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x72, - 0x61, 0x63, 0x65, 0x48, 0x00, 0x52, 0x08, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x65, 0x76, 0x61, 0x6c, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x08, 0x65, 0x76, 0x61, 0x6c, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x26, 0x0a, 0x05, - 0x73, 0x70, 0x61, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x6f, - 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x53, 0x70, 0x61, 0x6e, 0x52, 0x05, 0x73, - 0x70, 0x61, 0x6e, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x05, - 0x10, 0x06, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x22, 0x70, 0x0a, 0x04, 0x53, 0x70, 0x61, 0x6e, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x27, 0x0a, 0x03, 0x72, 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x52, 0x41, 0x47, 0x53, 0x70, - 0x61, 0x6e, 0x48, 0x00, 0x52, 0x03, 0x72, 0x61, 0x67, 0x12, 0x27, 0x0a, 0x03, 0x6c, 0x6c, 0x6d, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x73, 0x2e, 0x4c, 0x4c, 0x4d, 0x53, 0x70, 0x61, 0x6e, 0x48, 0x00, 0x52, 0x03, 0x6c, - 0x6c, 0x6d, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x45, 0x0a, 0x07, 0x52, 0x41, - 0x47, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x07, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x52, - 0x41, 0x47, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x22, 0x7d, 0x0a, 0x07, 0x4c, 0x4c, 0x4d, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x2a, 0x0a, 0x08, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, - 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x72, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4a, 0x73, 0x6f, 0x6e, - 0x22, 0x6a, 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x72, 0x61, 0x63, - 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, - 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x11, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4d, 0x0a, 0x0a, - 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, - 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x65, 0x73, - 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x21, 0x0a, 0x0f, 0x47, - 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x3b, - 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x22, 0x24, 0x0a, 0x12, 0x47, - 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x22, 0x48, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, - 0x6b, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x66, 0x6f, - 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, - 0x67, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x22, 0x49, 0x0a, 0x11, 0x47, - 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x19, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x74, 0x72, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x6c, - 0x6f, 0x67, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, - 0x6f, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x4c, - 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x74, 0x6d, 0x6c, - 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x68, 0x74, 0x6d, - 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x48, 0x74, 0x6d, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4a, 0x73, 0x6f, 0x6e, 0x32, 0xc8, 0x02, - 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x47, 0x0a, - 0x08, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x1b, 0x2e, 0x66, 0x6f, 0x79, 0x6c, - 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, - 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x12, 0x1e, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, - 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, - 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, - 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, - 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, - 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x20, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, - 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, - 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x9a, 0x01, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, - 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x42, 0x0b, 0x54, 0x72, 0x61, - 0x63, 0x65, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, - 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, - 0x6c, 0x65, 0x2f, 0x6c, 0x6f, 0x67, 0x73, 0x3b, 0x6c, 0x6f, 0x67, 0x73, 0x70, 0x62, 0xa2, 0x02, - 0x03, 0x46, 0x4c, 0x58, 0xaa, 0x02, 0x0a, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x4c, 0x6f, 0x67, - 0x73, 0xca, 0x02, 0x0a, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x5c, 0x4c, 0x6f, 0x67, 0x73, 0xe2, 0x02, - 0x16, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x5c, 0x4c, 0x6f, 0x67, 0x73, 0x5c, 0x47, 0x50, 0x42, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0b, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x3a, - 0x3a, 0x4c, 0x6f, 0x67, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x76, 0x61, 0x6c, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2f, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x17, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x6c, 0x6f, 0x67, 0x73, 0x2f, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x72, 0x75, 0x6e, 0x6d, + 0x65, 0x2f, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x2f, 0x72, 0x75, 0x6e, 0x6e, + 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x72, 0x75, 0x6e, 0x6d, 0x65, 0x2f, 0x70, + 0x61, 0x72, 0x73, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x72, 0x73, 0x65, 0x72, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc7, 0x02, 0x0a, 0x05, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, + 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x37, 0x0a, 0x08, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, + 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x48, 0x00, + 0x52, 0x08, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x76, + 0x61, 0x6c, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x65, + 0x76, 0x61, 0x6c, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x73, 0x70, 0x61, 0x6e, 0x73, + 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, + 0x6f, 0x67, 0x73, 0x2e, 0x53, 0x70, 0x61, 0x6e, 0x52, 0x05, 0x73, 0x70, 0x61, 0x6e, 0x73, 0x12, + 0x2a, 0x0a, 0x0a, 0x61, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x0a, 0x61, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x22, + 0x70, 0x0a, 0x04, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x03, 0x72, 0x61, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, + 0x73, 0x2e, 0x52, 0x41, 0x47, 0x53, 0x70, 0x61, 0x6e, 0x48, 0x00, 0x52, 0x03, 0x72, 0x61, 0x67, + 0x12, 0x27, 0x0a, 0x03, 0x6c, 0x6c, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x4c, 0x4c, 0x4d, 0x53, 0x70, + 0x61, 0x6e, 0x48, 0x00, 0x52, 0x03, 0x6c, 0x6c, 0x6d, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x22, 0x45, 0x0a, 0x07, 0x52, 0x41, 0x47, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x14, 0x0a, 0x05, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x12, 0x24, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x52, 0x41, 0x47, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, + 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x7d, 0x0a, 0x07, 0x4c, 0x4c, 0x4d, 0x53, + 0x70, 0x61, 0x6e, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, + 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x73, + 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x6a, + 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x4a, 0x73, 0x6f, 0x6e, 0x22, 0x6a, 0x0a, 0x0d, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x4d, 0x0a, 0x0a, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x69, 0x65, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x21, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x3b, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, + 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x22, 0x24, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x48, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x42, + 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x31, 0x0a, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6c, 0x6f, 0x67, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, + 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x08, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, + 0x6f, 0x67, 0x22, 0x49, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x6c, 0x6f, 0x67, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x46, 0x69, 0x6c, 0x65, 0x22, 0xa4, 0x01, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, + 0x68, 0x74, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x48, 0x74, 0x6d, 0x6c, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x5f, 0x68, 0x74, 0x6d, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x74, 0x6d, 0x6c, 0x12, 0x21, 0x0a, 0x0c, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4a, 0x73, 0x6f, 0x6e, 0x12, + 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x4a, 0x73, 0x6f, 0x6e, 0x32, 0xc8, 0x02, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x73, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x47, 0x0a, 0x08, 0x47, 0x65, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, + 0x12, 0x1b, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, + 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, + 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, + 0x0b, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x12, 0x1e, 0x2e, 0x66, + 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, + 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, + 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, + 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x4d, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x12, 0x1d, 0x2e, + 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x4c, + 0x4d, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, + 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x4c, 0x4d, + 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4f, + 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x20, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, + 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x6f, 0x79, + 0x6c, 0x65, 0x2e, 0x6c, 0x6f, 0x67, 0x73, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x73, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x9a, 0x01, 0x0a, 0x0e, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2e, 0x6c, 0x6f, + 0x67, 0x73, 0x42, 0x0b, 0x54, 0x72, 0x61, 0x63, 0x65, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, + 0x01, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, + 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x6c, 0x6f, 0x67, 0x73, 0x3b, 0x6c, + 0x6f, 0x67, 0x73, 0x70, 0x62, 0xa2, 0x02, 0x03, 0x46, 0x4c, 0x58, 0xaa, 0x02, 0x0a, 0x46, 0x6f, + 0x79, 0x6c, 0x65, 0x2e, 0x4c, 0x6f, 0x67, 0x73, 0xca, 0x02, 0x0a, 0x46, 0x6f, 0x79, 0x6c, 0x65, + 0x5c, 0x4c, 0x6f, 0x67, 0x73, 0xe2, 0x02, 0x16, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x5c, 0x4c, 0x6f, + 0x67, 0x73, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x0b, 0x46, 0x6f, 0x79, 0x6c, 0x65, 0x3a, 0x3a, 0x4c, 0x6f, 0x67, 0x73, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -931,40 +946,42 @@ var file_foyle_logs_traces_proto_goTypes = []interface{}{ (*GetLLMLogsRequest)(nil), // 10: foyle.logs.GetLLMLogsRequest (*GetLLMLogsResponse)(nil), // 11: foyle.logs.GetLLMLogsResponse (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp - (*v1alpha1.RAGResult)(nil), // 13: RAGResult - (v1alpha1.ModelProvider)(0), // 14: ModelProvider - (*v1alpha1.GenerateRequest)(nil), // 15: GenerateRequest - (*v1alpha1.GenerateResponse)(nil), // 16: GenerateResponse - (*BlockLog)(nil), // 17: foyle.logs.BlockLog - (*GetLogsStatusRequest)(nil), // 18: foyle.logs.GetLogsStatusRequest - (*GetLogsStatusResponse)(nil), // 19: foyle.logs.GetLogsStatusResponse + (*v1alpha1.Assertion)(nil), // 13: Assertion + (*v1alpha1.RAGResult)(nil), // 14: RAGResult + (v1alpha1.ModelProvider)(0), // 15: ModelProvider + (*v1alpha1.GenerateRequest)(nil), // 16: GenerateRequest + (*v1alpha1.GenerateResponse)(nil), // 17: GenerateResponse + (*BlockLog)(nil), // 18: foyle.logs.BlockLog + (*GetLogsStatusRequest)(nil), // 19: foyle.logs.GetLogsStatusRequest + (*GetLogsStatusResponse)(nil), // 20: foyle.logs.GetLogsStatusResponse } var file_foyle_logs_traces_proto_depIdxs = []int32{ 12, // 0: foyle.logs.Trace.end_time:type_name -> google.protobuf.Timestamp 12, // 1: foyle.logs.Trace.start_time:type_name -> google.protobuf.Timestamp 4, // 2: foyle.logs.Trace.generate:type_name -> foyle.logs.GenerateTrace 1, // 3: foyle.logs.Trace.spans:type_name -> foyle.logs.Span - 2, // 4: foyle.logs.Span.rag:type_name -> foyle.logs.RAGSpan - 3, // 5: foyle.logs.Span.llm:type_name -> foyle.logs.LLMSpan - 13, // 6: foyle.logs.RAGSpan.results:type_name -> RAGResult - 14, // 7: foyle.logs.LLMSpan.provider:type_name -> ModelProvider - 15, // 8: foyle.logs.GenerateTrace.request:type_name -> GenerateRequest - 16, // 9: foyle.logs.GenerateTrace.response:type_name -> GenerateResponse - 0, // 10: foyle.logs.GetTraceResponse.trace:type_name -> foyle.logs.Trace - 17, // 11: foyle.logs.GetBlockLogResponse.block_log:type_name -> foyle.logs.BlockLog - 6, // 12: foyle.logs.LogsService.GetTrace:input_type -> foyle.logs.GetTraceRequest - 8, // 13: foyle.logs.LogsService.GetBlockLog:input_type -> foyle.logs.GetBlockLogRequest - 10, // 14: foyle.logs.LogsService.GetLLMLogs:input_type -> foyle.logs.GetLLMLogsRequest - 18, // 15: foyle.logs.LogsService.Status:input_type -> foyle.logs.GetLogsStatusRequest - 7, // 16: foyle.logs.LogsService.GetTrace:output_type -> foyle.logs.GetTraceResponse - 9, // 17: foyle.logs.LogsService.GetBlockLog:output_type -> foyle.logs.GetBlockLogResponse - 11, // 18: foyle.logs.LogsService.GetLLMLogs:output_type -> foyle.logs.GetLLMLogsResponse - 19, // 19: foyle.logs.LogsService.Status:output_type -> foyle.logs.GetLogsStatusResponse - 16, // [16:20] is the sub-list for method output_type - 12, // [12:16] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 13, // 4: foyle.logs.Trace.assertions:type_name -> Assertion + 2, // 5: foyle.logs.Span.rag:type_name -> foyle.logs.RAGSpan + 3, // 6: foyle.logs.Span.llm:type_name -> foyle.logs.LLMSpan + 14, // 7: foyle.logs.RAGSpan.results:type_name -> RAGResult + 15, // 8: foyle.logs.LLMSpan.provider:type_name -> ModelProvider + 16, // 9: foyle.logs.GenerateTrace.request:type_name -> GenerateRequest + 17, // 10: foyle.logs.GenerateTrace.response:type_name -> GenerateResponse + 0, // 11: foyle.logs.GetTraceResponse.trace:type_name -> foyle.logs.Trace + 18, // 12: foyle.logs.GetBlockLogResponse.block_log:type_name -> foyle.logs.BlockLog + 6, // 13: foyle.logs.LogsService.GetTrace:input_type -> foyle.logs.GetTraceRequest + 8, // 14: foyle.logs.LogsService.GetBlockLog:input_type -> foyle.logs.GetBlockLogRequest + 10, // 15: foyle.logs.LogsService.GetLLMLogs:input_type -> foyle.logs.GetLLMLogsRequest + 19, // 16: foyle.logs.LogsService.Status:input_type -> foyle.logs.GetLogsStatusRequest + 7, // 17: foyle.logs.LogsService.GetTrace:output_type -> foyle.logs.GetTraceResponse + 9, // 18: foyle.logs.LogsService.GetBlockLog:output_type -> foyle.logs.GetBlockLogResponse + 11, // 19: foyle.logs.LogsService.GetLLMLogs:output_type -> foyle.logs.GetLLMLogsResponse + 20, // 20: foyle.logs.LogsService.Status:output_type -> foyle.logs.GetLogsStatusResponse + 17, // [17:21] is the sub-list for method output_type + 13, // [13:17] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name } func init() { file_foyle_logs_traces_proto_init() } diff --git a/protos/go/foyle/logs/traces.zap.go b/protos/go/foyle/logs/traces.zap.go index 5c32b21..ac938ff 100644 --- a/protos/go/foyle/logs/traces.zap.go +++ b/protos/go/foyle/logs/traces.zap.go @@ -7,11 +7,11 @@ import ( fmt "fmt" math "math" proto "github.com/golang/protobuf/proto" - _ "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1" - _ "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" _ "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1" _ "google.golang.org/protobuf/types/known/structpb" _ "google.golang.org/protobuf/types/known/timestamppb" + _ "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1" + _ "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" go_uber_org_zap_zapcore "go.uber.org/zap/zapcore" github_com_golang_protobuf_ptypes "github.com/golang/protobuf/ptypes" ) @@ -70,6 +70,20 @@ func (m *Trace) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) erro return nil })) + keyName = "assertions" // field assertions = 9 + enc.AddArray(keyName, go_uber_org_zap_zapcore.ArrayMarshalerFunc(func(aenc go_uber_org_zap_zapcore.ArrayEncoder) error { + for _, rv := range m.Assertions { + _ = rv + if rv != nil { + var vv interface{} = rv + if marshaler, ok := vv.(go_uber_org_zap_zapcore.ObjectMarshaler); ok { + aenc.AppendObject(marshaler) + } + } + } + return nil + })) + return nil } diff --git a/protos/go/foyle/v1alpha1/eval.pb.go b/protos/go/foyle/v1alpha1/eval.pb.go index b0dccaa..b1da202 100644 --- a/protos/go/foyle/v1alpha1/eval.pb.go +++ b/protos/go/foyle/v1alpha1/eval.pb.go @@ -222,6 +222,70 @@ func (BlockLogStatus) EnumDescriptor() ([]byte, []int) { return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{3} } +type Assertion_Name int32 + +const ( + Assertion_UNKNOWN Assertion_Name = 0 + Assertion_CODE_AFTER_MARKDOWN Assertion_Name = 1 + Assertion_ONE_CODE_CELL Assertion_Name = 2 + Assertion_ENDS_WITH_CODE_CELL Assertion_Name = 3 + // NON_EMPTY_DOC asserts that the document sent to the LLM is non empty + Assertion_NON_EMPTY_DOC Assertion_Name = 4 + // AT_LEAST_ONE_BLOCK asserts that at at least one block was generated from the response before post processing + Assertion_AT_LEAST_ONE_BLOCK Assertion_Name = 5 + // AT_LEAST_ONE_POST_PROCESSED asserts that at at least one block is returned after post-processing + Assertion_AT_LEAST_ONE_BLOCK_POST_PROCESSED Assertion_Name = 6 +) + +// Enum value maps for Assertion_Name. +var ( + Assertion_Name_name = map[int32]string{ + 0: "UNKNOWN", + 1: "CODE_AFTER_MARKDOWN", + 2: "ONE_CODE_CELL", + 3: "ENDS_WITH_CODE_CELL", + 4: "NON_EMPTY_DOC", + 5: "AT_LEAST_ONE_BLOCK", + 6: "AT_LEAST_ONE_BLOCK_POST_PROCESSED", + } + Assertion_Name_value = map[string]int32{ + "UNKNOWN": 0, + "CODE_AFTER_MARKDOWN": 1, + "ONE_CODE_CELL": 2, + "ENDS_WITH_CODE_CELL": 3, + "NON_EMPTY_DOC": 4, + "AT_LEAST_ONE_BLOCK": 5, + "AT_LEAST_ONE_BLOCK_POST_PROCESSED": 6, + } +) + +func (x Assertion_Name) Enum() *Assertion_Name { + p := new(Assertion_Name) + *p = x + return p +} + +func (x Assertion_Name) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Assertion_Name) Descriptor() protoreflect.EnumDescriptor { + return file_foyle_v1alpha1_eval_proto_enumTypes[4].Descriptor() +} + +func (Assertion_Name) Type() protoreflect.EnumType { + return &file_foyle_v1alpha1_eval_proto_enumTypes[4] +} + +func (x Assertion_Name) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Assertion_Name.Descriptor instead. +func (Assertion_Name) EnumDescriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{1, 0} +} + // EvalResult represents an evaluation result type EvalResult struct { state protoimpl.MessageState @@ -368,10 +432,14 @@ type Assertion struct { unknownFields protoimpl.UnknownFields // Name of the assertion - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Result AssertResult `protobuf:"varint,2,opt,name=result,proto3,enum=AssertResult" json:"result,omitempty"` + Name Assertion_Name `protobuf:"varint,1,opt,name=name,proto3,enum=Assertion_Name" json:"name,omitempty"` + Result AssertResult `protobuf:"varint,2,opt,name=result,proto3,enum=AssertResult" json:"result,omitempty"` // Human readable detail of the assertion. If there was an error this should contain the error message. Detail string `protobuf:"bytes,3,opt,name=detail,proto3" json:"detail,omitempty"` + // id is a unique id of the assertion. This is needed for real time processing of the logs. Since our log + // processing guarantees at least once semantics, we may end up processing the same log entry about an assertion + // multiple times. By assigning a unique id to each assertion we can dedupe them. + Id string `protobuf:"bytes,4,opt,name=id,proto3" json:"id,omitempty"` } func (x *Assertion) Reset() { @@ -406,11 +474,11 @@ func (*Assertion) Descriptor() ([]byte, []int) { return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{1} } -func (x *Assertion) GetName() string { +func (x *Assertion) GetName() Assertion_Name { if x != nil { return x.Name } - return "" + return Assertion_UNKNOWN } func (x *Assertion) GetResult() AssertResult { @@ -427,6 +495,13 @@ func (x *Assertion) GetDetail() string { return "" } +func (x *Assertion) GetId() string { + if x != nil { + return x.Id + } + return "" +} + type EvalResultListRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -891,6 +966,234 @@ func (x *GetEvalResultResponse) GetReportHTML() string { return "" } +// ExperimentReport is a report for an experiment +type ExperimentReport struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // num_examples is the number of examples evaluated in the experiment + NumExamples int64 `protobuf:"varint,2,opt,name=num_examples,json=numExamples,proto3" json:"num_examples,omitempty"` + // num_errors is the number of examples that resulted in an error + NumErrors int64 `protobuf:"varint,3,opt,name=num_errors,json=numErrors,proto3" json:"num_errors,omitempty"` + // Map from string representation of CellsMatchResult to the number of counts + CellsMatchCounts map[string]int32 `protobuf:"bytes,4,rep,name=cells_match_counts,json=cellsMatchCounts,proto3" json:"cells_match_counts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + AssertionCounts []*AssertionCounts `protobuf:"bytes,5,rep,name=assertion_counts,json=assertionCounts,proto3" json:"assertion_counts,omitempty"` + GenerateLatencyStats []*PercentileStat `protobuf:"bytes,6,rep,name=generate_latency_stats,json=generateLatencyStats,proto3" json:"generate_latency_stats,omitempty"` +} + +func (x *ExperimentReport) Reset() { + *x = ExperimentReport{} + if protoimpl.UnsafeEnabled { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExperimentReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExperimentReport) ProtoMessage() {} + +func (x *ExperimentReport) ProtoReflect() protoreflect.Message { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExperimentReport.ProtoReflect.Descriptor instead. +func (*ExperimentReport) Descriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{10} +} + +func (x *ExperimentReport) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ExperimentReport) GetNumExamples() int64 { + if x != nil { + return x.NumExamples + } + return 0 +} + +func (x *ExperimentReport) GetNumErrors() int64 { + if x != nil { + return x.NumErrors + } + return 0 +} + +func (x *ExperimentReport) GetCellsMatchCounts() map[string]int32 { + if x != nil { + return x.CellsMatchCounts + } + return nil +} + +func (x *ExperimentReport) GetAssertionCounts() []*AssertionCounts { + if x != nil { + return x.AssertionCounts + } + return nil +} + +func (x *ExperimentReport) GetGenerateLatencyStats() []*PercentileStat { + if x != nil { + return x.GenerateLatencyStats + } + return nil +} + +type AssertionCounts struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name Assertion_Name `protobuf:"varint,1,opt,name=name,proto3,enum=Assertion_Name" json:"name,omitempty"` + Passed int32 `protobuf:"varint,2,opt,name=passed,proto3" json:"passed,omitempty"` + Failed int32 `protobuf:"varint,3,opt,name=failed,proto3" json:"failed,omitempty"` + Unknown int32 `protobuf:"varint,4,opt,name=unknown,proto3" json:"unknown,omitempty"` + Skipped int32 `protobuf:"varint,5,opt,name=skipped,proto3" json:"skipped,omitempty"` +} + +func (x *AssertionCounts) Reset() { + *x = AssertionCounts{} + if protoimpl.UnsafeEnabled { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AssertionCounts) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AssertionCounts) ProtoMessage() {} + +func (x *AssertionCounts) ProtoReflect() protoreflect.Message { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AssertionCounts.ProtoReflect.Descriptor instead. +func (*AssertionCounts) Descriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{11} +} + +func (x *AssertionCounts) GetName() Assertion_Name { + if x != nil { + return x.Name + } + return Assertion_UNKNOWN +} + +func (x *AssertionCounts) GetPassed() int32 { + if x != nil { + return x.Passed + } + return 0 +} + +func (x *AssertionCounts) GetFailed() int32 { + if x != nil { + return x.Failed + } + return 0 +} + +func (x *AssertionCounts) GetUnknown() int32 { + if x != nil { + return x.Unknown + } + return 0 +} + +func (x *AssertionCounts) GetSkipped() int32 { + if x != nil { + return x.Skipped + } + return 0 +} + +// PercentileStat represents a percentile value +type PercentileStat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The percentile a value 0 to 1 + Percentile float64 `protobuf:"fixed64,1,opt,name=percentile,proto3" json:"percentile,omitempty"` + // The value of the percentile + Value float64 `protobuf:"fixed64,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *PercentileStat) Reset() { + *x = PercentileStat{} + if protoimpl.UnsafeEnabled { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PercentileStat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PercentileStat) ProtoMessage() {} + +func (x *PercentileStat) ProtoReflect() protoreflect.Message { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PercentileStat.ProtoReflect.Descriptor instead. +func (*PercentileStat) Descriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{12} +} + +func (x *PercentileStat) GetPercentile() float64 { + if x != nil { + return x.Percentile + } + return 0 +} + +func (x *PercentileStat) GetValue() float64 { + if x != nil { + return x.Value + } + return 0 +} + var File_foyle_v1alpha1_eval_proto protoreflect.FileDescriptor var file_foyle_v1alpha1_eval_proto_rawDesc = []byte{ @@ -940,102 +1243,154 @@ var file_foyle_v1alpha1_eval_proto_rawDesc = []byte{ 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, - 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x22, 0x5e, 0x0a, 0x09, 0x41, - 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x06, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, - 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x33, 0x0a, 0x15, 0x45, + 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x22, 0xac, 0x02, 0x0a, 0x09, + 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, + 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, + 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0xaa, 0x01, + 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x41, 0x46, 0x54, 0x45, + 0x52, 0x5f, 0x4d, 0x41, 0x52, 0x4b, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, + 0x4f, 0x4e, 0x45, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x10, 0x02, 0x12, + 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x44, 0x53, 0x5f, 0x57, 0x49, 0x54, 0x48, 0x5f, 0x43, 0x4f, 0x44, + 0x45, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x4e, 0x4f, 0x4e, 0x5f, + 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x44, 0x4f, 0x43, 0x10, 0x04, 0x12, 0x16, 0x0a, 0x12, 0x41, + 0x54, 0x5f, 0x4c, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x4f, 0x4e, 0x45, 0x5f, 0x42, 0x4c, 0x4f, 0x43, + 0x4b, 0x10, 0x05, 0x12, 0x25, 0x0a, 0x21, 0x41, 0x54, 0x5f, 0x4c, 0x45, 0x41, 0x53, 0x54, 0x5f, + 0x4f, 0x4e, 0x45, 0x5f, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x50, 0x4f, 0x53, 0x54, 0x5f, 0x50, + 0x52, 0x4f, 0x43, 0x45, 0x53, 0x53, 0x45, 0x44, 0x10, 0x06, 0x22, 0x33, 0x0a, 0x15, 0x45, 0x76, + 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x22, + 0x3b, 0x0a, 0x16, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, + 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xa4, 0x02, 0x0a, + 0x0c, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x20, 0x0a, + 0x0b, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, + 0x15, 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x5f, 0x6d, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x64, 0x6f, 0x63, 0x4d, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, + 0x5f, 0x6d, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x6e, 0x73, 0x77, 0x65, + 0x72, 0x4d, 0x64, 0x12, 0x3d, 0x0a, 0x13, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, + 0x72, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, + 0x11, 0x63, 0x6f, 0x64, 0x65, 0x41, 0x66, 0x74, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, + 0x77, 0x6e, 0x12, 0x31, 0x0a, 0x0d, 0x6f, 0x6e, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x63, + 0x65, 0x6c, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, + 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0b, 0x6f, 0x6e, 0x65, 0x43, 0x6f, 0x64, + 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x12, 0x3c, 0x0a, 0x13, 0x65, 0x6e, 0x64, 0x73, 0x5f, 0x77, 0x69, + 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x52, 0x10, 0x65, 0x6e, 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6f, 0x64, 0x65, 0x43, + 0x65, 0x6c, 0x6c, 0x22, 0x33, 0x0a, 0x15, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, + 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x22, 0xbc, 0x01, 0x0a, 0x0b, 0x45, 0x76, 0x61, + 0x6c, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x0c, 0x66, 0x75, 0x6c, 0x6c, + 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x46, 0x75, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x0b, 0x66, 0x75, + 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x75, 0x6e, 0x6d, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, 0x72, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x0d, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x22, 0x3b, 0x0a, 0x16, 0x41, 0x73, 0x73, 0x65, 0x72, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x21, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x52, 0x04, + 0x72, 0x6f, 0x77, 0x73, 0x22, 0x26, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x37, 0x0a, 0x15, + 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x48, + 0x54, 0x4d, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x48, 0x54, 0x4d, 0x4c, 0x22, 0x88, 0x03, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x6e, 0x75, 0x6d, 0x5f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, + 0x12, 0x55, 0x0a, 0x12, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x5f, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x45, + 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, + 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, + 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x3b, 0x0a, 0x10, 0x61, 0x73, 0x73, 0x65, 0x72, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x10, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x73, 0x52, 0x0f, 0x61, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x73, 0x12, 0x45, 0x0a, 0x16, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x69, 0x6c, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x52, 0x14, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x4c, + 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73, 0x1a, 0x43, 0x0a, 0x15, 0x43, + 0x65, 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x9a, 0x01, 0x0a, 0x0f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4e, + 0x61, 0x6d, 0x65, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x73, + 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x70, 0x61, 0x73, 0x73, 0x65, + 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x75, 0x6e, 0x6b, + 0x6e, 0x6f, 0x77, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x75, 0x6e, 0x6b, 0x6e, + 0x6f, 0x77, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x22, 0x46, 0x0a, + 0x0e, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x12, + 0x1e, 0x0a, 0x0a, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x0a, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2a, 0x47, 0x0a, 0x10, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, + 0x45, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x2a, 0x4d, + 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x18, + 0x0a, 0x14, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x41, 0x53, 0x53, + 0x45, 0x44, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, + 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x4b, 0x49, 0x50, 0x50, 0x45, 0x44, 0x10, 0x03, 0x2a, 0x49, 0x0a, + 0x10, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x43, 0x65, 0x6c, + 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, + 0x09, 0x0a, 0x05, 0x4d, 0x41, 0x54, 0x43, 0x48, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, 0x49, + 0x53, 0x4d, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x6a, 0x0a, 0x0e, 0x42, 0x6c, 0x6f, 0x63, + 0x6b, 0x4c, 0x6f, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4c, + 0x4f, 0x43, 0x4b, 0x5f, 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4c, 0x4f, 0x43, + 0x4b, 0x5f, 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x43, + 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x5f, + 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x4f, + 0x55, 0x54, 0x10, 0x02, 0x32, 0xcf, 0x01, 0x0a, 0x0b, 0x45, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, - 0x22, 0x3b, 0x0a, 0x16, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x69, 0x74, - 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x45, 0x76, 0x61, 0x6c, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xa4, 0x02, - 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x20, - 0x0a, 0x0b, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, - 0x12, 0x15, 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x5f, 0x6d, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x64, 0x6f, 0x63, 0x4d, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x61, 0x6e, 0x73, 0x77, 0x65, - 0x72, 0x5f, 0x6d, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x61, 0x6e, 0x73, 0x77, - 0x65, 0x72, 0x4d, 0x64, 0x12, 0x3d, 0x0a, 0x13, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x61, 0x66, 0x74, - 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x52, 0x11, 0x63, 0x6f, 0x64, 0x65, 0x41, 0x66, 0x74, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x64, - 0x6f, 0x77, 0x6e, 0x12, 0x31, 0x0a, 0x0d, 0x6f, 0x6e, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x5f, - 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, - 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0b, 0x6f, 0x6e, 0x65, 0x43, 0x6f, - 0x64, 0x65, 0x43, 0x65, 0x6c, 0x6c, 0x12, 0x3c, 0x0a, 0x13, 0x65, 0x6e, 0x64, 0x73, 0x5f, 0x77, - 0x69, 0x74, 0x68, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x10, 0x65, 0x6e, 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6f, 0x64, 0x65, - 0x43, 0x65, 0x6c, 0x6c, 0x22, 0x33, 0x0a, 0x15, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x22, 0xbc, 0x01, 0x0a, 0x0b, 0x45, 0x76, - 0x61, 0x6c, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x0c, 0x66, 0x75, 0x6c, - 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x46, 0x75, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x0b, 0x66, - 0x75, 0x6c, 0x6c, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x0e, 0x65, 0x78, - 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x75, 0x6e, 0x6d, 0x65, 0x2e, 0x70, 0x61, 0x72, 0x73, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x52, 0x0d, 0x65, 0x78, 0x70, 0x65, 0x63, - 0x74, 0x65, 0x64, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x22, 0x3b, 0x0a, 0x16, 0x41, 0x73, 0x73, 0x65, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x43, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x12, 0x16, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x52, - 0x04, 0x72, 0x6f, 0x77, 0x73, 0x22, 0x26, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x37, 0x0a, - 0x15, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x48, 0x54, 0x4d, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x48, 0x54, 0x4d, 0x4c, 0x2a, 0x47, 0x0a, 0x10, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x55, 0x4c, - 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, - 0x4e, 0x45, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x2a, - 0x4d, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x18, 0x0a, 0x14, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x73, 0x73, 0x65, 0x72, - 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x41, 0x53, - 0x53, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, - 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x4b, 0x49, 0x50, 0x50, 0x45, 0x44, 0x10, 0x03, 0x2a, 0x49, - 0x0a, 0x10, 0x43, 0x65, 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x43, 0x65, - 0x6c, 0x6c, 0x73, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x10, 0x00, - 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x41, 0x54, 0x43, 0x48, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x4d, - 0x49, 0x53, 0x4d, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x6a, 0x0a, 0x0e, 0x42, 0x6c, 0x6f, - 0x63, 0x6b, 0x4c, 0x6f, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x0a, 0x18, 0x42, - 0x4c, 0x4f, 0x43, 0x4b, 0x5f, 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4c, 0x4f, - 0x43, 0x4b, 0x5f, 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, - 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4c, 0x4f, 0x43, 0x4b, - 0x5f, 0x4c, 0x4f, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, 0x49, 0x4d, 0x45, - 0x4f, 0x55, 0x54, 0x10, 0x02, 0x32, 0xcf, 0x01, 0x0a, 0x0b, 0x45, 0x76, 0x61, 0x6c, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, - 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x43, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x12, 0x16, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x41, 0x73, 0x73, - 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x15, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3e, 0x42, 0x09, 0x45, 0x76, 0x61, 0x6c, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x15, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x47, + 0x65, 0x74, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3e, 0x42, 0x09, 0x45, 0x76, 0x61, 0x6c, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1050,56 +1405,66 @@ func file_foyle_v1alpha1_eval_proto_rawDescGZIP() []byte { return file_foyle_v1alpha1_eval_proto_rawDescData } -var file_foyle_v1alpha1_eval_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_foyle_v1alpha1_eval_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_foyle_v1alpha1_eval_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_foyle_v1alpha1_eval_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_foyle_v1alpha1_eval_proto_goTypes = []interface{}{ (EvalResultStatus)(0), // 0: EvalResultStatus (AssertResult)(0), // 1: AssertResult (CellsMatchResult)(0), // 2: CellsMatchResult (BlockLogStatus)(0), // 3: BlockLogStatus - (*EvalResult)(nil), // 4: EvalResult - (*Assertion)(nil), // 5: Assertion - (*EvalResultListRequest)(nil), // 6: EvalResultListRequest - (*EvalResultListResponse)(nil), // 7: EvalResultListResponse - (*AssertionRow)(nil), // 8: AssertionRow - (*AssertionTableRequest)(nil), // 9: AssertionTableRequest - (*EvalExample)(nil), // 10: EvalExample - (*AssertionTableResponse)(nil), // 11: AssertionTableResponse - (*GetEvalResultRequest)(nil), // 12: GetEvalResultRequest - (*GetEvalResultResponse)(nil), // 13: GetEvalResultResponse - (*v1.Cell)(nil), // 14: runme.parser.v1.Cell - (*RAGResult)(nil), // 15: RAGResult - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*FullContext)(nil), // 17: FullContext + (Assertion_Name)(0), // 4: Assertion.Name + (*EvalResult)(nil), // 5: EvalResult + (*Assertion)(nil), // 6: Assertion + (*EvalResultListRequest)(nil), // 7: EvalResultListRequest + (*EvalResultListResponse)(nil), // 8: EvalResultListResponse + (*AssertionRow)(nil), // 9: AssertionRow + (*AssertionTableRequest)(nil), // 10: AssertionTableRequest + (*EvalExample)(nil), // 11: EvalExample + (*AssertionTableResponse)(nil), // 12: AssertionTableResponse + (*GetEvalResultRequest)(nil), // 13: GetEvalResultRequest + (*GetEvalResultResponse)(nil), // 14: GetEvalResultResponse + (*ExperimentReport)(nil), // 15: ExperimentReport + (*AssertionCounts)(nil), // 16: AssertionCounts + (*PercentileStat)(nil), // 17: PercentileStat + nil, // 18: ExperimentReport.CellsMatchCountsEntry + (*v1.Cell)(nil), // 19: runme.parser.v1.Cell + (*RAGResult)(nil), // 20: RAGResult + (*timestamppb.Timestamp)(nil), // 21: google.protobuf.Timestamp + (*FullContext)(nil), // 22: FullContext } var file_foyle_v1alpha1_eval_proto_depIdxs = []int32{ - 10, // 0: EvalResult.example:type_name -> EvalExample - 14, // 1: EvalResult.actual_cells:type_name -> runme.parser.v1.Cell + 11, // 0: EvalResult.example:type_name -> EvalExample + 19, // 1: EvalResult.actual_cells:type_name -> runme.parser.v1.Cell 0, // 2: EvalResult.status:type_name -> EvalResultStatus - 15, // 3: EvalResult.best_rag_result:type_name -> RAGResult - 5, // 4: EvalResult.assertions:type_name -> Assertion + 20, // 3: EvalResult.best_rag_result:type_name -> RAGResult + 6, // 4: EvalResult.assertions:type_name -> Assertion 2, // 5: EvalResult.cells_match_result:type_name -> CellsMatchResult 3, // 6: EvalResult.block_log_status:type_name -> BlockLogStatus - 1, // 7: Assertion.result:type_name -> AssertResult - 4, // 8: EvalResultListResponse.items:type_name -> EvalResult - 1, // 9: AssertionRow.code_after_markdown:type_name -> AssertResult - 1, // 10: AssertionRow.one_code_cell:type_name -> AssertResult - 1, // 11: AssertionRow.ends_with_code_cell:type_name -> AssertResult - 16, // 12: EvalExample.time:type_name -> google.protobuf.Timestamp - 17, // 13: EvalExample.full_context:type_name -> FullContext - 14, // 14: EvalExample.expected_cells:type_name -> runme.parser.v1.Cell - 8, // 15: AssertionTableResponse.rows:type_name -> AssertionRow - 6, // 16: EvalService.List:input_type -> EvalResultListRequest - 9, // 17: EvalService.AssertionTable:input_type -> AssertionTableRequest - 12, // 18: EvalService.GetEvalResult:input_type -> GetEvalResultRequest - 7, // 19: EvalService.List:output_type -> EvalResultListResponse - 11, // 20: EvalService.AssertionTable:output_type -> AssertionTableResponse - 13, // 21: EvalService.GetEvalResult:output_type -> GetEvalResultResponse - 19, // [19:22] is the sub-list for method output_type - 16, // [16:19] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 4, // 7: Assertion.name:type_name -> Assertion.Name + 1, // 8: Assertion.result:type_name -> AssertResult + 5, // 9: EvalResultListResponse.items:type_name -> EvalResult + 1, // 10: AssertionRow.code_after_markdown:type_name -> AssertResult + 1, // 11: AssertionRow.one_code_cell:type_name -> AssertResult + 1, // 12: AssertionRow.ends_with_code_cell:type_name -> AssertResult + 21, // 13: EvalExample.time:type_name -> google.protobuf.Timestamp + 22, // 14: EvalExample.full_context:type_name -> FullContext + 19, // 15: EvalExample.expected_cells:type_name -> runme.parser.v1.Cell + 9, // 16: AssertionTableResponse.rows:type_name -> AssertionRow + 18, // 17: ExperimentReport.cells_match_counts:type_name -> ExperimentReport.CellsMatchCountsEntry + 16, // 18: ExperimentReport.assertion_counts:type_name -> AssertionCounts + 17, // 19: ExperimentReport.generate_latency_stats:type_name -> PercentileStat + 4, // 20: AssertionCounts.name:type_name -> Assertion.Name + 7, // 21: EvalService.List:input_type -> EvalResultListRequest + 10, // 22: EvalService.AssertionTable:input_type -> AssertionTableRequest + 13, // 23: EvalService.GetEvalResult:input_type -> GetEvalResultRequest + 8, // 24: EvalService.List:output_type -> EvalResultListResponse + 12, // 25: EvalService.AssertionTable:output_type -> AssertionTableResponse + 14, // 26: EvalService.GetEvalResult:output_type -> GetEvalResultResponse + 24, // [24:27] is the sub-list for method output_type + 21, // [21:24] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_foyle_v1alpha1_eval_proto_init() } @@ -1231,14 +1596,50 @@ func file_foyle_v1alpha1_eval_proto_init() { return nil } } + file_foyle_v1alpha1_eval_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExperimentReport); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_foyle_v1alpha1_eval_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AssertionCounts); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_foyle_v1alpha1_eval_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PercentileStat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_foyle_v1alpha1_eval_proto_rawDesc, - NumEnums: 4, - NumMessages: 10, + NumEnums: 5, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/protos/go/foyle/v1alpha1/eval.zap.go b/protos/go/foyle/v1alpha1/eval.zap.go index de997cf..ee19db7 100644 --- a/protos/go/foyle/v1alpha1/eval.zap.go +++ b/protos/go/foyle/v1alpha1/eval.zap.go @@ -104,7 +104,7 @@ func (m *Assertion) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) } keyName = "name" // field name = 1 - enc.AddString(keyName, m.Name) + enc.AddString(keyName, m.Name.String()) keyName = "result" // field result = 2 enc.AddString(keyName, m.Result.String()) @@ -112,6 +112,9 @@ func (m *Assertion) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) keyName = "detail" // field detail = 3 enc.AddString(keyName, m.Detail) + keyName = "id" // field id = 4 + enc.AddString(keyName, m.Id) + return nil } @@ -293,3 +296,104 @@ func (m *GetEvalResultResponse) MarshalLogObject(enc go_uber_org_zap_zapcore.Obj return nil } + +func (m *ExperimentReport) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) error { + var keyName string + _ = keyName + + if m == nil { + return nil + } + + keyName = "name" // field name = 1 + enc.AddString(keyName, m.Name) + + keyName = "num_examples" // field num_examples = 2 + enc.AddInt64(keyName, m.NumExamples) + + keyName = "num_errors" // field num_errors = 3 + enc.AddInt64(keyName, m.NumErrors) + + keyName = "cells_match_counts" // field cells_match_counts = 4 + enc.AddObject(keyName, go_uber_org_zap_zapcore.ObjectMarshalerFunc(func(enc go_uber_org_zap_zapcore.ObjectEncoder) error { + for mk, mv := range m.CellsMatchCounts { + key := mk + _ = key + enc.AddInt32(key, mv) + } + return nil + })) + + keyName = "assertion_counts" // field assertion_counts = 5 + enc.AddArray(keyName, go_uber_org_zap_zapcore.ArrayMarshalerFunc(func(aenc go_uber_org_zap_zapcore.ArrayEncoder) error { + for _, rv := range m.AssertionCounts { + _ = rv + if rv != nil { + var vv interface{} = rv + if marshaler, ok := vv.(go_uber_org_zap_zapcore.ObjectMarshaler); ok { + aenc.AppendObject(marshaler) + } + } + } + return nil + })) + + keyName = "generate_latency_stats" // field generate_latency_stats = 6 + enc.AddArray(keyName, go_uber_org_zap_zapcore.ArrayMarshalerFunc(func(aenc go_uber_org_zap_zapcore.ArrayEncoder) error { + for _, rv := range m.GenerateLatencyStats { + _ = rv + if rv != nil { + var vv interface{} = rv + if marshaler, ok := vv.(go_uber_org_zap_zapcore.ObjectMarshaler); ok { + aenc.AppendObject(marshaler) + } + } + } + return nil + })) + + return nil +} + +func (m *AssertionCounts) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) error { + var keyName string + _ = keyName + + if m == nil { + return nil + } + + keyName = "name" // field name = 1 + enc.AddString(keyName, m.Name.String()) + + keyName = "passed" // field passed = 2 + enc.AddInt32(keyName, m.Passed) + + keyName = "failed" // field failed = 3 + enc.AddInt32(keyName, m.Failed) + + keyName = "unknown" // field unknown = 4 + enc.AddInt32(keyName, m.Unknown) + + keyName = "skipped" // field skipped = 5 + enc.AddInt32(keyName, m.Skipped) + + return nil +} + +func (m *PercentileStat) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) error { + var keyName string + _ = keyName + + if m == nil { + return nil + } + + keyName = "percentile" // field percentile = 1 + enc.AddFloat64(keyName, m.Percentile) + + keyName = "value" // field value = 2 + enc.AddFloat64(keyName, m.Value) + + return nil +}