diff --git a/app/pkg/agent/prompt.go b/app/pkg/agent/prompt.go index f85330c8..303f36bb 100644 --- a/app/pkg/agent/prompt.go +++ b/app/pkg/agent/prompt.go @@ -9,12 +9,21 @@ const ( systemPrompt = `You are a helpful AI assistant for software developers. You are helping software engineers write markdown documents to deploy and operate software. Your job is to help users reason about problems and tasks and come up with the appropriate commands to accomplish them. You should never try to execute commands. You should always tell the user -to execute the commands themselves. To help the user place the commands inside a code block with the language set to +to execute the commands themselves. To help the user, place the commands inside a code block with the language set to bash. Users can then execute the commands inside VSCode notebooks. The output will then be appended to the document. You can then use that output to reason about the next steps. You are only helping users with tasks related to building, deploying, and operating software. You should interpret any questions or commands in that context. + +Keep these rules in mind when generating responses + +* If the document ends in a markdown cell, the response should start with a code cell. +* The response should almost never have more than one code cell +* If you want to suggest multiple commands put them all in a code cell +* Once you suggest a code cell you should usually stop and wait for the user to execute the command +* Only respond with a markdown cell if the previous cell contains the output of a command and you are explaining + the output ` ) diff --git a/app/pkg/eval/assertions.go b/app/pkg/eval/assertions.go index 972fca81..e86a9ee4 100644 --- a/app/pkg/eval/assertions.go +++ b/app/pkg/eval/assertions.go @@ -2,12 +2,15 @@ package eval import ( "context" + "fmt" "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" ) const ( CodeAfterMarkdownName = "AssertCodeAfterMarkdown" + OneCodeCellName = "AssertOneCodeCell" + EndsWithCodeCellName = "AssertEndsWithCodeCell" ) // AssertCodeAfterMarkdown is an assertion that checks that if the prompt ends in a markdown cell then the response @@ -50,3 +53,86 @@ func (a *AssertCodeAfterMarkdown) Assert(ctx context.Context, doc *v1alpha1.Doc, func (a *AssertCodeAfterMarkdown) Name() string { return CodeAfterMarkdownName } + +// AssertOneCodeCell is an assertion that checks the response has one code cell. +// We don't want to suggest multiple code cells because that can be confusing. If one command depends on the +// output of another we should generate one at a time. +type AssertOneCodeCell struct { +} + +func (a *AssertOneCodeCell) Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) { + assertion := &v1alpha1.Assertion{ + Name: a.Name(), + } + + if len(doc.Blocks) == 0 { + assertion.Result = v1alpha1.AssertResult_FAILED + return assertion, nil + } + + numCode := 0 + for _, b := range answer { + if b.GetKind() == v1alpha1.BlockKind_CODE { + numCode++ + } + } + + if numCode != 1 { + assertion.Result = v1alpha1.AssertResult_FAILED + assertion.Detail = fmt.Sprintf("Answer doesn't contain exactly one code cell; it has %d code cells ", numCode) + return assertion, nil + } + + assertion.Result = v1alpha1.AssertResult_PASSED + return assertion, nil +} + +func (a *AssertOneCodeCell) Name() string { + return OneCodeCellName +} + +// AssertEndsWithCodeCell is an assertion that checks that if the response has a code cell then it ends with the +// code cell. If we need the user to run a command we should just suggest the command and not additional output +// after that. +type AssertEndsWithCodeCell struct { +} + +func (a *AssertEndsWithCodeCell) Assert(ctx context.Context, doc *v1alpha1.Doc, examples []*v1alpha1.Example, answer []*v1alpha1.Block) (*v1alpha1.Assertion, error) { + assertion := &v1alpha1.Assertion{ + Name: a.Name(), + } + + if len(doc.Blocks) == 0 { + assertion.Result = v1alpha1.AssertResult_SKIPPED + assertion.Detail = "Doc is empty" + return assertion, nil + } + + hasCodeCell := false + for _, b := range answer { + if b.GetKind() == v1alpha1.BlockKind_CODE { + hasCodeCell = true + break + } + } + + if !hasCodeCell { + assertion.Result = v1alpha1.AssertResult_SKIPPED + assertion.Detail = "Answer doesn't contain a code cell" + return assertion, nil + } + + last := answer[len(answer)-1] + + if last.GetKind() != v1alpha1.BlockKind_CODE { + assertion.Result = v1alpha1.AssertResult_FAILED + return assertion, nil + } + + assertion.Result = v1alpha1.AssertResult_PASSED + return assertion, nil +} + +func (a *AssertEndsWithCodeCell) Name() string { + return EndsWithCodeCellName +} diff --git a/app/pkg/eval/assertions_test.go b/app/pkg/eval/assertions_test.go index daef7484..dde3b278 100644 --- a/app/pkg/eval/assertions_test.go +++ b/app/pkg/eval/assertions_test.go @@ -82,3 +82,142 @@ func TestAssertCodeAfterMarkdown(t *testing.T) { }) } } + +func TestAssertOneCodeCell(t *testing.T) { + cases := []testCase{ + { + name: "Empty", + doc: &v1alpha1.Doc{}, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{}, + expected: &v1alpha1.Assertion{ + Name: "AssertCodeAfterMarkdown", + Result: v1alpha1.AssertResult_SKIPPED, + }, + }, + { + name: "Passed", + doc: &v1alpha1.Doc{ + Blocks: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + }, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_CODE, + }, + }, + expected: &v1alpha1.Assertion{ + Name: CodeAfterMarkdownName, + Result: v1alpha1.AssertResult_PASSED, + }, + }, + { + name: "Failed", + doc: &v1alpha1.Doc{ + Blocks: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + }, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + expected: &v1alpha1.Assertion{ + Name: OneCodeCellName, + Result: v1alpha1.AssertResult_FAILED, + }, + }, + } + + for _, c := range cases { + a := &AssertCodeAfterMarkdown{} + t.Run(c.name, func(t *testing.T) { + got, err := a.Assert(context.Background(), c.doc, c.examples, c.answer) + if err != nil { + t.Fatalf("Error: %v", err) + } + if got.Result != c.expected.Result { + t.Fatalf("Expected %v but got %v", c.expected.Result, got.Result) + } + }) + } +} + +func TestAssertEndsWithCodeCell(t *testing.T) { + cases := []testCase{ + { + name: "Empty", + doc: &v1alpha1.Doc{}, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{}, + expected: &v1alpha1.Assertion{ + Name: EndsWithCodeCellName, + Result: v1alpha1.AssertResult_SKIPPED, + }, + }, + { + name: "Passed", + doc: &v1alpha1.Doc{ + Blocks: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + }, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_CODE, + }, + }, + expected: &v1alpha1.Assertion{ + Name: EndsWithCodeCellName, + Result: v1alpha1.AssertResult_PASSED, + }, + }, + { + name: "Failed", + doc: &v1alpha1.Doc{ + Blocks: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + }, + examples: []*v1alpha1.Example{}, + answer: []*v1alpha1.Block{ + { + Kind: v1alpha1.BlockKind_CODE, + }, + { + Kind: v1alpha1.BlockKind_MARKUP, + }, + }, + expected: &v1alpha1.Assertion{ + Name: EndsWithCodeCellName, + Result: v1alpha1.AssertResult_FAILED, + }, + }, + } + + for _, c := range cases { + a := &AssertEndsWithCodeCell{} + t.Run(c.name, func(t *testing.T) { + got, err := a.Assert(context.Background(), c.doc, c.examples, c.answer) + if err != nil { + t.Fatalf("Error: %v", err) + } + if got.Result != c.expected.Result { + t.Fatalf("Expected %v but got %v", c.expected.Result, got.Result) + } + }) + } +} diff --git a/app/pkg/eval/assertor.go b/app/pkg/eval/assertor.go index 0a6b0e8b..c1f07ad8 100644 --- a/app/pkg/eval/assertor.go +++ b/app/pkg/eval/assertor.go @@ -32,6 +32,8 @@ func NewAssertRunner(config config.Config) (*AssertRunner, error) { // Load the assertions runner.assertions = make([]Assertion, 0, 10) runner.assertions = append(runner.assertions, &AssertCodeAfterMarkdown{}) + runner.assertions = append(runner.assertions, &AssertOneCodeCell{}) + runner.assertions = append(runner.assertions, &AssertEndsWithCodeCell{}) return runner, nil } diff --git a/app/pkg/eval/service.go b/app/pkg/eval/service.go index a5b9ec50..22b097b8 100644 --- a/app/pkg/eval/service.go +++ b/app/pkg/eval/service.go @@ -150,6 +150,10 @@ func toAssertionRow(result *v1alpha1.EvalResult) (*v1alpha1.AssertionRow, error) switch a.Name { case CodeAfterMarkdownName: row.CodeAfterMarkdown = a.GetResult() + case OneCodeCellName: + row.OneCodeCell = a.GetResult() + case EndsWithCodeCellName: + row.EndsWithCodeCell = a.GetResult() default: log.Info("Unknown assertion", "name", a.Name) } diff --git a/developer_guides/eval.md b/developer_guides/eval.md index 4da372bd..05e990a8 100644 --- a/developer_guides/eval.md +++ b/developer_guides/eval.md @@ -89,4 +89,11 @@ foyle apply ~/git_foyle/experiments/norag.yaml ```sh {"id":"01HZ38QWPZ565XH11CCKYCF1M7"} foyle apply ~/git_foyle/experiments/rag.yaml -``` \ No newline at end of file +``` + +### Adding Level 1 Evals + +1. Define the Assertion in [eval/assertions.go](../app/pkg/eval/assertions.go) +2. Update [Assertor in assertor.go](../app/pkg/eval/assertor.go) to include the new assertion +3. Update [AssertRow proto](../protos/eval/eval.proto) to include the new assertion +4. Update [toAssertionRow](../app/pkg/eval/service.go) to include the new assertion in `AssertRow` \ No newline at end of file diff --git a/developer_guides/runme.md b/developer_guides/runme.md index 0e0e651a..d5acfb08 100644 --- a/developer_guides/runme.md +++ b/developer_guides/runme.md @@ -43,7 +43,7 @@ Now we can install the extension using the vscode binary * It also seemed like when I didn't bump the version I might have actually been using an old version of the extension ```bash {"id":"01HYZVG8KZKYSTFS4R1RJZDS7P"} -/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/git_vscode-runme/runme-3.5.9.vsix +/Applications/Visual\ Studio\ Code.app/Contents/Resources/app/bin/code --force --install-extension ~/git_vscode-runme/runme-extension.vsix ``` ```sh {"id":"01HY264KZTS4J9NHJASJT1GYJ7"} @@ -60,10 +60,6 @@ Lets try installing and reinstalling it ``` -```bash {"id":"01HY75KYKE3SFAM5EXMDAVJDTQ"} -echo "hello world"sdfds -``` - ## Debugging the Runme Extension in vscode * It seems like you may need to run `yarn build` for changes to get picked up; running `F5` doesn't always seem to work diff --git a/protos/foyle/v1alpha1/eval.proto b/protos/foyle/v1alpha1/eval.proto index 2548e001..c1208225 100644 --- a/protos/foyle/v1alpha1/eval.proto +++ b/protos/foyle/v1alpha1/eval.proto @@ -81,6 +81,8 @@ message AssertionRow { // TODO(jeremy): How can we avoid having to add each assertion here AssertResult code_after_markdown = 5; + AssertResult one_code_cell = 6; + AssertResult ends_with_code_cell = 7; } message AssertionTableRequest { diff --git a/protos/go/foyle/v1alpha1/eval.pb.go b/protos/go/foyle/v1alpha1/eval.pb.go index 87dbd014..6c9c279d 100644 --- a/protos/go/foyle/v1alpha1/eval.pb.go +++ b/protos/go/foyle/v1alpha1/eval.pb.go @@ -425,6 +425,8 @@ type AssertionRow struct { AnswerMd string `protobuf:"bytes,4,opt,name=answer_md,json=answerMd,proto3" json:"answer_md,omitempty"` // TODO(jeremy): How can we avoid having to add each assertion here CodeAfterMarkdown AssertResult `protobuf:"varint,5,opt,name=code_after_markdown,json=codeAfterMarkdown,proto3,enum=AssertResult" json:"code_after_markdown,omitempty"` + OneCodeCell AssertResult `protobuf:"varint,6,opt,name=one_code_cell,json=oneCodeCell,proto3,enum=AssertResult" json:"one_code_cell,omitempty"` + EndsWithCodeCell AssertResult `protobuf:"varint,7,opt,name=ends_with_code_cell,json=endsWithCodeCell,proto3,enum=AssertResult" json:"ends_with_code_cell,omitempty"` } func (x *AssertionRow) Reset() { @@ -494,6 +496,20 @@ func (x *AssertionRow) GetCodeAfterMarkdown() AssertResult { return AssertResult_UNKNOWN_AssertResult } +func (x *AssertionRow) GetOneCodeCell() AssertResult { + if x != nil { + return x.OneCodeCell + } + return AssertResult_UNKNOWN_AssertResult +} + +func (x *AssertionRow) GetEndsWithCodeCell() AssertResult { + if x != nil { + return x.EndsWithCodeCell + } + return AssertResult_UNKNOWN_AssertResult +} + type AssertionTableRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -636,7 +652,7 @@ var file_foyle_v1alpha1_eval_proto_rawDesc = []byte{ 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xb3, 0x01, 0x0a, 0x0c, 0x41, 0x73, + 0x6c, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xa4, 0x02, 0x0a, 0x0c, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -647,38 +663,45 @@ var file_foyle_v1alpha1_eval_proto_rawDesc = []byte{ 0x12, 0x3d, 0x0a, 0x13, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x11, 0x63, 0x6f, - 0x64, 0x65, 0x41, 0x66, 0x74, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x22, - 0x33, 0x0a, 0x15, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, - 0x62, 0x61, 0x73, 0x65, 0x22, 0x3b, 0x0a, 0x16, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, - 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x41, - 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, - 0x73, 0x2a, 0x47, 0x0a, 0x10, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, - 0x5f, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, - 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x2a, 0x4d, 0x0a, 0x0c, 0x41, 0x73, - 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x18, 0x0a, 0x14, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x41, 0x53, 0x53, 0x45, 0x44, 0x10, 0x01, - 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, - 0x53, 0x4b, 0x49, 0x50, 0x50, 0x45, 0x44, 0x10, 0x03, 0x32, 0x8d, 0x01, 0x0a, 0x0b, 0x45, 0x76, - 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x16, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x45, 0x76, 0x61, 0x6c, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x16, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, - 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3e, 0x42, 0x09, 0x45, 0x76, 0x61, - 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, - 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x64, 0x65, 0x41, 0x66, 0x74, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x64, 0x6f, 0x77, 0x6e, 0x12, + 0x31, 0x0a, 0x0d, 0x6f, 0x6e, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x6c, 0x6c, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0b, 0x6f, 0x6e, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x43, 0x65, + 0x6c, 0x6c, 0x12, 0x3c, 0x0a, 0x13, 0x65, 0x6e, 0x64, 0x73, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x5f, + 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x63, 0x65, 0x6c, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x0d, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x10, + 0x65, 0x6e, 0x64, 0x73, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6f, 0x64, 0x65, 0x43, 0x65, 0x6c, 0x6c, + 0x22, 0x33, 0x0a, 0x15, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x61, 0x74, + 0x61, 0x62, 0x61, 0x73, 0x65, 0x22, 0x3b, 0x0a, 0x16, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x21, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, + 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, + 0x77, 0x73, 0x2a, 0x47, 0x0a, 0x10, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x5f, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x01, + 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x2a, 0x4d, 0x0a, 0x0c, 0x41, + 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x18, 0x0a, 0x14, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x50, 0x41, 0x53, 0x53, 0x45, 0x44, 0x10, + 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0b, 0x0a, + 0x07, 0x53, 0x4b, 0x49, 0x50, 0x50, 0x45, 0x44, 0x10, 0x03, 0x32, 0x8d, 0x01, 0x0a, 0x0b, 0x45, + 0x76, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x04, 0x4c, 0x69, + 0x73, 0x74, 0x12, 0x16, 0x2e, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x45, 0x76, 0x61, + 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x0e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, + 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x16, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x17, 0x2e, 0x41, 0x73, 0x73, 0x65, 0x72, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x3e, 0x42, 0x09, 0x45, 0x76, + 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x66, 0x6f, 0x79, 0x6c, + 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -718,16 +741,18 @@ var file_foyle_v1alpha1_eval_proto_depIdxs = []int32{ 1, // 5: Assertion.result:type_name -> AssertResult 2, // 6: EvalResultListResponse.items:type_name -> EvalResult 1, // 7: AssertionRow.code_after_markdown:type_name -> AssertResult - 6, // 8: AssertionTableResponse.rows:type_name -> AssertionRow - 4, // 9: EvalService.List:input_type -> EvalResultListRequest - 7, // 10: EvalService.AssertionTable:input_type -> AssertionTableRequest - 5, // 11: EvalService.List:output_type -> EvalResultListResponse - 8, // 12: EvalService.AssertionTable:output_type -> AssertionTableResponse - 11, // [11:13] is the sub-list for method output_type - 9, // [9:11] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 1, // 8: AssertionRow.one_code_cell:type_name -> AssertResult + 1, // 9: AssertionRow.ends_with_code_cell:type_name -> AssertResult + 6, // 10: AssertionTableResponse.rows:type_name -> AssertionRow + 4, // 11: EvalService.List:input_type -> EvalResultListRequest + 7, // 12: EvalService.AssertionTable:input_type -> AssertionTableRequest + 5, // 13: EvalService.List:output_type -> EvalResultListResponse + 8, // 14: EvalService.AssertionTable:output_type -> AssertionTableResponse + 13, // [13:15] is the sub-list for method output_type + 11, // [11:13] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_foyle_v1alpha1_eval_proto_init() } diff --git a/protos/go/foyle/v1alpha1/eval.zap.go b/protos/go/foyle/v1alpha1/eval.zap.go index 2cc8840a..5ce934e8 100644 --- a/protos/go/foyle/v1alpha1/eval.zap.go +++ b/protos/go/foyle/v1alpha1/eval.zap.go @@ -171,6 +171,12 @@ func (m *AssertionRow) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncode keyName = "code_after_markdown" // field code_after_markdown = 5 enc.AddString(keyName, m.CodeAfterMarkdown.String()) + keyName = "one_code_cell" // field one_code_cell = 6 + enc.AddString(keyName, m.OneCodeCell.String()) + + keyName = "ends_with_code_cell" // field ends_with_code_cell = 7 + enc.AddString(keyName, m.EndsWithCodeCell.String()) + return nil }