From 1be711075f98c9f7b1a36c329592a7db5a7b906f Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Fri, 3 May 2024 16:57:56 -0700 Subject: [PATCH] Eval (#87) See tn003_learning_eval.md for more description of how we are doing eval. This is a first pass at evaluation. * Implement a distance metric based on edit distance * Implement the infrastructure to compute it. Add an apply command and use it to run different experiments * Requires starting to move some of the Agent config into the API package because we want to reuse it in the experiment type. Start an initial evaluation dataset. ### API Updates - Added `EvalResult` structure to represent the evaluation outcome, including the actual commands generated, the expected commands, and the evaluation distance. - Introduced `EvalResultStatus` to indicate the status of an evaluation, such as `DONE` or `ERROR`. ### Agent Updates - Updated the `Agent` service to support evaluation mode, allowing it to operate without impacting the learning process. ### Executor Updates - Enhanced the `Executor` service to handle execution in evaluation mode, ensuring that execution traces are marked accordingly. ### Evaluator Implementation - Implemented the `Evaluator` component responsible for orchestrating the evaluation process, loading evaluation examples, generating predictions with the Agent, calculating distances, and updating results. ### Google Sheets Integration - Added functionality to export evaluation results to Google Sheets, enabling easy review and analysis of Foyle's performance. ### CLI Tool Enhancements - Extended the CLI tool with commands for running evaluations ### Miscellaneous - Added necessary protobuf definitions for new data structures related to evaluations. - Updated server setup to handle evaluation logic and integrate with the learning mechanism. - Provided sample evaluation datasets for initial testing and validation of the evaluation process. --- app/api/config.go | 21 + app/api/experiment.go | 30 + app/api/meta.go | 21 + app/cmd/apply.go | 52 ++ app/cmd/root.go | 1 + app/go.mod | 23 +- app/go.sum | 371 +++++++++++- app/pkg/agent/agent.go | 2 +- app/pkg/agent/agent_test.go | 4 +- app/pkg/application/app.go | 95 ++++ app/pkg/config/config.go | 41 +- app/pkg/eval/distance.go | 140 +++++ app/pkg/eval/distance_test.go | 197 +++++++ app/pkg/eval/evaluator.go | 533 ++++++++++++++++++ app/pkg/eval/evaluator_test.go | 110 ++++ app/pkg/executor/executor.go | 12 +- app/pkg/executor/executor_test.go | 6 +- app/pkg/learn/learner.go | 2 + app/pkg/server/server.go | 2 +- data/eval/gcp/gcb_list.foyle | 32 ++ data/eval/gcp/gke_list.foyle | 32 ++ data/eval/gcp/images_list.foyle | 40 ++ data/eval/iac/pulumi_up.foyle | 32 ++ docs/content/en/docs/evaluation/_index.md | 71 +++ experiments/rag.yaml | 15 + .../foyle/src/gen/foyle/v1alpha1/eval_pb.ts | 115 ++++ iac/dev/main.go | 20 + protos/foyle/v1alpha1/eval.proto | 34 ++ protos/go/foyle/v1alpha1/eval.pb.go | 271 +++++++++ protos/go/foyle/v1alpha1/eval.zap.go | 62 ++ tech_notes/tn002_learning.md | 2 +- tech_notes/tn003_learning_eval.md | 198 +++++++ 32 files changed, 2555 insertions(+), 32 deletions(-) create mode 100644 app/api/config.go create mode 100644 app/api/experiment.go create mode 100644 app/api/meta.go create mode 100644 app/cmd/apply.go create mode 100644 app/pkg/eval/distance.go create mode 100644 app/pkg/eval/distance_test.go create mode 100644 app/pkg/eval/evaluator.go create mode 100644 app/pkg/eval/evaluator_test.go create mode 100644 data/eval/gcp/gcb_list.foyle create mode 100644 data/eval/gcp/gke_list.foyle create mode 100644 data/eval/gcp/images_list.foyle create mode 100644 data/eval/iac/pulumi_up.foyle create mode 100644 docs/content/en/docs/evaluation/_index.md create mode 100644 experiments/rag.yaml create mode 100644 frontend/foyle/src/gen/foyle/v1alpha1/eval_pb.ts create mode 100644 protos/foyle/v1alpha1/eval.proto create mode 100644 protos/go/foyle/v1alpha1/eval.pb.go create mode 100644 protos/go/foyle/v1alpha1/eval.zap.go create mode 100644 tech_notes/tn003_learning_eval.md diff --git a/app/api/config.go b/app/api/config.go new file mode 100644 index 00000000..9789719b --- /dev/null +++ b/app/api/config.go @@ -0,0 +1,21 @@ +package api + +type AgentConfig struct { + // Model is the name of the model to use to generate completions + Model string `json:"model" yaml:"model"` + + // RAG is the configuration for the RAG model + RAG *RAGConfig `json:"rag,omitempty" yaml:"rag,omitempty"` + + // EvalMode is whether to run in evaluation mode or not. + // In EvalMode logs are specially marked so requests won't be used for training. + EvalMode bool `json:"evalMode" yaml:"evalMode"` +} + +// RAGConfig configures the RAG model +type RAGConfig struct { + // Enabled is whether to enable the RAG model or not + Enabled bool `json:"enabled" yaml:"enabled"` + // MaxResults is the maximum number of results to return + MaxResults int `json:"maxResults" yaml:"maxResults"` +} diff --git a/app/api/experiment.go b/app/api/experiment.go new file mode 100644 index 00000000..3a8595cb --- /dev/null +++ b/app/api/experiment.go @@ -0,0 +1,30 @@ +package api + +import "k8s.io/apimachinery/pkg/runtime/schema" + +var ( + ExperimentGVK = schema.FromAPIVersionAndKind(Group+"/"+Version, "Experiment") +) + +// Experiment is a struct that represents an experiment +type Experiment struct { + Metadata Metadata `json:"metadata" yaml:"metadata"` + Spec ExperimentSpec `json:"spec" yaml:"spec"` +} + +type ExperimentSpec struct { + // EvalDir is the directory containing the evaluation the evaluation input + EvalDir string `json:"evalDir" yaml:"evalDir"` + + // DBDir is the directory for the pebble database that will store the results + DBDir string `json:"dbDir" yaml:"dbDir"` + + // SheetID is the ID of the Google Sheet to update with the results. + SheetID string `json:"sheetID" yaml:"sheetID"` + + // SheetName is the name of the sheet to update. + SheetName string `json:"sheetName" yaml:"sheetName"` + + // Agent is the configuration for the agent + Agent *AgentConfig `json:"agent,omitempty" yaml:"agent,omitempty"` +} diff --git a/app/api/meta.go b/app/api/meta.go new file mode 100644 index 00000000..2f450101 --- /dev/null +++ b/app/api/meta.go @@ -0,0 +1,21 @@ +package api + +const ( + Group = "foyle.io" + Version = "v1alpha1" +) + +// N.B. We need to redefine Metadata and not reuse the version in the K8s libraries +// because we want it to have yaml tags so we can serialize with the YAML library. + +// Metadata holds an optional name of the project. +type Metadata struct { + Name string `yaml:"name,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Labels map[string]string `yaml:"labels"` + Annotations map[string]string `yaml:"annotations,omitempty"` + // ResourceVersion is used for optimistic concurrency. + // Ref: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + // This should be treated as an opaque value by clients. + ResourceVersion string `yaml:"resourceVersion,omitempty"` +} diff --git a/app/cmd/apply.go b/app/cmd/apply.go new file mode 100644 index 00000000..48cc62d8 --- /dev/null +++ b/app/cmd/apply.go @@ -0,0 +1,52 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/go-logr/zapr" + "github.com/jlewi/foyle/app/pkg/application" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +// NewApplyCmd create an apply command +func NewApplyCmd() *cobra.Command { + // TODO(jeremy): We should update apply to support the image resource. + applyCmd := &cobra.Command{ + Use: "apply ...", + Short: "Apply the specified resource.", + Run: func(cmd *cobra.Command, args []string) { + err := func() error { + log := zapr.NewLogger(zap.L()) + if len(args) == 0 { + log.Info("apply takes at least one argument which should be the file or directory YAML to apply.") + return errors.New("apply takes at least one argument which should be the file or directory YAML to apply.") + } + logVersion() + + app := application.NewApp() + if err := app.LoadConfig(cmd); err != nil { + return err + } + if err := app.SetupLogging(false); err != nil { + return err + } + + if err := app.SetupRegistry(); err != nil { + return err + } + + return app.ApplyPaths(context.Background(), args) + }() + if err != nil { + fmt.Printf("Error running apply;\n %+v\n", err) + os.Exit(1) + } + }, + } + + return applyCmd +} diff --git a/app/cmd/root.go b/app/cmd/root.go index 65db59c3..3c41811f 100644 --- a/app/cmd/root.go +++ b/app/cmd/root.go @@ -30,5 +30,6 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(NewLearnCmd()) rootCmd.AddCommand(NewConfigCmd()) rootCmd.AddCommand(NewLogsCmd()) + rootCmd.AddCommand(NewApplyCmd()) return rootCmd } diff --git a/app/go.mod b/app/go.mod index cb352a1c..097a2b2c 100644 --- a/app/go.mod +++ b/app/go.mod @@ -6,6 +6,8 @@ replace github.com/jlewi/foyle/protos/go => ../protos/go require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 + github.com/agnivade/levenshtein v1.1.1 + github.com/cockroachdb/pebble v1.1.0 github.com/gin-contrib/cors v1.7.1 github.com/gin-gonic/gin v1.9.1 github.com/go-cmd/cmd v1.4.1 @@ -19,7 +21,7 @@ require ( github.com/honeycombio/honeycomb-opentelemetry-go v0.10.0 github.com/honeycombio/otel-config-go v1.15.0 github.com/jlewi/foyle/protos/go v0.0.0-00010101000000-000000000000 - github.com/jlewi/hydros v0.0.6 + github.com/jlewi/hydros v0.0.7-0.20240503183011-8f99ead373fb github.com/jlewi/monogo v0.0.0-20240123191147-401afe194d74 github.com/maxence-charriere/go-app/v9 v9.8.0 github.com/pkg/errors v0.9.1 @@ -50,17 +52,24 @@ require ( cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/secretmanager v1.11.5 // indirect cloud.google.com/go/storage v1.36.0 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.11.3 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cli/go-gh v0.1.3-0.20221102170023-e3ec45fb1d1b // indirect github.com/cli/safeexec v1.0.0 // indirect github.com/cloudflare/circl v1.1.0 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/cli v24.0.0+incompatible // indirect @@ -71,9 +80,10 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-errors/errors v1.0.1 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-git/go-git/v5 v5.6.1 // indirect @@ -89,6 +99,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect @@ -104,6 +115,8 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -111,6 +124,7 @@ require ( github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -121,7 +135,12 @@ require ( github.com/pelletier/go-toml/v2 v2.2.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/client_golang v1.12.0 // indirect + github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.4.2 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect diff --git a/app/go.sum b/app/go.sum index 472d7f08..e8d18c8b 100644 --- a/app/go.sum +++ b/app/go.sum @@ -1,25 +1,59 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/artifactregistry v1.14.7 h1:W9sVlyb1VRcUf83w7aM3yMsnp4HS4PoyGqYQNG0O5lI= cloud.google.com/go/artifactregistry v1.14.7/go.mod h1:0AUKhzWQzfmeTvT4SjfI4zjot72EMfrkvL9g9aRjnnM= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/cloudbuild v1.15.1 h1:ZB6oOmJo+MTov9n629fiCrO9YZPOg25FZvQ7gIHu5ng= cloud.google.com/go/cloudbuild v1.15.1/go.mod h1:gIofXZSu+XD2Uy+qkOrGKEx45zd7s28u/k8f99qKals= cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Kunde21/markdownfmt/v3 v3.1.0 h1:KiZu9LKs+wFFBQKhrZJrFZwtLnCCWJahL+S+E/3VnM0= github.com/Kunde21/markdownfmt/v3 v3.1.0/go.mod h1:tPXN1RTyOzJwhfHoon9wUr4HGYmWgVxSQN6VBJDkrVc= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -30,11 +64,24 @@ github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwF github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -47,6 +94,10 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -56,6 +107,9 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpV github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cli/go-gh v0.1.3-0.20221102170023-e3ec45fb1d1b h1:W17Cf1UmOvLPbrHcFs9InoY3VPxC0TJWx3QwnjnD4TY= github.com/cli/go-gh v0.1.3-0.20221102170023-e3ec45fb1d1b/go.mod h1:bqxLdCoTZ73BuiPEJx4olcO/XKhVZaFDchFagYRBweE= github.com/cli/safeexec v1.0.0 h1:0VngyaIyqACHdcMNWfo6+KdUYnqEr2Sg+bSP1pdF+dI= @@ -68,6 +122,18 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -77,6 +143,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qeXMx3O0wqM= github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= @@ -107,6 +175,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.7.1 h1:s9SIppU/rk8enVvkzwiC2VK3UZ/0NNGsWfUKvV55rqs= @@ -119,8 +189,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-cmd/cmd v1.4.1 h1:JUcEIE84v8DSy02XTZpUDeGKExk2oW3DA10hTjbQwmc= github.com/go-cmd/cmd v1.4.1/go.mod h1:tbBenttXtZU4c5djS1o7PWL5pd2xAr5sIqH1kGdNiRc= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= @@ -130,6 +200,15 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= @@ -155,20 +234,33 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -181,14 +273,21 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -199,8 +298,19 @@ github.com/google/go-containerregistry v0.18.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -208,6 +318,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -220,12 +332,15 @@ github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+ github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/honeycombio/honeycomb-opentelemetry-go v0.10.0 h1:VEfkR+WP35D2H7PbeUqZtfmbUuKjnv39CdQLgR7TfTY= github.com/honeycombio/honeycomb-opentelemetry-go v0.10.0/go.mod h1:3AcXbJMBChalA+cgGqGn/tKF/Ce5pW5bZtnci60GqJo= github.com/honeycombio/otel-config-go v1.15.0 h1:wUiYG2gGaQEmyZeMMLCb14bx1DR/rxOylkZ05YWN1Jg= github.com/honeycombio/otel-config-go v1.15.0/go.mod h1:9s9M2qozR0OfVCvTMEQBLMglo1SzuRTxulsU0i/NIkA= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -235,12 +350,22 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jlewi/hydros v0.0.6 h1:gylMzSH5VvO+uTtz20JUtJPPr3ZLAQxvH3j/0FHK4FE= github.com/jlewi/hydros v0.0.6/go.mod h1:4fV+JUCnexPY2ZbKzdfV/RsyrfralN832MsUSq/7FqE= +github.com/jlewi/hydros v0.0.7-0.20240503183011-8f99ead373fb h1:2G2k606S3Qcg40czr7gnkeIG5KgQ2wXJ1BMxAuC+P3I= +github.com/jlewi/hydros v0.0.7-0.20240503183011-8f99ead373fb/go.mod h1:4fV+JUCnexPY2ZbKzdfV/RsyrfralN832MsUSq/7FqE= github.com/jlewi/monogo v0.0.0-20240123191147-401afe194d74 h1:pbOw/rOMs0AZ494bGnI6DieGKwqoJQEjHWaJZrvxsJo= github.com/jlewi/monogo v0.0.0-20240123191147-401afe194d74/go.mod h1:4QWrn/wb+L6rHHO3u5N250qJYWaZlHn1a8tFuWhZCP8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -251,6 +376,9 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -279,6 +407,9 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxence-charriere/go-app/v9 v9.8.0 h1:rDfLNvxIKXyjpRS76P45kn9Xj8IumwfoqpsEJYxfd+E= github.com/maxence-charriere/go-app/v9 v9.8.0/go.mod h1:gzgFoeaDuoNHw9MbJraTCKIoKtZ/SoIfOIHHn2FOffc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -289,10 +420,14 @@ github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -300,8 +435,13 @@ github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0 github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -310,11 +450,35 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -335,6 +499,9 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= @@ -356,6 +523,7 @@ github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= @@ -398,12 +566,19 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/detectors/aws/lambda v0.50.0 h1:oQ6efIH7uVEFjMD8dc3Nvf9iVgRiDeKNuBOXwpFkWFI= @@ -455,7 +630,10 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -467,11 +645,35 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -480,17 +682,37 @@ golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -501,25 +723,59 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -528,8 +784,10 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -552,7 +810,10 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -562,15 +823,52 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= @@ -584,16 +882,62 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps= google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= @@ -602,9 +946,17 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= @@ -619,27 +971,34 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -650,7 +1009,12 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= @@ -660,7 +1024,10 @@ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= diff --git a/app/pkg/agent/agent.go b/app/pkg/agent/agent.go index 880e2e0b..1a904a91 100644 --- a/app/pkg/agent/agent.go +++ b/app/pkg/agent/agent.go @@ -72,7 +72,7 @@ func NewAgent(cfg config.Config, client *openai.Client) (*Agent, error) { func (a *Agent) Generate(ctx context.Context, req *v1alpha1.GenerateRequest) (*v1alpha1.GenerateResponse, error) { span := trace.SpanFromContext(ctx) log := logs.FromContext(ctx) - log = log.WithValues("traceId", span.SpanContext().TraceID()) + log = log.WithValues("traceId", span.SpanContext().TraceID(), "evalMode", a.config.EvalMode()) ctx = logr.NewContext(ctx, log) var examples []*v1alpha1.Example diff --git a/app/pkg/agent/agent_test.go b/app/pkg/agent/agent_test.go index 5c1f9473..ab99a4a6 100644 --- a/app/pkg/agent/agent_test.go +++ b/app/pkg/agent/agent_test.go @@ -5,6 +5,8 @@ import ( "os" "testing" + "github.com/jlewi/foyle/app/api" + "github.com/jlewi/foyle/app/pkg/config" "github.com/jlewi/foyle/app/pkg/oai" "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" @@ -71,7 +73,7 @@ func Test_Generate(t *testing.T) { t.Fatalf("Error creating OpenAI client; %v", err) } - cfg.Agent.RAG = &config.RAGConfig{ + cfg.Agent.RAG = &api.RAGConfig{ MaxResults: 3, } cfg.Agent.RAG.Enabled = true diff --git a/app/pkg/application/app.go b/app/pkg/application/app.go index 74cdd6a9..a94bfc25 100644 --- a/app/pkg/application/app.go +++ b/app/pkg/application/app.go @@ -10,6 +10,11 @@ import ( "strings" "time" + "github.com/jlewi/foyle/app/api" + "github.com/jlewi/foyle/app/pkg/eval" + "github.com/jlewi/hydros/pkg/util" + "k8s.io/apimachinery/pkg/runtime/schema" + "go.uber.org/zap/zapcore" "github.com/honeycombio/honeycomb-opentelemetry-go" @@ -22,6 +27,7 @@ import ( "github.com/go-logr/zapr" "github.com/jlewi/foyle/app/pkg/config" "github.com/jlewi/foyle/app/pkg/server" + "github.com/jlewi/hydros/pkg/controllers" "github.com/pkg/errors" "github.com/spf13/cobra" "go.uber.org/zap" @@ -36,6 +42,7 @@ type App struct { Out io.Writer otelShutdownFn func() logClosers []logCloser + Registry *controllers.Registry } // NewApp creates a new application. You should call one more setup/Load functions to properly set it up. @@ -173,6 +180,24 @@ func (a *App) SetupLogging(logToFile bool) error { return nil } +// SetupRegistry sets up the registry with a list of registered controllers +func (a *App) SetupRegistry() error { + if a.Config == nil { + return errors.New("Config is nil; call LoadConfig first") + } + a.Registry = &controllers.Registry{} + + // Register controllers + eval, err := eval.NewEvaluator(*a.Config) + if err != nil { + return err + } + if err := a.Registry.Register(api.ExperimentGVK, eval); err != nil { + return err + } + return nil +} + func (a *App) createCoreForConsole() (zapcore.Core, error) { // Configure encoder for non-JSON format (console-friendly) c := zap.NewDevelopmentEncoderConfig() @@ -276,6 +301,76 @@ func (a *App) SetupServer() (*server.Server, error) { return s, nil } +// ApplyPaths applies the resources in the specified paths. +// Paths can be files or directories. +func (a *App) ApplyPaths(ctx context.Context, paths []string) error { + log := util.LogFromContext(ctx) + + for _, resourcePath := range paths { + newPaths, err := util.FindYamlFiles(resourcePath) + if err != nil { + log.Error(err, "Failed to find YAML files", "path", resourcePath) + return err + } + + paths = append(paths, newPaths...) + } + + for _, path := range paths { + err := a.apply(ctx, path) + if err != nil { + log.Error(err, "Apply failed", "path", path) + } + } + + return nil +} + +func (a *App) apply(ctx context.Context, path string) error { + if a.Registry == nil { + return errors.New("Registry is nil; call SetupRegistry first") + } + + log := zapr.NewLogger(zap.L()) + log.Info("Reading file", "path", path) + rNodes, err := util.ReadYaml(path) + if err != nil { + return err + } + + allErrors := &util.ListOfErrors{ + Causes: []error{}, + } + + for _, n := range rNodes { + m, err := n.GetMeta() + if err != nil { + log.Error(err, "Failed to get metadata", "n", n) + continue + } + log.Info("Read resource", "meta", m) + // Going forward we should be using the registry + gvk := schema.FromAPIVersionAndKind(m.APIVersion, m.Kind) + controller, err := a.Registry.GetController(gvk) + if err != nil { + log.Error(err, "Unsupported kind", "gvk", gvk) + allErrors.AddCause(err) + continue + } + + if err := controller.ReconcileNode(ctx, n); err != nil { + log.Error(err, "Failed to reconcile resource", "name", m.Name, "namespace", m.Namespace, "gvk", gvk) + allErrors.AddCause(err) + } + } + + if len(allErrors.Causes) == 0 { + return nil + } + allErrors.Final = fmt.Errorf("failed to apply one or more resources") + return allErrors +} + // Shutdown the application. func (a *App) Shutdown() error { // N.B. This is a placeholder for any operations that should be performed when shutting down the app diff --git a/app/pkg/config/config.go b/app/pkg/config/config.go index 79545164..b1c2a082 100644 --- a/app/pkg/config/config.go +++ b/app/pkg/config/config.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/jlewi/foyle/app/api" + "github.com/go-logr/zapr" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -18,6 +20,10 @@ import ( "gopkg.in/yaml.v3" ) +// TODO(jeremy): We should finish moving the configuration datastructure into the API package. +// However, we should keep the API package free of other dependencies (e.g. Cobra) so that might necessitate +// refactoring the code a bit more. + // Note: The application uses viper for configuration management. Viper merges configurations from various sources //such as files, environment variables, and command line flags. After merging, viper unmarshals the configuration into the Configuration struct, which is then used throughout the application. @@ -37,31 +43,23 @@ type Config struct { APIVersion string `json:"apiVersion" yaml:"apiVersion" yamltags:"required"` Kind string `json:"kind" yaml:"kind" yamltags:"required"` - Logging Logging `json:"logging" yaml:"logging"` - Server ServerConfig `json:"server" yaml:"server"` - Assets *AssetConfig `json:"assets,omitempty" yaml:"assets,omitempty"` - Agent *AgentConfig `json:"agent,omitempty" yaml:"agent,omitempty"` - OpenAI *OpenAIConfig `json:"openai,omitempty" yaml:"openai,omitempty"` + Logging Logging `json:"logging" yaml:"logging"` + Server ServerConfig `json:"server" yaml:"server"` + Assets *AssetConfig `json:"assets,omitempty" yaml:"assets,omitempty"` + Agent *api.AgentConfig `json:"agent,omitempty" yaml:"agent,omitempty"` + OpenAI *OpenAIConfig `json:"openai,omitempty" yaml:"openai,omitempty"` // AzureOpenAI contains configuration for Azure OpenAI. A non nil value means use Azure OpenAI. AzureOpenAI *AzureOpenAIConfig `json:"azureOpenAI,omitempty" yaml:"azureOpenAI,omitempty"` Telemetry *TelemetryConfig `json:"telemetry,omitempty" yaml:"telemetry,omitempty"` -} - -type AgentConfig struct { - // Model is the name of the model to use to generate completions - Model string `json:"model" yaml:"model"` - // RAG is the configuration for the RAG model - RAG *RAGConfig `json:"rag,omitempty" yaml:"rag,omitempty"` + // TODO(jeremy): Should we move this into the experiment? + Eval *EvalConfig `json:"eval,omitempty" yaml:"eval,omitempty"` } -// RAGConfig configures the RAG model -type RAGConfig struct { - // Enabled is whether to enable the RAG model or not - Enabled bool `json:"enabled" yaml:"enabled"` - // MaxResults is the maximum number of results to return - MaxResults int `json:"maxResults" yaml:"maxResults"` +type EvalConfig struct { + // GCPServiceAccount is the service account to use to update Google Sheets + GCPServiceAccount string `json:"gcpServiceAccount" yaml:"gcpServiceAccount"` } // ServerConfig configures the server @@ -241,6 +239,13 @@ func (c *Config) UseHoneycomb() bool { return true } +func (c *Config) EvalMode() bool { + if c.Agent == nil { + return false + } + return c.Agent.EvalMode +} + // DeepCopy returns a deep copy. func (c *Config) DeepCopy() Config { b, err := json.Marshal(c) diff --git a/app/pkg/eval/distance.go b/app/pkg/eval/distance.go new file mode 100644 index 00000000..1e60d912 --- /dev/null +++ b/app/pkg/eval/distance.go @@ -0,0 +1,140 @@ +package eval + +import ( + "errors" + "strings" + + "github.com/agnivade/levenshtein" + "github.com/jlewi/foyle/app/pkg/executor" +) + +const ( + alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +type command struct { + unnamed []string + named map[string]string +} + +// Distance computes the distance between two instructions +// +// For details refer to tn003_learning_eval.md. +func Distance(left executor.Instruction, right executor.Instruction) (int, error) { + // Split each instruction into named and unnamed arguments + leftArgs := splitInstruction(left) + rightArgs := splitInstruction(right) + + // Compute the distance of the unnamed arguments + unamedDistance, err := editDistance(leftArgs.unnamed, rightArgs.unnamed) + if err != nil { + return -1, err + } + + // Compute the distance of the named arguments + namedDistance := dictDistance(leftArgs.named, rightArgs.named) + return unamedDistance + namedDistance, nil +} + +// editDistance computes the edit distance between two slices of strings. +// Each string in the slice is considered a single token. +func editDistance(left []string, right []string) (int, error) { + // Our levenstein distance function operates on strings. + // So we need to map our tokens to single character strings. + // We do this by building up a dictionary mapping the tokens to single character strings. + // We currently use a fixed alphabet of 62 characters. We should be able to easily extend this to 100s or thousands + // of characters because our levenstein library works with UTF8. Just using 1 byte we should be able to represent + // 128 characters. I wanted to keep it to printable characters. Seems unlikely we will have commands + // of more than 62 tokens. + + t := tokenizer{ + index: 0, + dict: map[string]string{}, + } + leftVal, err := t.tokenize(left) + if err != nil { + return 0, err + } + + rightVal, err := t.tokenize(right) + if err != nil { + return 0, err + } + // I picked this particular library because + // Its code was readable and pretty compact. + // https://github.com/ka-weihe/fast-levenshtein claims to be 15 times faster but its code is unreadable because + // its so heavily optimized. Its also not thread safe although it would be trivial to make it so. + return levenshtein.ComputeDistance(leftVal, rightVal), nil +} + +type tokenizer struct { + index int + dict map[string]string +} + +func (t *tokenizer) tokenize(vals []string) (string, error) { + result := "" + for _, l := range vals { + if _, ok := t.dict[l]; !ok { + t.dict[l] = string(alphabet[t.index]) + t.index++ + if t.index >= len(alphabet) { + return "", errors.New("Too many tokens") + } + } + result += t.dict[l] + } + return result, nil +} + +// dictDistance computes the distance between two dictionaries. +func dictDistance(left map[string]string, right map[string]string) int { + // Each key in one dictionary but not the other contributes 1 to the distance. + distance := 0 + distance += countKeysNotInRight(left, right) + distance += countKeysNotInRight(right, left) + + // Now we need to check the values of the keys that are in both dictionaries. + // If the values don't match then we need to add 1 to the distance. + for k := range left { + if _, ok := right[k]; !ok { + continue + } + + if left[k] != right[k] { + distance += 1 + } + } + + return distance +} + +func countKeysNotInRight(left map[string]string, right map[string]string) int { + d := 0 + for k := range left { + if _, ok := right[k]; !ok { + d += 1 + } + } + return d +} + +func splitInstruction(instruction executor.Instruction) command { + c := command{ + unnamed: []string{instruction.Command.Name}, + named: map[string]string{}, + } + + for _, arg := range instruction.Command.Args { + if strings.HasPrefix(arg, "--") { + pieces := strings.Split(arg, "=") + if len(pieces) >= 2 { + c.named[pieces[0]] = strings.Join(pieces[1:], "=") + continue + } + } + c.unnamed = append(c.unnamed, arg) + } + + return c +} diff --git a/app/pkg/eval/distance_test.go b/app/pkg/eval/distance_test.go new file mode 100644 index 00000000..42b29bb0 --- /dev/null +++ b/app/pkg/eval/distance_test.go @@ -0,0 +1,197 @@ +package eval + +import ( + "testing" + + "github.com/go-cmd/cmd" + "github.com/google/go-cmp/cmp" + "github.com/jlewi/foyle/app/pkg/executor" +) + +func Test_Distance(t *testing.T) { + type testCase struct { + name string + left []string + right []string + expected int + } + + cases := []testCase{ + { + name: "equal", + left: []string{"gcloud", "-p", "acme", "--foo=bar", "baz"}, + right: []string{"gcloud", "-p", "acme", "--foo=bar", "baz"}, + expected: 0, + }, + { + name: "notequal", + left: []string{"gcloud", "-p", "acme", "--foo=bar", "baz"}, + right: []string{"gcloud", "acme", "--foo=lab", "baz"}, + expected: 2, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + left := executor.Instruction{ + Command: cmd.NewCmd(c.left[0], c.left[1:]...), + } + right := executor.Instruction{ + Command: cmd.NewCmd(c.right[0], c.right[1:]...), + } + actual, err := Distance(left, right) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if actual != c.expected { + t.Errorf("Expected %d but got %d", c.expected, actual) + } + }) + } +} + +func Test_SplitInstruction(t *testing.T) { + type testCase struct { + name string + args []string + expected command + } + + cases := []testCase{ + { + name: "simple", + args: []string{"gcloud", "-p", "acme", "--foo=bar", "baz"}, + expected: command{ + unnamed: []string{"gcloud", "-p", "acme", "baz"}, + named: map[string]string{ + "--foo": "bar", + }, + }, + }, + { + name: "equals-in-value", + args: []string{"foyle", "config", "--foo=bar=baz"}, + expected: command{ + unnamed: []string{"foyle", "config"}, + named: map[string]string{ + "--foo": "bar=baz", + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + instruction := executor.Instruction{ + Command: cmd.NewCmd(c.args[0], c.args[1:]...), + } + actual := splitInstruction(instruction) + if d := cmp.Diff(c.expected, actual, cmp.AllowUnexported(command{})); d != "" { + t.Errorf("Unexpected result (-want +got):\n%s", d) + } + }) + } +} + +func Test_editDistance(t *testing.T) { + type testCase struct { + name string + left []string + right []string + expected int + } + + cases := []testCase{ + { + name: "simple", + left: []string{"gcloud", "-p", "acme", "baz"}, + right: []string{"gcloud", "-p", "acme", "baz"}, + expected: 0, + }, + { + name: "simple", + left: []string{"-p", "acme", "baz", "extra"}, + right: []string{"gcloud", "-p", "acme", "baz"}, + expected: 2, + }, + { + name: "substitution", + left: []string{"acme", "foo", "baz"}, + right: []string{"acme", "bar", "baz"}, + expected: 1, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + actual, err := editDistance(c.left, c.right) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if actual != c.expected { + t.Errorf("Expected %d but got %d", c.expected, actual) + } + }) + } +} + +func Test_dictDistance(t *testing.T) { + type testCase struct { + name string + left map[string]string + right map[string]string + expected int + } + + cases := []testCase{ + { + name: "equal", + left: map[string]string{ + "a": "1", + "b": "2", + }, + right: map[string]string{ + "a": "1", + "b": "2", + }, + expected: 0, + }, + { + name: "extra", + left: map[string]string{ + "a": "1", + "b": "2", + "e": "3", + }, + right: map[string]string{ + "a": "1", + "b": "2", + "f": "4", + }, + expected: 2, + }, + { + name: "diff", + left: map[string]string{ + "a": "1", + "b": "2", + "c": "3", + }, + right: map[string]string{ + "a": "1", + "b": "2", + "c": "4", + }, + expected: 1, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + actual := dictDistance(c.left, c.right) + if actual != c.expected { + t.Errorf("Expected %d but got %d", c.expected, actual) + } + }) + } +} diff --git a/app/pkg/eval/evaluator.go b/app/pkg/eval/evaluator.go new file mode 100644 index 00000000..48a6a82e --- /dev/null +++ b/app/pkg/eval/evaluator.go @@ -0,0 +1,533 @@ +package eval + +import ( + "context" + "os" + "path/filepath" + + "github.com/jlewi/foyle/app/api" + "github.com/jlewi/foyle/app/pkg/agent" + "github.com/jlewi/foyle/app/pkg/oai" + "sigs.k8s.io/kustomize/kyaml/yaml" + + "github.com/cockroachdb/pebble" + "github.com/google/uuid" + "github.com/jlewi/foyle/app/pkg/config" + "github.com/jlewi/foyle/app/pkg/docs" + "github.com/jlewi/foyle/app/pkg/executor" + "github.com/jlewi/foyle/app/pkg/logs" + "github.com/jlewi/foyle/protos/go/foyle/v1alpha1" + "github.com/jlewi/monogo/helpers" + "github.com/pkg/errors" + "google.golang.org/api/googleapi" + "google.golang.org/api/impersonate" + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +type Evaluator struct { + config config.Config + parser *executor.BashishParser +} + +func NewEvaluator(cfg config.Config) (*Evaluator, error) { + parser, err := executor.NewBashishParser() + + if err != nil { + return nil, err + } + + return &Evaluator{ + config: cfg, + parser: parser, + }, nil +} + +func (e *Evaluator) ReconcileNode(ctx context.Context, node *yaml.RNode) error { + experiment := &api.Experiment{} + if err := node.YNode().Decode(experiment); err != nil { + return errors.Wrapf(err, "Failed to decode experiment") + } + + return e.Reconcile(ctx, *experiment) +} + +func (e *Evaluator) Reconcile(ctx context.Context, experiment api.Experiment) error { + db, err := pebble.Open(experiment.Spec.DBDir, &pebble.Options{}) + if err != nil { + return err + } + defer helpers.DeferIgnoreError(db.Close) + + if experiment.Spec.Agent == nil { + return errors.New("Agent is required") + } + agent, err := e.setupAgent(ctx, *experiment.Spec.Agent) + if err != nil { + return err + } + + // List all the files + files, err := e.listEvalFiles(ctx, experiment.Spec.EvalDir) + if err != nil { + return err + } + + log := logs.FromContext(ctx) + log.Info("Found eval files", "numFiles", len(files)) + + // Now iterate over the DB and figure out which files haven't been loaded into the db. + + unloadedFiles, err := e.findUnloadedFiles(ctx, db, files) + if err != nil { + return err + } + log.Info("Found unloaded files", "numFiles", len(unloadedFiles)) + + // We need to load the evaluation data into the database. + if err := e.loadFoyleFiles(ctx, db, unloadedFiles); err != nil { + return err + } + + // Now generate predictions for any results that are missing them. + if err := e.reconcilePredictions(ctx, db, agent); err != nil { + return err + } + + // Compute the distance + if err := e.reconcileDistance(ctx, db); err != nil { + return err + } + + // Update the Google Sheet + if err := e.updateGoogleSheet(ctx, experiment, db); err != nil { + return err + } + return nil +} + +func (e *Evaluator) setupAgent(ctx context.Context, agentConfig api.AgentConfig) (*agent.Agent, error) { + cfg := e.config.DeepCopy() + + // Swap out the AgentConfig + cfg.Agent = &agentConfig + + // Ensure we are in evaluation mode. + cfg.Agent.EvalMode = true + + client, err := oai.NewClient(cfg) + if err != nil { + return nil, err + } + agent, err := agent.NewAgent(cfg, client) + + if err != nil { + return nil, err + } + return agent, nil +} + +func (e *Evaluator) reconcilePredictions(ctx context.Context, db *pebble.DB, agent *agent.Agent) error { + olog := logs.FromContext(ctx) + iter, err := db.NewIterWithContext(ctx, nil) + if err != nil { + return err + } + defer iter.Close() + + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + if key == nil { + break + } + + log := olog.WithValues("id", string(key)) + value, err := iter.ValueAndErr() + if err != nil { + return errors.Wrapf(err, "Failed to read value for key %s", string(key)) + } + + result := &v1alpha1.EvalResult{} + if err := proto.Unmarshal(value, result); err != nil { + return errors.Wrapf(err, "Failed to unmarshal value for key %s", string(key)) + } + + if len(result.GetActual()) > 0 { + log.Info("Skipping; already have answer", "path", result.ExampleFile) + // We have the answer so we don't need to generate it. + continue + } + + if len(result.Actual) == 0 { + // We need to generate the answer. + resp, err := agent.Generate(ctx, &v1alpha1.GenerateRequest{ + Doc: result.Example.Query, + }) + if err != nil { + result.Error = err.Error() + result.Status = v1alpha1.EvalResultStatus_ERROR + continue + } + + result.Actual = resp.GetBlocks() + b, err := proto.Marshal(result) + if err != nil { + return errors.Wrapf(err, "Failed to marshal result") + } + if err := db.Set(key, b, nil); err != nil { + return errors.Wrapf(err, "Failed to write result to DB") + } + } + } + return nil +} + +func (e *Evaluator) updateResult(ctx context.Context, id string, result *v1alpha1.EvalResult, db *pebble.DB) error { + b, err := proto.Marshal(result) + if err != nil { + return errors.Wrapf(err, "Failed to marshal result") + } + if err := db.Set([]byte(id), b, nil); err != nil { + return errors.Wrapf(err, "Failed to write result to DB") + } + return nil +} + +func (e *Evaluator) reconcileDistance(ctx context.Context, db *pebble.DB) error { + olog := logs.FromContext(ctx) + iter, err := db.NewIterWithContext(ctx, nil) + if err != nil { + return err + } + defer iter.Close() + + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + if key == nil { + break + } + + log := olog.WithValues("id", string(key)) + value, err := iter.ValueAndErr() + if err != nil { + return errors.Wrapf(err, "Failed to read value for key %s", string(key)) + } + + result := &v1alpha1.EvalResult{} + if err := proto.Unmarshal(value, result); err != nil { + return errors.Wrapf(err, "Failed to unmarshal value for key %s", string(key)) + } + + if result.Distance >= 0 && result.Status != v1alpha1.EvalResultStatus_UNKNOWN_EVAL_RESULT_STATUS { + log.Info("Skipping; distance already computed") + continue + } + + var actualBlock *v1alpha1.Block + + for _, b := range result.Actual { + if b.Kind == v1alpha1.BlockKind_CODE { + actualBlock = b + break + } + } + if actualBlock == nil { + log.Info("Skipping; no code blocks found in the answer") + continue + } + + if len(result.Example.GetAnswer()) > 1 { + log.Info("Warning; expected answer more than one answer block. Only the first is used") + } + + expected, err := e.parser.Parse(result.Example.Answer[0].GetContents()) + if err != nil { + log.Error(err, "Failed to parse expected answer to command") + result.Error = err.Error() + result.Status = v1alpha1.EvalResultStatus_ERROR + if err := e.updateResult(ctx, string(key), result, db); err != nil { + log.Error(err, "Failed to update result") + } + continue + } + + actual, err := e.parser.Parse(actualBlock.GetContents()) + if err != nil { + log.Error(err, "Failed to parse actual answer to command") + result.Error = err.Error() + result.Status = v1alpha1.EvalResultStatus_ERROR + if err := e.updateResult(ctx, string(key), result, db); err != nil { + log.Error(err, "Failed to update result") + } + continue + } + + distance, err := Distance(expected[0], actual[0]) + + if err != nil { + log.Error(err, "Failed to compute distance") + result.Error = err.Error() + result.Status = v1alpha1.EvalResultStatus_ERROR + if err := e.updateResult(ctx, string(key), result, db); err != nil { + log.Error(err, "Failed to update result") + } + continue + } + + result.Distance = int32(distance) + result.Status = v1alpha1.EvalResultStatus_DONE + if err := e.updateResult(ctx, string(key), result, db); err != nil { + log.Error(err, "Failed to update result") + } + } + return nil +} + +func (e *Evaluator) updateGoogleSheet(ctx context.Context, experiment api.Experiment, db *pebble.DB) error { + log := logs.FromContext(ctx) + if e.config.Eval == nil || e.config.Eval.GCPServiceAccount == "" { + return errors.New("GCPServiceAccount is required to update Google Sheet") + } + + sheetName := experiment.Spec.SheetName + sheetID := experiment.Spec.SheetID + + if sheetID == "" { + return errors.New("SheetID is required to update Google Sheet") + } + + if sheetName == "" { + return errors.New("SheetName is required to update Google Sheet") + } + + log = log.WithValues("spreadsheetID", sheetID, "sheetName", sheetName) + log.Info("Updating Google Sheet") + credentialsConfig := &impersonate.CredentialsConfig{ + TargetPrincipal: e.config.Eval.GCPServiceAccount, + Scopes: []string{"https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive"}, + } + + credentials, err := impersonate.CredentialsTokenSource(ctx, *credentialsConfig) + if err != nil { + log.Error(err, "Unable to create impersonated credentials") + return err + } + + srv, err := sheets.NewService(ctx, option.WithTokenSource(credentials)) + if err != nil { + log.Error(err, "Unable to retrieve Sheets client") + return err + } + + // Create the sheet if it doesn't exist + batchUpdateRequest := &sheets.BatchUpdateSpreadsheetRequest{ + Requests: []*sheets.Request{ + { + AddSheet: &sheets.AddSheetRequest{ + Properties: &sheets.SheetProperties{ + Title: experiment.Spec.SheetName, + }, + }, + }, + }, + } + + _, err = srv.Spreadsheets.BatchUpdate(experiment.Spec.SheetID, batchUpdateRequest).Context(ctx).Do() + if err != nil { + apiErr, ok := err.(*googleapi.Error) + if ok { + if apiErr.Code == 400 { + log.V(1).Info("Sheet already exists") + } else { + log.Error(err, "Unable to create new sheet ") + return errors.Wrapf(err, "Unable to create new sheet named: %s", sheetName) + } + } else { + return errors.Wrapf(err, "Unable to create new sheet named: %s", sheetName) + } + } + + // Prepare the value range to write + writeRange := sheetName + values := [][]interface{}{{"id", "prompt", "actual", "expected", "distance"}} + + iter, err := db.NewIterWithContext(ctx, nil) + if err != nil { + return err + } + defer iter.Close() + + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + if key == nil { + break + } + + value, err := iter.ValueAndErr() + if err != nil { + return errors.Wrapf(err, "Failed to read value for key %s", string(key)) + } + + result := &v1alpha1.EvalResult{} + if err := proto.Unmarshal(value, result); err != nil { + return errors.Wrapf(err, "Failed to unmarshal value for key %s", string(key)) + } + + prompt := docs.DocToMarkdown(result.Example.Query) + row := []interface{}{result.Example.Id, prompt, docs.BlocksToMarkdown(result.Actual), docs.BlocksToMarkdown(result.Example.Answer), result.Distance} + values = append(values, row) + } + valueRange := &sheets.ValueRange{ + Values: values, + } + + // Write the value range to the sheet + _, err = srv.Spreadsheets.Values.Update(sheetID, writeRange, valueRange). + ValueInputOption("USER_ENTERED"). + Context(ctx). + Do() + if err != nil { + log.Error(err, "Unable to write data to sheet") + return errors.Wrapf(err, "Unable to write data to sheet") + } + + return nil +} + +func (e *Evaluator) findUnloadedFiles(ctx context.Context, db *pebble.DB, files []string) ([]string, error) { + unprocessed := map[string]bool{} + + iter, err := db.NewIterWithContext(ctx, nil) + if err != nil { + return nil, err + } + defer iter.Close() + + for _, file := range files { + unprocessed[file] = true + } + + // Iterate over the files in the DB and remove them from the list of files to load. + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + if key == nil { + break + } + + value, err := iter.ValueAndErr() + if err != nil { + // Should we ignore the error? + return nil, errors.Wrapf(err, "Failed to read value for key %s", string(key)) + } + + result := &v1alpha1.EvalResult{} + if err := proto.Unmarshal(value, result); err != nil { + return nil, errors.Wrapf(err, "Failed to unmarshal value for key %s", string(key)) + } + + delete(unprocessed, result.ExampleFile) + + } + + toProcess := make([]string, 0, len(unprocessed)) + for file := range unprocessed { + toProcess = append(toProcess, file) + } + + return toProcess, nil +} + +// listEvalFiles returns a list of the all the Foyle files in the eval directory. +func (e *Evaluator) listEvalFiles(ctx context.Context, evalDir string) ([]string, error) { + examples := make([]string, 0, 100) + err := filepath.Walk(evalDir, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + if filepath.Ext(path) != ".foyle" { + return nil + } + + examples = append(examples, path) + return nil + }) + + return examples, err +} + +// loadFoyleFiles loads a bunch of Foyle files representing evaluation data and converts them into example +// protos. +func (e *Evaluator) loadFoyleFiles(ctx context.Context, db *pebble.DB, files []string) error { + oLog := logs.FromContext(ctx) + + allErrors := &helpers.ListOfErrors{} + for _, path := range files { + log := oLog.WithValues("path", path) + log.Info("Processing file") + + contents, err := os.ReadFile(path) + if err != nil { + log.Error(err, "Failed to read file") + allErrors.AddCause(err) + // Keep going + continue + } + + doc := &v1alpha1.Doc{} + if err := protojson.Unmarshal(contents, doc); err != nil { + log.Error(err, "Failed to unmarshal example") + allErrors.AddCause(err) + // Keep going + continue + } + + if len(doc.GetBlocks()) < 2 { + log.Info("Skipping doc; too few blocks; at least two are required") + continue + } + + answer := doc.GetBlocks()[len(doc.GetBlocks())-1] + doc.Blocks = doc.Blocks[:len(doc.GetBlocks())-1] + if answer.Kind != v1alpha1.BlockKind_CODE { + log.Info("Skipping doc; last block must be code") + continue + } + + id := uuid.NewString() + example := &v1alpha1.Example{ + Id: id, + Query: doc, + Answer: []*v1alpha1.Block{answer}, + } + + result := &v1alpha1.EvalResult{ + Example: example, + ExampleFile: path, + // initialize distance to a negative value so we can tell when it hasn't been computed + Distance: -1, + } + + b, err := proto.Marshal(result) + if err != nil { + log.Error(err, "Failed to marshal result") + allErrors.AddCause(err) + // Keep going + continue + } + if err := db.Set([]byte(id), b, nil); err != nil { + log.Error(err, "Failed to write result to DB") + allErrors.AddCause(err) + // Keep going + continue + } + } + + if len(allErrors.Causes) > 0 { + return allErrors + } + + return nil +} diff --git a/app/pkg/eval/evaluator_test.go b/app/pkg/eval/evaluator_test.go new file mode 100644 index 00000000..731fc276 --- /dev/null +++ b/app/pkg/eval/evaluator_test.go @@ -0,0 +1,110 @@ +package eval + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/jlewi/foyle/app/api" + "github.com/pkg/errors" + + "github.com/cockroachdb/pebble" + "github.com/jlewi/foyle/app/pkg/config" + "github.com/jlewi/monogo/helpers" + "go.uber.org/zap" +) + +func Test_Evaluator(t *testing.T) { + if os.Getenv("GITHUB_ACTIONS") != "" { + t.Skipf("Test is skipped in GitHub actions") + } + + log, err := zap.NewDevelopmentConfig().Build() + if err != nil { + t.Fatalf("Error creating logger; %v", err) + } + zap.ReplaceGlobals(log) + + if err := config.InitViper(nil); err != nil { + t.Fatalf("Error initializing Viper; %v", err) + } + cfg := config.GetConfig() + + e, err := NewEvaluator(*cfg) + if err != nil { + t.Fatalf("Error creating evaluator; %v", err) + } + + experiment, err := experimentForTesting() + if err != nil { + t.Fatalf("Error creating experiment; %v", err) + } + + if err := e.Reconcile(context.Background(), *experiment); err != nil { + t.Fatalf("Error reconciling; %v", err) + } +} + +func Test_Evaluator_Google_Sheets(t *testing.T) { + if os.Getenv("GITHUB_ACTIONS") != "" { + t.Skipf("Test is skipped in GitHub actions") + } + + log, err := zap.NewDevelopmentConfig().Build() + if err != nil { + t.Fatalf("Error creating logger; %v", err) + } + zap.ReplaceGlobals(log) + + if err := config.InitViper(nil); err != nil { + t.Fatalf("Error initializing Viper; %v", err) + } + cfg := config.GetConfig() + + e, err := NewEvaluator(*cfg) + if err != nil { + t.Fatalf("Error creating evaluator; %v", err) + } + + experiment, err := experimentForTesting() + if err != nil { + t.Fatalf("Error creating experiment; %v", err) + } + + db, err := pebble.Open(experiment.Spec.DBDir, &pebble.Options{}) + if err != nil { + t.Fatalf("Error opening DB; %v", err) + } + defer helpers.DeferIgnoreError(db.Close) + if err := e.updateGoogleSheet(context.Background(), *experiment, db); err != nil { + t.Fatalf("Error updating Google Sheet; %v", err) + } +} + +func experimentForTesting() (*api.Experiment, error) { + cwd, err := os.Getwd() + if err != nil { + return nil, errors.Wrapf(err, "Error getting working directory") + } + evalDir, err := filepath.Abs(filepath.Join(cwd, "..", "..", "..", "data", "eval")) + if err != nil { + return nil, errors.Wrapf(err, "Error getting eval directory") + } + + return &api.Experiment{ + Spec: api.ExperimentSpec{ + EvalDir: evalDir, + DBDir: "/tmp/foyle/eval", + SheetID: "1O0thD-p9DBF4G_shGMniivBB3pdaYifgSzWXBxELKqE", + SheetName: "Results", + Agent: &api.AgentConfig{ + Model: config.DefaultModel, + // No need to test RAG as part of testing evaluation. + RAG: &api.RAGConfig{ + Enabled: false, + }, + }, + }, + }, nil +} diff --git a/app/pkg/executor/executor.go b/app/pkg/executor/executor.go index af9bfc16..607e0713 100644 --- a/app/pkg/executor/executor.go +++ b/app/pkg/executor/executor.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/jlewi/foyle/app/pkg/config" + "go.uber.org/zap" "github.com/go-logr/logr" @@ -22,23 +24,25 @@ import ( // Executor is responsible for executing the commands type Executor struct { v1alpha1.UnimplementedExecuteServiceServer - p *BashishParser + p *BashishParser + config config.Config } -func NewExecutor() (*Executor, error) { +func NewExecutor(cfg config.Config) (*Executor, error) { p, err := NewBashishParser() if err != nil { return nil, err } return &Executor{ - p: p, + p: p, + config: cfg, }, nil } func (e *Executor) Execute(ctx context.Context, req *v1alpha1.ExecuteRequest) (*v1alpha1.ExecuteResponse, error) { span := trace.SpanFromContext(ctx) log := logs.FromContext(ctx) - log = log.WithValues("traceId", span.SpanContext().TraceID()) + log = log.WithValues("traceId", span.SpanContext().TraceID(), "evalMode", e.config.EvalMode()) ctx = logr.NewContext(ctx, log) log.Info("Executor.Execute", "blockId", req.GetBlock().GetId(), zap.Object("request", req)) diff --git a/app/pkg/executor/executor_test.go b/app/pkg/executor/executor_test.go index 45093c10..a4a662b1 100644 --- a/app/pkg/executor/executor_test.go +++ b/app/pkg/executor/executor_test.go @@ -5,6 +5,8 @@ import ( "fmt" "testing" + "github.com/jlewi/foyle/app/pkg/config" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp" @@ -43,8 +45,8 @@ func Test_Executor(t *testing.T) { }, }, } - - e, err := NewExecutor() + cfg := config.Config{} + e, err := NewExecutor(cfg) if err != nil { t.Fatalf("Failed to create executor: %v", err) } diff --git a/app/pkg/learn/learner.go b/app/pkg/learn/learner.go index ffc52f58..4c85513c 100644 --- a/app/pkg/learn/learner.go +++ b/app/pkg/learn/learner.go @@ -46,6 +46,8 @@ func (l *Learner) Reconcile(ctx context.Context) error { // TODO(jeremy): Can we call Analyze to compute the latest logs? log := logs.FromContext(ctx) + log.Error(errors.New("Not implemented"), "The learning code needs to be updated to filter out examples that are used for evaluation") + trainDir := l.Config.GetTrainingDir() if _, err := os.Stat(trainDir); err != nil { if os.IsNotExist(err) { diff --git a/app/pkg/server/server.go b/app/pkg/server/server.go index 791a26f5..f1d30342 100644 --- a/app/pkg/server/server.go +++ b/app/pkg/server/server.go @@ -60,7 +60,7 @@ type Server struct { // NewServer creates a new server func NewServer(config config.Config) (*Server, error) { - e, err := executor.NewExecutor() + e, err := executor.NewExecutor(config) if err != nil { return nil, err } diff --git a/data/eval/gcp/gcb_list.foyle b/data/eval/gcp/gcb_list.foyle new file mode 100644 index 00000000..69049ce9 --- /dev/null +++ b/data/eval/gcp/gcb_list.foyle @@ -0,0 +1,32 @@ +{ + "blocks": [ + { + "kind": "MARKUP", + "language": "markdown", + "contents": "List the most recent image builds" + }, + { + "kind": "CODE", + "language": "bash", + "contents": "gcloud builds list --project=foyle-public", + "outputs": [ + { + "items": [ + { + "mime": "text/plain", + "textData": "exitCode: 0" + } + ] + }, + { + "items": [ + { + "mime": "text/plain", + "textData": "stdout:\nID CREATE_TIME DURATION SOURCE IMAGES STATUS\n124c5ddd-b040-4749-87ba-47b68cc5326e 2024-04-19T20:41:14+00:00 2M21S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.09fe389c90b80da08b773145a3e35b7032f4e0b5.tgz - SUCCESS\nfb836b86-2b3f-4e0c-aefd-4ec60a6abdb9 2024-04-19T20:07:16+00:00 2M7S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.0b15ea610db1d5c1bb566f06e69a502a5bcbca1d.tgz - SUCCESS\n75556df2-dc55-410c-b9b5-038b9937e74d 2024-04-17T22:29:21+00:00 2M37S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.da26d1b1d8b0fae2309ed90f72f9d9c609945d39.tgz - SUCCESS\n47a93531-99c0-4855-932e-4365db430fae 2024-04-17T01:52:28+00:00 2M43S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.ce6affc61ec2a93e463e5787a799e42199f42213.tgz - SUCCESS\nbcea8b7a-27a9-48fe-914f-2781ce222a82 2024-04-13T22:27:10+00:00 2M57S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.c6d2e14d309b57fac3eef004197db6f41b90786f.tgz - SUCCESS\n9b6cc343-29fe-4fa1-90cc-8cb5f7f6f049 2024-04-13T00:10:31+00:00 2M2S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.51a604f692eb0803c7d2dcc00fac7709df1fa822.tgz - SUCCESS\n47bcfc80-7a32-4d95-abc6-dd61c11ccb46 2024-04-12T22:44:14+00:00 2M34S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.8b27185ae391801b216274ce80946244dbf84fd6.tgz - SUCCESS\nc907682e-e477-4900-95fc-d73ec210902b 2024-04-12T22:39:28+00:00 4M30S gs://builds-foyle-public/foyle-public/images/hydros/hydros.48988a89c2dbabe5fb7f4e6970fb8c403d8c8c83.tgz - FAILURE\ne9b641b4-ab8d-45ec-abae-c7d76ede19e0 2024-04-12T22:35:00+00:00 4M16S gs://builds-foyle-public/foyle-public/images/hydros/hydros.61c688b161974544d94933e8160e56508a4ef67f.tgz - FAILURE\na59a9932-37f6-42fd-afd4-4c727efb5e47 2024-04-12T21:57:29+00:00 3M21S gs://builds-foyle-public/foyle-public/images/hydros/hydros.fd32250d8e277ab5d6f71b4efb1337bb0ff2e95b.tgz - FAILURE\nec7d695f-f8a8-416c-b06c-7d9af2b974c9 2024-04-12T21:54:04+00:00 3M15S gs://builds-foyle-public/foyle-public/images/hydros/hydros.fd32250d8e277ab5d6f71b4efb1337bb0ff2e95b.tgz - FAILURE\nc92cd6f9-fe6d-447e-80f0-de5280c37140 2024-04-11T20:04:48+00:00 2M33S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.e8901ec0fb1dd84a26b4d4976de16319fad10cb1.tgz - SUCCESS\nba916139-8ba3-4b58-9f9d-f1b0a9617f79 2024-04-11T15:41:21+00:00 3M16S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.d2641e55fea10d79305aa25f6fa2fd19ae554b7f.tgz - SUCCESS\nafdac249-1290-4844-be41-3d2d6d8bccb2 2024-04-11T15:30:05+00:00 4M50S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.dd52c555034fe0e16efaed64cce276d5ebc0f2c2.tgz - SUCCESS\na7889c13-7069-473b-9fa7-dba7940662a6 2024-04-11T15:16:34+00:00 2M21S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.d2415d1fc8c6647cc6f8ecb3191f31b0a0fd8c7f.tgz - SUCCESS\ncc398381-ea0b-40e5-a5c5-0afe4c311338 2024-04-09T01:08:05+00:00 4M25S gs://builds-foyle-public/foyle-public/images/hydros/hydros.f97f18c5edb20de30bc3fa6ba25835d9afed06c9.tgz - SUCCESS\n323523fb-6503-4838-b92c-07659e634894 2024-04-08T23:01:46+00:00 2M gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.6ba56ae46fc150d3a39a2d592183f83cc8c67d44.tgz - SUCCESS\ne792f9a1-5540-4c28-bc6a-d1330cf18e08 2024-04-04T14:42:43+00:00 3M31S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.7a82dd2768441944b620cf22561222e925fee624.tgz - SUCCESS\nb2952cf5-b814-4dc6-92cc-95c74c1f02a1 2024-04-04T01:54:18+00:00 8M7S gs://builds-foyle-public/foyle-public/images/vscode-web-assets.107d350496ec50984207cdccdab73602dd32de11.tgz - SUCCESS\n98980d6f-ca20-404c-abcd-544098d5ab3e 2024-04-04T00:21:26+00:00 46M27S gs://builds-foyle-public/foyle-public/images/vscode.0a44acc96351fd10d0373d487f5d558397f0ac39.tgz - SUCCESS\n\nID CREATE_TIME DURATION SOURCE IMAGES STATUS\n848d642f-f9d9-4920-90ab-9d7934f46206 2024-03-29T02:24:40+00:00 2M41S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.0f57f56d6fb1d73418d854b600591a571f2417ff.tgz - SUCCESS\n64b885c4-395b-4810-b42f-24a280a6d7e4 2024-03-27T15:37:49+00:00 1M31S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.9f4215579f9b3cbb411ce872b1297fe81ecf347a.tgz - SUCCESS\n031d2e4a-2219-46cb-97e0-7acae74dfb0d 2024-03-27T15:23:56+00:00 2M9S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.52123d6f90f72054a11667b56bc071d322612e89.tgz - FAILURE\ne891ebfc-973a-498a-85d5-9ccdc04a387b 2024-03-27T15:20:25+00:00 51S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.5a8e3e527ba56ff63423afa1cffd31395565a89a.tgz - FAILURE\ne5b89e3a-fa8d-4f24-b018-b837a1fbd539 2024-03-27T15:15:49+00:00 1M16S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.00ebe5dcfc930752cc8aca1383c8524ac9afa984.tgz - FAILURE\n538bb476-3754-4623-b12e-9afd9948815d 2024-03-27T15:10:58+00:00 12S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.9c0fead30a336e37e8436510165eff2ad8acb106.tgz - FAILURE\n4df9adb4-924d-43f6-84ef-1b68c9428baf 2024-03-27T15:05:37+00:00 13S gs://builds-foyle-public/foyle-public/images/foyle-vscode-ext.9c0fead30a336e37e8436510165eff2ad8acb106.tgz - FAILURE\n4f528397-18a3-4aaf-b687-8551ec3fe33a 2024-03-26T21:10:40+00:00 8M49S gs://builds-foyle-public/foyle-public/images/vscode-web-assets.897c800c918a1c869c80aa739b3b8c38b12afc86.tgz - SUCCESS\n739c7a30-7a70-4d84-b409-eee540a2e490 2024-03-26T21:05:24+00:00 8S gs://builds-foyle-public/foyle-public/images/vscode-web-assets.82e00b39c4a3dd31c89d9732c59836ff2b658a8a.tgz - FAILURE\n53c35095-a9b2-424a-bccd-67005b055259 2024-03-26T18:35:34+00:00 46M26S gs://builds-foyle-public/foyle-public/images/vscode.493523cf6e84f3eecbde1d3371116282568f65e6.tgz - SUCCESS\n03120030-08a1-486a-bb17-6259953a3caf 2024-03-26T14:16:26+00:00 4M11S gs://builds-foyle-public/foyle-public/images/vscode.8c1bf4cb441a9c11d21c57fabc4493909a4ba79b.tgz - FAILURE\nd5b85047-9479-4956-adf7-0844c1903b97 2024-03-26T14:10:57+00:00 46S gs://builds-foyle-public/foyle-public/images/vscode.9e59b6edf3a32a5df991477b7be92352bf487b5a.tgz - FAILURE\n7fb2ab0e-1107-4624-83b4-e1ea5d9e6264 2024-03-26T14:03:51+00:00 11S gs://builds-foyle-public/foyle-public/images/vscode.34b4b67032d31b97f09b6f708e6a2b0fcd039d54.tgz - FAILURE\n2af6cb17-8c8a-4e21-b338-fd7e6a0857ae 2024-03-26T13:47:56+00:00 9S gs://builds-foyle-public/foyle-public/images/vscode.2cef2e862e3ee3e7520e8b44b3a93b0b903faa54.tgz - FAILURE\n4bf0e191-b1e6-4d35-b31f-51f8fd2f8439 2024-03-26T03:14:24+00:00 7S gs://builds-foyle-public/foyle-public/images/frontend.474aac87f6d426357ec353fd9724ea9092b9f7a9.tgz - FAILURE" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/data/eval/gcp/gke_list.foyle b/data/eval/gcp/gke_list.foyle new file mode 100644 index 00000000..3bf39664 --- /dev/null +++ b/data/eval/gcp/gke_list.foyle @@ -0,0 +1,32 @@ +{ + "blocks": [ + { + "kind": "MARKUP", + "language": "markdown", + "contents": "Describe the dev cluster?" + }, + { + "kind": "CODE", + "language": "bash", + "contents": "gcloud container clusters describe --region=us-west1 --project=foyle-dev dev", + "outputs": [ + { + "items": [ + { + "mime": "text/plain", + "textData": "exitCode: 1" + } + ] + }, + { + "items": [ + { + "mime": "text/plain", + "textData": "stderr:\nERROR: (gcloud.container.clusters.describe) ResponseError: code=404, message=Not found: projects/foyle-dev/locations/us-west1/clusters/dev.\nNo cluster named 'dev' in foyle-dev." + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/data/eval/gcp/images_list.foyle b/data/eval/gcp/images_list.foyle new file mode 100644 index 00000000..18762725 --- /dev/null +++ b/data/eval/gcp/images_list.foyle @@ -0,0 +1,40 @@ +{ + "blocks": [ + { + "kind": "MARKUP", + "language": "markdown", + "contents": "List images" + }, + { + "kind": "CODE", + "language": "bash", + "contents": "gcloud artifacts docker images list --sort-by=~create_time --include-tags us-west1-docker.pkg.dev/foyle-public/images", + "outputs": [ + { + "items": [ + { + "mime": "text/plain", + "textData": "exitCode: 0" + } + ] + }, + { + "items": [ + { + "mime": "text/plain", + "textData": "stdout:\nIMAGE DIGEST TAGS CREATE_TIME UPDATE_TIME\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:515259a40203b165782de239660f36386d82b8fd01c17db81e862082fa2a3748 09fe389c90b80da08b773145a3e35b7032f4e0b5, latest, v20240419T134114 2024-04-19T13:44:15 2024-04-19T13:44:17\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:165aed79adf0328e260e4e52858ae20057935dedf9e894c4df5577cb2a44b6a7 c46eb9ccb5dcee975232d549311ee3fb437908b7984ac2c969bbd1de0b76179d 2024-04-19T13:44:07 2024-04-19T13:44:07\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:a368f9e783d2624375579955a7a44b9096650c6a40918998fc1ce9923de63b91 9acfde50e8220637b4ad60701476d1b57ea5f832bf090c4909cae35c6476fad1 2024-04-19T13:42:58 2024-04-19T13:42:58\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:60364d64ad0bf0c36ac8fa9910c9788a6148e42691874ee0d4f33e135db9d6f2 e6ecbe4a76c0b3ecb7d0b289a0412cb2993c5ba6357e3ba8a8c7ba3d11d7926c 2024-04-19T13:42:51 2024-04-19T13:42:51\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:69c619f744c7e94ef90e212df04865f5eb9a26a1220829742fd20a419aa0cc07 0b15ea610db1d5c1bb566f06e69a502a5bcbca1d, v20240419T130716 2024-04-19T13:10:07 2024-04-19T13:10:09\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:1ba308cb99b4ac2cf28c3b6fcf5a4cfa743ba60db12ef706f2146f207f67104d 3ffcc4ede9b4f5c7e79458f29a43804b3b42d595897deb329f3877c8a5947cef 2024-04-19T13:09:52 2024-04-19T13:09:52\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:6a5de2dd46ca3c145925119bd60c48c68f188d2849a34ad72e66e5e619017a53 aff45c73f182a7b3984793df8e8c8d98a4346f52717546348393621846e0b7be 2024-04-19T13:08:56 2024-04-19T13:08:56\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:03a51c503805e4a627cb4f7f12ed2872c8dcbf4cb002f3a602e2aeefa11ce28f e1b84dcc6ee49b83a267fca2507a9c54fd2e5e892ca0b1629733788e62a6e312 2024-04-19T13:08:50 2024-04-19T13:08:50\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:09694e2b68b2843b91daf71f399d0e9cd88da2ccb2e77e72e60b8ffbab5ad777 da26d1b1d8b0fae2309ed90f72f9d9c609945d39, v20240417T152920 2024-04-17T15:32:35 2024-04-17T15:32:35\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:d1c6afedbb3f67f8cd5df3af614197dd0855ce5c5a00e2205af52fdbe5b46fd7 d527218deeffaad95d37bd4f2c842c145b6cdd66e8bb57a7988063afb2cf942c 2024-04-17T15:32:27 2024-04-17T15:32:27\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:ae0d998f3c04f6bd0be4a24ee0e59ed0edc9de3982c195dd727ef52d78f0742a ea7df8aea6740a84b29f0c2a11f45071d244d25964ce9c1aabe58e6e4e9ed8dd 2024-04-17T15:31:01 2024-04-17T15:31:01\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:77dacac70a054d3c97643c5e8188aba9f8b617e8287a7909b98509d65f4e5361 187ac177441eb4d68a0baa9e2be77cfbcdc14a438575d5fbff8b61546411e0e9 2024-04-17T15:30:54 2024-04-17T15:30:54\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:062903d5a8058076b0aa1b734c342a92ed70561cfa4b4ca3169ebef4554bf995 ce6affc61ec2a93e463e5787a799e42199f42213, v20240416T185228 2024-04-16T18:55:32 2024-04-16T18:55:34\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:2f26989e698c2e3adb2d56890821c89133d57d87f554321fb3706eb8b22311cd cb581c7911b8e0d910589eaaa8cac3a12e1efddc7d9be2583c6eb7671c9d7522 2024-04-16T18:55:21 2024-04-16T18:55:21\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:2a7d689ff5023a7e519c08e7df9365ad3745ac5b1d1d1f9415d406a2ee779d03 00c89700e58f20561ced8bcd7c79f11153f589fc627e84bdc75327ef79d1e5b2 2024-04-16T18:54:18 2024-04-16T18:54:18\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:6dc637674e0b1c07cf1c0441164e009639a6ea7e0ea3ca5dfe15e4376968c28f 3c6699facfa148c0ffa97669f3f2df33c38d13f6fdea700dac997df550c15afd 2024-04-16T18:54:04 2024-04-16T18:54:04\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:b9537d9bb802006ff89fa0fff080dd3ab40f9a342fbafbdb0d92415df0bfc47a c6d2e14d309b57fac3eef004197db6f41b90786f, v20240413T152709 2024-04-13T15:30:47 2024-04-13T15:30:49\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:fd0cd017a1bbc2b5f36a2c33418c08af3ff6ebd6c1439b93b6d875f0f5d3e1de cb292436f1b0d478e1fb41e32598027434d402405e7afbc3b6368b969da4af23 2024-04-13T15:30:35 2024-04-13T15:30:35\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:9ef45a054dd89153050e81ad8c5989da6c9a76f75f1169a24cd451cb8733ea24 9d1e05e29a98583bb0932c2c7cf07cdb4eac56f1d0c569a00153a0a99809a7ea 2024-04-13T15:29:06 2024-04-13T15:29:06\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:3397050cd50f110bd456fb1faac671ac9ed783d475db4675840205f9201a60e2 4a4fe781fb8b0115d52783dc1db7502bc9a7222c718ca0111ed9b0750577dbe7 2024-04-13T15:28:59 2024-04-13T15:28:59\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:3f831a9afc4fff9680c020153b837b9d2fcd9579b4e0c2619cb0d9c9592d29fa 51a604f692eb0803c7d2dcc00fac7709df1fa822, v20240412T171031 2024-04-12T17:13:08 2024-04-12T17:13:09\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:9731a75e5dc6fa1c0a52057669afe4b2b5e81008db5aee3993e47d37b18e4f0e d2ba6f1c0b6f1e5193a12f2bdbfa55cbffcadd4c3a858e91ecdb5ddb29e25e02 2024-04-12T17:12:49 2024-04-12T17:12:49\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:c2ecebbbcbe9e9837f3ba1a582f74ccdc750339bce346e1ccc1d3c0405242fe1 38809c271ed65b704f8499e5b41867d9f39585c03b426266b4718080361fdf44 2024-04-12T17:11:56 2024-04-12T17:11:56\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:77b8c3dbfb9be03b6f04e6323f23637451157bc57eb68f14414396bb3937c925 8bce2dbb8857b9b126586fc9d8a698888230a29ea8cdc5f3a26596ceca458c31 2024-04-12T17:11:51 2024-04-12T17:11:51\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:9110d11720827326da59481266dfa7ab3af5604b72da06e35d67d1b9211d8f56 8b27185ae391801b216274ce80946244dbf84fd6, v20240412T154413 2024-04-12T15:47:26 2024-04-12T15:47:29\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:1212b3f0c6169566cb2c4b20da5d3d0c1ad119e57b1367ee0980c73615e58ad9 fca7c3907ef75b03d2bc4cfa450a93081184ce08f0d3d0c03af50f6716b0f5a1 2024-04-12T15:47:16 2024-04-12T15:47:16\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:c850939b97ab7b4b1e68f1414f939f1f2ee2f97765c78dacd55c8fd89a374469 656ef16b975f255cadb2b9235806ddedde924c65c13545b2d21515f54b5a88dc 2024-04-12T15:46:01 2024-04-12T15:46:01\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:e77b3f5265133b4dac76b7ddd70973f0cc3799a2036f13df010ebb384d218673 8b4fd6521839f17f15fc3646fbd814ad60f8ae13a4ab4a13267c0d76f1dd171c 2024-04-12T15:45:53 2024-04-12T15:45:53\nus-west1-docker.pkg.dev/foyle-public/images/hydros/hydros/cache sha256:936ed408e9a4efb060f93e1e89cfb056f65bbbae72bb5f0965031f5ac7ea0c92 95b3049068dda92ae5e87381ada03ecd09a6262d9e7dc2713b7c75afc12ab8ce 2024-04-12T15:43:55 2024-04-12T15:43:55\nus-west1-docker.pkg.dev/foyle-public/images/hydros/hydros/cache sha256:f51341be82494155d51548bc98b63acfc31348f2d3fc89061137d76391cfa99d 3ad158a8e366298154c3fa822ee033277d3149aee2b2ce06b63300bef0f553d3 2024-04-12T15:39:13 2024-04-12T15:39:13\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:d901cf9dd6a62a3cee0e32ccc7f8547212097eb7d3f9f020141d230a0963b669 e8901ec0fb1dd84a26b4d4976de16319fad10cb1, v20240411T130448 2024-04-11T13:07:58 2024-04-11T13:08:00\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:bba9e8f6731f8c3063297dd1cc88219f01cc08f22e103432695d0c8815270334 d4866d839c47c743f5e19fbb97135072be0658a27816a5cb2a497c579b78d199 2024-04-11T13:07:48 2024-04-11T13:07:48\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:c37263165ac9ae9857e3348c391e38d522a48accb24fb9698f5b75909e3ff01f 3d2c0ec298cdac6977caf2de9b8e90bd2ce431015b36e13b0663617d9f323156 2024-04-11T13:06:35 2024-04-11T13:06:35\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:32f59ed4da83eae76b45513016c98dfab10e9fdbd93feb975dcafad61f5af2c1 bb370bed9f4c2eb04e2d6e82ab0d1566505f1ed05484d193aa4829b958610ca4 2024-04-11T13:06:27 2024-04-11T13:06:27\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:761ae64616846bb178290edfadecbc45121fbf839ffd62c1daba1d8a7ca3c1d0 d2641e55fea10d79305aa25f6fa2fd19ae554b7f, v20240411T084121 2024-04-11T08:45:45 2024-04-11T08:45:46\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:0013468ebf3338a8be595d316f8d431bf9bc78de44cabdbdccf36b2819d8f760 86a4b7e4d3fb26c778ed5ceb6e071ac73649e608a3395069aed8f7c4bdc823af 2024-04-11T08:45:33 2024-04-11T08:45:33\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:f32765946b4ed800c113448bf5d460c65b86bac6d276ae6f4eeee10338f44795 d6d142a6994f513f6ca6fb1ef7e7dc7a801fa2df38e4fbda3073723e805cfced 2024-04-11T08:43:48 2024-04-11T08:43:48\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:78ea7ed44c7035517cf4f676abce1bb949fca2953e5487b02664f98f9dc81585 8f5e8255be4cbba7f4f74911f3b289999dbafd33e4b897aaac712a7c7d3116dc 2024-04-11T08:43:38 2024-04-11T08:43:38\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:1a75fa7b04b31076889f9aae65dbf4739bfd8753015f9c1c04d2f0deedd77e27 dd52c555034fe0e16efaed64cce276d5ebc0f2c2, v20240411T083005 2024-04-11T08:35:36 2024-04-11T08:35:38\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:f006b32f52d9e4d6232b208f43cf9170daf2820d1dc67ea28e598804adffd4e6 e74581a595492ad0164fa2c9a999c66b709d382b24259f60bde33836e8470160 2024-04-11T08:35:25 2024-04-11T08:35:25\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:a41077867745e67c4fff28e96aec36a1f92b6aaa97c3907fcc9bba81074b1d4d 1b4e357081a8bf1e4b893bb01b50364215e04f1635ffa2bf3091f3c164cb7421 2024-04-11T08:32:09 2024-04-11T08:32:09\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:39164517bd46cdf8685e0265551bda160a694ae034ddbba3fed660f3e195c7c5 af74231e81d7c23025a9c10f1263ab207f4aa7f37d07ea9c2b3a38f72b41e018 2024-04-11T08:32:00 2024-04-11T08:32:00\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:a6a885b9ccb7b9fd35b4a69caabfe8bd1ff7a48b7b9cd61b32e21e6f81e04241 d2415d1fc8c6647cc6f8ecb3191f31b0a0fd8c7f, v20240411T081633 2024-04-11T08:19:43 2024-04-11T08:19:45\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:297ecd11611629ff7a7758afc65c5ef33aa55d1682372013fe77cd31ee871a6c ca93548a576fabbbf6d6c043ee53852570832013e2e5c01ee697a35e9170d7ea 2024-04-11T08:19:34 2024-04-11T08:19:34\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:517dac5170eef841e6b36f2aef489eb7488f08e9eec764ee326b94a44e7a6af3 3e76209aa560f628f1fa067ab0b3af0efa2af25e269c639c143e3e52ea21f618 2024-04-11T08:18:37 2024-04-11T08:18:37\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:b9d77faf04df58363f7046b0fd1f995349393d01cd877a69af1ee670bd8c27c7 3a5b899c97046a121e901447d711dba4d223ba8749101ebcc498a16757657f7e 2024-04-11T08:18:23 2024-04-11T08:18:23\nus-west1-docker.pkg.dev/foyle-public/images/hydros/hydros sha256:1f0e08cef8bf7507eb5b22a8e8c773f61ebb270c1562128e036d61196abe436f f97f18c5edb20de30bc3fa6ba25835d9afed06c9, latest, v20240408T180805 2024-04-08T18:12:29 2024-04-08T18:12:30\nus-west1-docker.pkg.dev/foyle-public/images/hydros/hydros/cache sha256:6e455c846343df4bf3e4afc207c3015074e370ae15e6934c33fd7d7285f3b2ed fac4dc7040dd4306a3bc4d60f59ae55f668350fde56f3d45e0867cbb21535720 2024-04-08T18:12:28 2024-04-08T18:12:28\nus-west1-docker.pkg.dev/foyle-public/images/hydros/hydros/cache sha256:38f38f52a614d78d714cb8be926cc856db315b77f9ecef83e213da35d813a67b 34d69cbb1c47e1cee326b009deba2646f0cc42e48038ff50aed51376615cdeed 2024-04-08T18:12:21 2024-04-08T18:12:21\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext sha256:be062edeb19af0a4977af2e12662226ba8fdfd146a0eea7414adfa6e9dd9ed07 6ba56ae46fc150d3a39a2d592183f83cc8c67d44, v20240408T160146 2024-04-08T16:04:25 2024-04-08T16:04:27\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:8d784f5cb939aa4135b1e127a873f0652fdff9796ea80dad25371d8838e1bbb8 39a18189b5dba4f7d46dd271015872db6f3376dca6141e8556ba6c3f339b89e0 2024-04-08T16:04:17 2024-04-08T16:04:17\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:24ea8f7ffc1deada58d8175aaa8055fa443511c0b084b1fda011fab409a80562 aa7ad74742526b5c91bcaa362bd4c63620d47d68c5f60265bed23ad64876575a 2024-04-08T16:03:22 2024-04-08T16:03:22\nus-west1-docker.pkg.dev/foyle-public/images/foyle-vscode-ext/cache sha256:7de98027a39a22aa74da1671dffda7b528c23a75b6a2dd32956a930fcf55993e 9b053af0e22ce7d50935b16e4fa8ca349d5578ab0237ff1f7c30413fa855b6f1 2024-04-08T16:03:16 2024-04-08T16:03:16\nus-west1-docker.pkg.dev/foyle-public/images/vscode-web-assets sha256:2dd61b117721c7273b11b6dbe6b4057c9c98da2b1fe46db82c16be9281625390 107d350496ec50984207cdccdab73602dd32de11, latest, v20240403T185418 2024-04-03T19:03:02 2024-04-03T19:03:04\nus-west1-docker.pkg.dev/foyle-public/images/vscode-web-assets/cache sha256:5fa278ffbb51fa3fb6302756143140c0c53dd12d1741cacc1b0d3d192fc4ae5a e9a079c688a6f6e17d01798fa31b78f4833a8507413260606ef06b57632c71ff 2024-04-03T19:02:08 2024-04-03T19:02:08\nus-west1-docker.pkg.dev/foyle-public/images/vscode sha256:51c8d93ec053cb033c6c967ad4f901274d9fc65305fd7f1d58c2e8a05348ace9 0a44acc96351fd10d0373d487f5d558397f0ac39, latest, v20240403T172126 2024-04-03T18:08:23 2024-04-03T18:08:28\nus-west1-docker.pkg.dev/foyle-public/images/vscode/cache sha256:574a30f31f163f82fbd710c0dcb3f2974409d6de8ee19fafe51e3fb3bf49b828 833262f8b8832be34ca6f9d101a34345ef33b35282525f501530d5167e6c9641 2024-04-03T18:08:05 2024-04-03T18:08:05\nus-west1-docker.pkg.dev/foyle-public/images/vscode/cache sha256:84cb2b8b71234d217219b96e1016cb968b948f4f22f3b4eaccdebb9d8a41d1a4 aa7ad74742526b5c91bcaa362bd4c63620d47d68c5f60265bed23ad64876575a 2024-04-03T17:23:15 2024-04-03T17:23:15\nus-west1-docker.pkg.dev/foyle-public/images/vscode/cache sha256:1c7f9ed8be9ec4ba44a92487655020583e2f197f2391137ede9579d2d8fbd176 9b053af0e22ce7d50935b16e4fa8ca349d5578ab0237ff1f7c30413fa855b6f1 2024-04-03T17:23:09 2024-04-03T17:23:09" + } + ] + }, + { + "items": [ + { + "mime": "text/plain", + "textData": "stderr:\nListing items under project foyle-public, location us-west1, repository images.\n" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/data/eval/iac/pulumi_up.foyle b/data/eval/iac/pulumi_up.foyle new file mode 100644 index 00000000..04ef9f9f --- /dev/null +++ b/data/eval/iac/pulumi_up.foyle @@ -0,0 +1,32 @@ +{ + "blocks": [ + { + "kind": "CODE", + "language": "bash", + "contents": "Sync the dev infra" + }, + { + "kind": "CODE", + "language": "bash", + "contents": "pulumi up -C /Users/jlewi/git_foyle/iac/dev --non-interactive --yes\n", + "outputs": [ + { + "items": [ + { + "mime": "text/plain", + "textData": "exitCode: 0" + } + ] + }, + { + "items": [ + { + "mime": "text/plain", + "textData": "stdout:\nPreviewing update (dev)\n\nView Live: https://app.pulumi.com/jlewi/foyle-dev/dev/previews/e0d6037d-1bf5-4fbf-92d7-2b1d9f574053\n\n\n@ Previewing update..................\n pulumi:pulumi:Stack foyle-dev-dev running \n@ Previewing update.....\n pulumi:pulumi:Stack foyle-dev-dev running warning: unable to detect a global setting for GCP Project;\n@ Previewing update....\n gcp:organizations:Project project \n gcp:projects:Service cloudbuild.googleapis.com \n gcp:projects:Service artifactregistry.googleapis.com \n gcp:projects:Service container.googleapis.com \n gcp:projects:Service storage.googleapis.com \n gcp:projects:Service secretmanager.googleapis.com \n gcp:container:Cluster dev \n pulumi:pulumi:Stack foyle-dev-dev 1 warning\nDiagnostics:\n pulumi:pulumi:Stack (foyle-dev-dev):\n warning: unable to detect a global setting for GCP Project;\n Pulumi will rely on per-resource settings for this operation.\n Set the GCP Project by using:\n \t`pulumi config set gcp:project `\n\nResources:\n 8 unchanged\n\nUpdating (dev)\n\nView Live: https://app.pulumi.com/jlewi/foyle-dev/dev/updates/7\n\n\n@ Updating.....\n pulumi:pulumi:Stack foyle-dev-dev running \n@ Updating....\n pulumi:pulumi:Stack foyle-dev-dev running warning: unable to detect a global setting for GCP Project;\n gcp:organizations:Project project \n gcp:projects:Service storage.googleapis.com \n gcp:projects:Service secretmanager.googleapis.com \n gcp:projects:Service artifactregistry.googleapis.com \n gcp:projects:Service container.googleapis.com \n gcp:projects:Service cloudbuild.googleapis.com \n gcp:container:Cluster dev \n pulumi:pulumi:Stack foyle-dev-dev 1 warning\nDiagnostics:\n pulumi:pulumi:Stack (foyle-dev-dev):\n warning: unable to detect a global setting for GCP Project;\n Pulumi will rely on per-resource settings for this operation.\n Set the GCP Project by using:\n \t`pulumi config set gcp:project `\n\nResources:\n 8 unchanged\n\nDuration: 4s\n" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/content/en/docs/evaluation/_index.md b/docs/content/en/docs/evaluation/_index.md new file mode 100644 index 00000000..75fdbd50 --- /dev/null +++ b/docs/content/en/docs/evaluation/_index.md @@ -0,0 +1,71 @@ +--- +title: "Evaluation" +description: "How to evaluate Foyle's performance" +weight: 3 +--- + +## What You'll Learn + +How to quantitatively evaluate Foyle's AI quality + +## Building an Evaluation Dataset + +To evaluate Foyle's performance, you need to build an evaluation dataset. This dataset should contain a set of examples +where you know the correct answer. Each example should be a `.foyle` file that ends with a code block containing a +command. The last command represents the command you'd like Foyle to predict when the input is the rest of the document. + +Its usually a good idea to start by hand crafting a few examples based on knowledge of your infrastructure +and the kinds of questions you expect to ask. A good place to start is by creating examples that test basic knowledge +of how your infrastructure is configured e.g. + +1. Describe the cluster used for dev/prod? +1. List the service accounts used for dev +1. List image repositories + +A common theme here is to think about what information a new hire would need to know to get started. Even a new hire +experienced in the technologies you use would need to be informed about your specific configuration; e.g. +what AWS Account/GCP project to use. + +For the next set of examples, you can start to think about conventions your teams use. For example, if you have +specific tagging conventions for images or labels you should come up with examples that test whether Foyle has +learned those conventions + +1. Describe the prod image for service foo? +1. Fetch the logs for the service bar? +1. List the deployment for application foo? + +A good way to test whether Foyle has learned these conventions is to create examples for non-existent services, images, +etc... This makes it likely users haven't actually issued those requests causing Foyle to memorize the answers. + +## Evaluating Foyle + +Once you have an evaluation dataset define an Experiment resource in a YAML file. Here's an example + +```yaml +kind: Experiment +metadata: + name: "learning" +spec: + evalDir: /Users/jlewi/git_foyle/data/eval + dbDir: /Users/jlewi/foyle_experiments/learning + sheetID: "1O0thD-p9DBF4G_shGMniivBB3pdaYifgSzWXBxELKqE" + sheetName: "Results" + Agent: + model: gpt-3.5-turbo-0125 + rag: + enabled: true + maxResults: 3 +``` + +You can then run evaluation by running + +```bash +foyle apply /path/to/experiment.yaml +``` + +This will run evaluation. If evaluation succeeds it will create a Google Sheet with the results. Each row in the +Google Sheet contains the results for one example in the evaluation dataset. The distance column measures +the distance between the predicted command and the actual command. The distance is the number of arguments +that would need to be changed to turn the predicted command into the actual command. If the actual and expected +are the same the distance will be zero. The maximum value of the edit distance is the maximum of the number of arguments +in the actual and expected commands. diff --git a/experiments/rag.yaml b/experiments/rag.yaml new file mode 100644 index 00000000..a5d36380 --- /dev/null +++ b/experiments/rag.yaml @@ -0,0 +1,15 @@ +# A basic experiment to test learning +kind: Experiment +apiVersion: foyle.io/v1alpha1 +metadata: + name: "learning" +spec: + evalDir: /Users/jlewi/git_foyle/data/eval + dbDir: /Users/jlewi/foyle_experiments/learning + sheetID: "1O0thD-p9DBF4G_shGMniivBB3pdaYifgSzWXBxELKqE" + sheetName: "Learning-Results" + agent: + model: gpt-3.5-turbo-0125 + rag: + enabled: true + maxResults: 3 diff --git a/frontend/foyle/src/gen/foyle/v1alpha1/eval_pb.ts b/frontend/foyle/src/gen/foyle/v1alpha1/eval_pb.ts new file mode 100644 index 00000000..75c4a6cf --- /dev/null +++ b/frontend/foyle/src/gen/foyle/v1alpha1/eval_pb.ts @@ -0,0 +1,115 @@ +// @generated by protoc-gen-es v1.8.0 with parameter "target=ts,import_extension=none" +// @generated from file foyle/v1alpha1/eval.proto (syntax proto3) +/* eslint-disable */ +// @ts-nocheck + +import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf"; +import { Message, proto3 } from "@bufbuild/protobuf"; +import { Example } from "./trainer_pb"; +import { Block } from "./doc_pb"; + +/** + * @generated from enum EvalResultStatus + */ +export enum EvalResultStatus { + /** + * @generated from enum value: UNKNOWN_EVAL_RESULT_STATUS = 0; + */ + UNKNOWN_EVAL_RESULT_STATUS = 0, + + /** + * @generated from enum value: DONE = 1; + */ + DONE = 1, + + /** + * @generated from enum value: ERROR = 2; + */ + ERROR = 2, +} +// Retrieve enum metadata with: proto3.getEnumType(EvalResultStatus) +proto3.util.setEnumType(EvalResultStatus, "EvalResultStatus", [ + { no: 0, name: "UNKNOWN_EVAL_RESULT_STATUS" }, + { no: 1, name: "DONE" }, + { no: 2, name: "ERROR" }, +]); + +/** + * EvalResult represents an evaluation result + * + * @generated from message EvalResult + */ +export class EvalResult extends Message { + /** + * Example is the answer and expected result + * + * @generated from field: Example example = 1; + */ + example?: Example; + + /** + * example_file is the file containing the example + * + * @generated from field: string example_file = 2; + */ + exampleFile = ""; + + /** + * Actual response + * + * @generated from field: repeated Block actual = 3; + */ + actual: Block[] = []; + + /** + * The distance between the actual and expected response + * + * @generated from field: int32 distance = 4; + */ + distance = 0; + + /** + * @generated from field: string error = 5; + */ + error = ""; + + /** + * Status of the evaluation + * + * @generated from field: EvalResultStatus status = 6; + */ + status = EvalResultStatus.UNKNOWN_EVAL_RESULT_STATUS; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "EvalResult"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "example", kind: "message", T: Example }, + { no: 2, name: "example_file", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 3, name: "actual", kind: "message", T: Block, repeated: true }, + { no: 4, name: "distance", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 5, name: "error", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 6, name: "status", kind: "enum", T: proto3.getEnumType(EvalResultStatus) }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): EvalResult { + return new EvalResult().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): EvalResult { + return new EvalResult().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): EvalResult { + return new EvalResult().fromJsonString(jsonString, options); + } + + static equals(a: EvalResult | PlainMessage | undefined, b: EvalResult | PlainMessage | undefined): boolean { + return proto3.util.equals(EvalResult, a, b); + } +} + diff --git a/iac/dev/main.go b/iac/dev/main.go index 1b13bd65..d6c4855e 100644 --- a/iac/dev/main.go +++ b/iac/dev/main.go @@ -4,6 +4,7 @@ import ( "github.com/pulumi/pulumi-gcp/sdk/v7/go/gcp/container" "github.com/pulumi/pulumi-gcp/sdk/v7/go/gcp/organizations" "github.com/pulumi/pulumi-gcp/sdk/v7/go/gcp/projects" + "github.com/pulumi/pulumi-gcp/sdk/v7/go/gcp/serviceaccount" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -33,6 +34,7 @@ func main() { "artifactregistry.googleapis.com", "cloudbuild.googleapis.com", "container.googleapis.com", + "sheets.googleapis.com", "storage.googleapis.com", "secretmanager.googleapis.com", } @@ -64,6 +66,24 @@ func main() { if err != nil { return err } + account, err := serviceaccount.NewAccount(ctx, "developer", &serviceaccount.AccountArgs{ + AccountId: pulumi.String("developer"), + DisplayName: pulumi.String("Developer Service Account"), + Project: p.ProjectId, + }) + if err != nil { + return err + } + + // Create IAM binding for the service account + _, err = serviceaccount.NewIAMBinding(ctx, "serviceAccountIAMBinding", &serviceaccount.IAMBindingArgs{ + ServiceAccountId: account.Name, + Role: pulumi.String("roles/iam.serviceAccountTokenCreator"), + Members: pulumi.StringArray{pulumi.String("user:jeremy@lewi.us")}, + }) + if err != nil { + return err + } return nil }) diff --git a/protos/foyle/v1alpha1/eval.proto b/protos/foyle/v1alpha1/eval.proto new file mode 100644 index 00000000..3bfa62c5 --- /dev/null +++ b/protos/foyle/v1alpha1/eval.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +import "foyle/v1alpha1/doc.proto"; +import "foyle/v1alpha1/trainer.proto"; + +import "google/protobuf/struct.proto"; + +option go_package = "github.com/jlewi/foyle/protos/go/v1alpha1"; + +enum EvalResultStatus { + UNKNOWN_EVAL_RESULT_STATUS = 0; + DONE = 1; + ERROR = 2; +} + +// EvalResult represents an evaluation result +message EvalResult { + // Example is the answer and expected result + Example example = 1; + + // example_file is the file containing the example + string example_file = 2; + + // Actual response + repeated Block actual = 3; + + // The distance between the actual and expected response + int32 distance = 4; + + string error = 5; + + // Status of the evaluation + EvalResultStatus status = 6; +} diff --git a/protos/go/foyle/v1alpha1/eval.pb.go b/protos/go/foyle/v1alpha1/eval.pb.go new file mode 100644 index 00000000..4ef99249 --- /dev/null +++ b/protos/go/foyle/v1alpha1/eval.pb.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc (unknown) +// source: foyle/v1alpha1/eval.proto + +package v1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/known/structpb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EvalResultStatus int32 + +const ( + EvalResultStatus_UNKNOWN_EVAL_RESULT_STATUS EvalResultStatus = 0 + EvalResultStatus_DONE EvalResultStatus = 1 + EvalResultStatus_ERROR EvalResultStatus = 2 +) + +// Enum value maps for EvalResultStatus. +var ( + EvalResultStatus_name = map[int32]string{ + 0: "UNKNOWN_EVAL_RESULT_STATUS", + 1: "DONE", + 2: "ERROR", + } + EvalResultStatus_value = map[string]int32{ + "UNKNOWN_EVAL_RESULT_STATUS": 0, + "DONE": 1, + "ERROR": 2, + } +) + +func (x EvalResultStatus) Enum() *EvalResultStatus { + p := new(EvalResultStatus) + *p = x + return p +} + +func (x EvalResultStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EvalResultStatus) Descriptor() protoreflect.EnumDescriptor { + return file_foyle_v1alpha1_eval_proto_enumTypes[0].Descriptor() +} + +func (EvalResultStatus) Type() protoreflect.EnumType { + return &file_foyle_v1alpha1_eval_proto_enumTypes[0] +} + +func (x EvalResultStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EvalResultStatus.Descriptor instead. +func (EvalResultStatus) EnumDescriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{0} +} + +// EvalResult represents an evaluation result +type EvalResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Example is the answer and expected result + Example *Example `protobuf:"bytes,1,opt,name=example,proto3" json:"example,omitempty"` + // example_file is the file containing the example + ExampleFile string `protobuf:"bytes,2,opt,name=example_file,json=exampleFile,proto3" json:"example_file,omitempty"` + // Actual response + Actual []*Block `protobuf:"bytes,3,rep,name=actual,proto3" json:"actual,omitempty"` + // The distance between the actual and expected response + Distance int32 `protobuf:"varint,4,opt,name=distance,proto3" json:"distance,omitempty"` + Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"` + // Status of the evaluation + Status EvalResultStatus `protobuf:"varint,6,opt,name=status,proto3,enum=EvalResultStatus" json:"status,omitempty"` +} + +func (x *EvalResult) Reset() { + *x = EvalResult{} + if protoimpl.UnsafeEnabled { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EvalResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EvalResult) ProtoMessage() {} + +func (x *EvalResult) ProtoReflect() protoreflect.Message { + mi := &file_foyle_v1alpha1_eval_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EvalResult.ProtoReflect.Descriptor instead. +func (*EvalResult) Descriptor() ([]byte, []int) { + return file_foyle_v1alpha1_eval_proto_rawDescGZIP(), []int{0} +} + +func (x *EvalResult) GetExample() *Example { + if x != nil { + return x.Example + } + return nil +} + +func (x *EvalResult) GetExampleFile() string { + if x != nil { + return x.ExampleFile + } + return "" +} + +func (x *EvalResult) GetActual() []*Block { + if x != nil { + return x.Actual + } + return nil +} + +func (x *EvalResult) GetDistance() int32 { + if x != nil { + return x.Distance + } + return 0 +} + +func (x *EvalResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *EvalResult) GetStatus() EvalResultStatus { + if x != nil { + return x.Status + } + return EvalResultStatus_UNKNOWN_EVAL_RESULT_STATUS +} + +var File_foyle_v1alpha1_eval_proto protoreflect.FileDescriptor + +var file_foyle_v1alpha1_eval_proto_rawDesc = []byte{ + 0x0a, 0x19, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2f, 0x65, 0x76, 0x61, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x66, 0x6f, 0x79, + 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x64, 0x6f, 0x63, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0xd0, 0x01, 0x0a, 0x0a, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x22, 0x0a, 0x07, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x08, 0x2e, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x07, 0x65, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x78, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x75, 0x61, + 0x6c, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x06, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, + 0x06, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, + 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, + 0x6e, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, 0x2e, 0x45, 0x76, 0x61, 0x6c, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x2a, 0x47, 0x0a, 0x10, 0x45, 0x76, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x5f, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x52, 0x45, 0x53, 0x55, 0x4c, 0x54, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, + 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x42, 0x2b, 0x5a, + 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x6c, 0x65, 0x77, + 0x69, 0x2f, 0x66, 0x6f, 0x79, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x67, + 0x6f, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_foyle_v1alpha1_eval_proto_rawDescOnce sync.Once + file_foyle_v1alpha1_eval_proto_rawDescData = file_foyle_v1alpha1_eval_proto_rawDesc +) + +func file_foyle_v1alpha1_eval_proto_rawDescGZIP() []byte { + file_foyle_v1alpha1_eval_proto_rawDescOnce.Do(func() { + file_foyle_v1alpha1_eval_proto_rawDescData = protoimpl.X.CompressGZIP(file_foyle_v1alpha1_eval_proto_rawDescData) + }) + return file_foyle_v1alpha1_eval_proto_rawDescData +} + +var file_foyle_v1alpha1_eval_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_foyle_v1alpha1_eval_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_foyle_v1alpha1_eval_proto_goTypes = []interface{}{ + (EvalResultStatus)(0), // 0: EvalResultStatus + (*EvalResult)(nil), // 1: EvalResult + (*Example)(nil), // 2: Example + (*Block)(nil), // 3: Block +} +var file_foyle_v1alpha1_eval_proto_depIdxs = []int32{ + 2, // 0: EvalResult.example:type_name -> Example + 3, // 1: EvalResult.actual:type_name -> Block + 0, // 2: EvalResult.status:type_name -> EvalResultStatus + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_foyle_v1alpha1_eval_proto_init() } +func file_foyle_v1alpha1_eval_proto_init() { + if File_foyle_v1alpha1_eval_proto != nil { + return + } + file_foyle_v1alpha1_doc_proto_init() + file_foyle_v1alpha1_trainer_proto_init() + if !protoimpl.UnsafeEnabled { + file_foyle_v1alpha1_eval_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EvalResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_foyle_v1alpha1_eval_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_foyle_v1alpha1_eval_proto_goTypes, + DependencyIndexes: file_foyle_v1alpha1_eval_proto_depIdxs, + EnumInfos: file_foyle_v1alpha1_eval_proto_enumTypes, + MessageInfos: file_foyle_v1alpha1_eval_proto_msgTypes, + }.Build() + File_foyle_v1alpha1_eval_proto = out.File + file_foyle_v1alpha1_eval_proto_rawDesc = nil + file_foyle_v1alpha1_eval_proto_goTypes = nil + file_foyle_v1alpha1_eval_proto_depIdxs = nil +} diff --git a/protos/go/foyle/v1alpha1/eval.zap.go b/protos/go/foyle/v1alpha1/eval.zap.go new file mode 100644 index 00000000..9dd85753 --- /dev/null +++ b/protos/go/foyle/v1alpha1/eval.zap.go @@ -0,0 +1,62 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: foyle/v1alpha1/eval.proto + +package v1alpha1 + +import ( + fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" + _ "google.golang.org/protobuf/types/known/structpb" + go_uber_org_zap_zapcore "go.uber.org/zap/zapcore" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +func (m *EvalResult) MarshalLogObject(enc go_uber_org_zap_zapcore.ObjectEncoder) error { + var keyName string + _ = keyName + + if m == nil { + return nil + } + + keyName = "example" // field example = 1 + if m.Example != nil { + var vv interface{} = m.Example + if marshaler, ok := vv.(go_uber_org_zap_zapcore.ObjectMarshaler); ok { + enc.AddObject(keyName, marshaler) + } + } + + keyName = "example_file" // field example_file = 2 + enc.AddString(keyName, m.ExampleFile) + + keyName = "actual" // field actual = 3 + enc.AddArray(keyName, go_uber_org_zap_zapcore.ArrayMarshalerFunc(func(aenc go_uber_org_zap_zapcore.ArrayEncoder) error { + for _, rv := range m.Actual { + _ = rv + if rv != nil { + var vv interface{} = rv + if marshaler, ok := vv.(go_uber_org_zap_zapcore.ObjectMarshaler); ok { + aenc.AppendObject(marshaler) + } + } + } + return nil + })) + + keyName = "distance" // field distance = 4 + enc.AddInt32(keyName, m.Distance) + + keyName = "error" // field error = 5 + enc.AddString(keyName, m.Error) + + keyName = "status" // field status = 6 + enc.AddString(keyName, m.Status.String()) + + return nil +} diff --git a/tech_notes/tn002_learning.md b/tech_notes/tn002_learning.md index 6b867719..0bc43b15 100644 --- a/tech_notes/tn002_learning.md +++ b/tech_notes/tn002_learning.md @@ -2,7 +2,7 @@ * **Author**: Jeremy Lewi * **Last Updated**: 2024-04-23 -* **Status**: Being Drafted +* **Status**: Publish ## Objective diff --git a/tech_notes/tn003_learning_eval.md b/tech_notes/tn003_learning_eval.md new file mode 100644 index 00000000..d0c91e4b --- /dev/null +++ b/tech_notes/tn003_learning_eval.md @@ -0,0 +1,198 @@ +--- +runme: + id: 01HWP5TGX8T7WR6FDJNH0T5MAY + version: v3 +--- + +# Learning Evaluation + +* **Author**: Jeremy Lewi +* **Last Updated**: 2024-04-27 +* **Status**: Being Drafted + +## Objective + +Measure the efficacy of the learning system. + +## TL;DR + +The key hypothesis of Foyle is that by using implicit human feedback, we can create an AI that automatically learns +about your infrastructure. In [TN002_Learning](tn002_learning) and [PR#83](https://github.com/jlewi/foyle/pull/83) +we implemented a very simple learning mechanism based on query dependent few shot prompting. The next step is to +quantiatively evaluate the effectiveness of this system. We'd like to construct an evaluation data set that consists of +queries whose answers depend on private knowledge of your infrastructure. We'd like to compare the performance of +Foyle before and after learning from human feedback. We propose to achieve this as follows + +1. Manually construct an evaluation dataset consisting of queries whose answers depend on private knowledge of your + infrastructure $\{(q_0, r_0), .... (q_n, r_n)\}$. +1. For each query $q_i$ in the evaluation dataset, we will generate a command $r'_i$ using the Agent. +1. Compute an evaluation score using a metric similar to the edit distance + + $$ + S = \sum_{i=0}^n D(r_i, r'_i) + $$ + +1. Compare the scores before and after learning from human feedback. + +## Learning: What Do We Want To Learn + +In the context of DevOps there are different types of learning we can test for. The simplest things are facts. +These are discrete pieces of information that can't be derived from other information. For example: + +Query: "Which cluster is used for development? + +Response: "gcloud container clusters describe --region=us-west1 --project=foyle-dev dev" + +The fact that your organization is using a GKE cluster in the us-west1 region in project foyle-dev for development +(and not an EKS cluster in us-west1) is something a teammate needs to tell you. In the context of our Agent, what +we want the agent to learn is alternative ways to ask the same question. For example, +"Show me the cluster where dev workloads run" should return the same response. + +As a team's infrastructure grows, they often develop conventions for doing things in a consistent way. A simple +example of this is naming conventions. For example, as the number of docker images grows, most organizations develop +conventions around image tagging; e.g. using `live` or `prod` to indicate the production version of an image. +In the context of Foyle, we'd like to evaluate the efficacy with which Foyle learns these conventions from examples. +For example, our training set might include the example + +Query: "Show me the latest version of the production image for hydros" + +Response: "gcloud artifacts docker images describe us-west1-docker.pkg.dev/foyle-public/images/hydros/hydros:prod" + +Our evaluation set might include the example + +Query: "Describe the production image for foyle" + +Response: "gcloud artifacts docker images describe us-west1-docker.pkg.dev/foyle-public/images/foyle/foyle:prod" + +Notably, unlike the previous example, in this case both the query and response are different. In order for the agent +to generalize to images not in the training set, it needs to learn the convention that we use to tag images. + +### Multi-step + +We'd eventually like to be able to support multi-step queries. For example, a user might ask +"Show me the logs for the most recent image build for foyle". This requires two steps + +1. Listing the images to find the sha of the most recent image +2. Querying for the logs of the image build with that sha + +We leave this for future work. + +## Building an Evaluation Set + +We will start by hand crafting our evaluation set. + +Our evaluation set will consist of a set of `.foyle documents` checked into source control in the [data](../data) +directory. The evaluation data set is specific to an organization and its infrastructure. We will use the infrastructure +of Foyle's development team as the basis for our evaluation set. + +To test Foyle's ability to learn conventions we will include examples that exercise conventions for non-existent +infrastructure (e.g. images, deployments, etc...). Since they don't exist, a user wouldn't actually query for them +so they shouldn't appear in our logs. + +We can automatically classify each examples in the evaluation set as either a memorization or generalization example +based on whether the response matches one of the responses in the training set. We can use the distance metric proposed +below to measure the similarity between the generated command and the actual command. + +## Evaluating correctness + +We can evaluate the correctness of the Agent by comparing the generated commands with the actual commands. +We can use an error metric similar to edit distance. Rather than comparing individual characters, we will compare arguments as a whole. +First we will divide a command into positional and named arguments. For this purpose the command +itself is the first positional argument. For the positional we compute the edit distance but looking at the entirety +of the positional argument. For the named arguments we match arguments by name and count the number of incorrect, +missing, and extra arguments. We can denote this as follows. Let $a$ and $b$ be the sequence of positionsal +arguments for two different commands $r_a$ and $r_b$ that we wish to compare. + +$$ +a = {a_0, ..., a_{m}} +$$ + +$$ +b = {b_0, ..., b_{n}} +$$ + +Then we can define the edit distance between the positional arguments as + +$$ +distance = D_p(m,n) +$$ + +$$ +D_p(i,0) = \sum_{k=0}^{i} w_{del}(a_k) +$$ + +$$ +D_p(0,j) = \sum_{k=0}^{j} w_{ins}(b_k) +$$ + +$$ +D_p(i, j) = \begin{cases} +D_p(i-1, j-1) & \text{if } a_i = b_j \\ +min \begin{cases} +D_p(i-1, j) + w_{del}(a_i) \\ +D_p(i, j-1) + w_{ins}(b_j) \\ +D_p(i-1, j-1) + w_{sub}(a_i, b_j) \\ +\end{cases} & \text{if } a[i] \neq b[j] +\end{cases} +$$ + +Here $w_{del}$, $w_{ins}$, and $w_{sub}$ are weights for deletion, insertion, and substitution respectively. If $w_{del} = w_{ins}$ then the distance is symetric $D(r_a, r_b) = D(r_b, r_a)$. + +We can treat named arguments as two dictionaries `c` and `d`. We can define the edit distance between the named arguments as follows + +$$ +K = \text{keys}(c) \cup \text{keys}(d) +$$ + +$$ +D_n = \sum_{k \in K} f(k) +$$ + +$$ +f(k) = \begin{cases} +w_{del}(c[k]) & \text{if } k \notin \text{keys}(d) \\ +w_{ins}(d[k]) & \text{if } k \notin \text{keys}(c) \\ +w_{sub}(c[k], d[k]) & \text{if } k \in \text{keys}(c), k \in \text{keys}(d), c[k] \neq d[k] \\ +0 & \text{otherwise} \\ +\end{cases} +$$ + +This definition doesn't properly account for the fact that named arguments often have a long and short version. We ignore this for now and accept that if one command uses the long version and the other uses the short version, we will count these as errors. In the future, we could try building a dictionary of known commands and normalizing the arguments to a standard form. + +This definition also doesn't account for the fact that many CLIs have subcommands and certain named arguments must appear before or after certain subcommands. The definition above +would treat a named argument as a match as long it appears anywhere in the command. + +The total distance between two commands is then + +$$ +D = D_p + D_n +$$ + +## Guardrails to avoid data leakage + +We want to ensure that the evaluation set doesn't contain any queries that are in the training set. There are two cases we need to consider + +1. Memorization: Queries are different but command is the same + * These are valid training, evaluation pairs because we expect the command to exactly match even though the queries are different +1. Contamination: Queries are the same and commands are the same + +We can use the distance metric proposed in the previous section to determine if a command in our training set matches the command in the eval dataset. + +We can also use this to automatically classifiy each evaluation example as a memorization or generalization example. If the distance from the command in the evaluation set to the closest command in the training set is less than some threshold, we can classify it as a memorization example. Otherwise we can classify it as a generalization example. + +## Add an Option to filter out eval + +When processing an evaluation dataset we should probably denote in the logs that the request was part of the evaluation set. This way during the learning process we can filter it out to avoid learning our evaluation dataset. + +An easy way to do this would be to include a log field `eval` that is set to `true` for all evaluation examples. When constructing the logger in [Agent.Generate](https://github.com/jlewi/foyle/blob/4a5e3b573cbfad34b3060eafff33bceeeef21636/app/pkg/agent/agent.go#L73) we can set this field to `true` if the Agent is in `eval` mode. + +We still want the Agent to log the evaluation examples so we can use the logs and traces to evaluate the Agent. + +In the future we could potentially set the `eval` field in the request if we want to use the same server for both training and evaluation. For now, we'd probably run the Agent as a batch job and not start a server. + +## Alternatives + +### LLM Evaluation + +Using an LLM to evaluate the correctness of generated responses is another option. I'm not sure what advantage this would offer over the proposed metric. The proposed +metric has the advantage that it is interpratable and deteriminstic. LLM scoring would introduce another dimension of complexity; in particular we'd potentially need to calibrarte the LLM to align its scores with human scoring. \ No newline at end of file