From 69b04fb1f8ebd58f1418771a90e7f3f5567c526a Mon Sep 17 00:00:00 2001 From: sajit Date: Fri, 16 Aug 2024 19:40:44 -0400 Subject: [PATCH 01/28] Signed-off-by: Sajit Kunnumkal Init commit cmd files intermediate stop Changing signature adding scripts to gitignore address review comments address review comments ignore mock test failing test Signed-off-by: sajit fixed test Make test optional Signed-off-by: sajit adding tests Signed-off-by: sajit Added tests Signed-off-by: sajit Delete unused struct Signed-off-by: sajit Signed-off-by: sajit --- .gitignore | 1 + mongodb-atlas/cmd/main.go | 66 +++ mongodb-atlas/cmd/main_test.go | 110 ++++ mongodb-atlas/go.mod | 30 ++ mongodb-atlas/go.sum | 604 ++++++++++++++++++++++ mongodb-atlas/plugin/costexplorerquery.go | 16 + 6 files changed, 827 insertions(+) create mode 100644 mongodb-atlas/cmd/main.go create mode 100644 mongodb-atlas/cmd/main_test.go create mode 100644 mongodb-atlas/go.mod create mode 100644 mongodb-atlas/go.sum create mode 100644 mongodb-atlas/plugin/costexplorerquery.go diff --git a/.gitignore b/.gitignore index d7323bc..bd922fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ build *.out +mongodb-atlas/scripts \ No newline at end of file diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go new file mode 100644 index 0000000..28681cd --- /dev/null +++ b/mongodb-atlas/cmd/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" + "github.com/opencost/opencost/core/pkg/log" +) + +func main() { + fmt.Println("Initialize plugin") + +} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +// pass list of orgs , start date, end date +func createCostExplorerQueryToken(org string, startDate time.Time, endDate time.Time, + client HTTPClient, url string) (string, error) { + // Define the layout for the desired format + layout := "2006-01-02" + + // Convert the time.Time object to a string in yyyy-mm-dd format + startDateString := startDate.Format(layout) + endDateString := endDate.Format(layout) + + payload := plugin.CreateCostExplorerQueryPayload{ + + EndDate: endDateString, + StartDate: startDateString, + Organizations: []string{org}, + } + payloadJson, _ := json.Marshal(payload) + + request, _ := http.NewRequest("POST", url, bytes.NewBuffer(payloadJson)) + + request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") + request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") + + response, error := client.Do(request) + if error != nil { + msg := fmt.Sprintf("createCostExplorerQueryToken: error from server: %v", error) + log.Errorf(msg) + return "", fmt.Errorf(msg) + + } + defer response.Body.Close() + + body, _ := io.ReadAll(response.Body) + //fmt.Println("response Body:", string(body)) + var createCostExplorerQueryResponse plugin.CreateCostExplorerQueryResponse + respUnmarshalError := json.Unmarshal([]byte(body), &createCostExplorerQueryResponse) + if respUnmarshalError != nil { + msg := fmt.Sprintf("createCostExplorerQueryToken: error unmarshalling response: %v", respUnmarshalError) + log.Errorf(msg) + return "", fmt.Errorf(msg) + } + return createCostExplorerQueryResponse.Token, nil +} diff --git a/mongodb-atlas/cmd/main_test.go b/mongodb-atlas/cmd/main_test.go new file mode 100644 index 0000000..5219a2a --- /dev/null +++ b/mongodb-atlas/cmd/main_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/icholy/digest" + "github.com/stretchr/testify/assert" +) + +type ClientMock struct { +} + +func (c *ClientMock) Do(req *http.Request) (*http.Response, error) { + // Implement a mock response + return nil, errors.New("Test Error") +} + +func TestCallToCreateCostExplorerQuery(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"Token":"mockToken"}`)) + })) + defer server.Close() + + mockClient := &http.Client{} + // Define the layout that matches the format of the date string + layout := "2006-01-02" + endTime, _ := time.Parse(layout, "2024-07-01") + startTime, _ := time.Parse(layout, "2023-12-01") + resp, error := createCostExplorerQueryToken("myOrg", startTime, endTime, mockClient, server.URL) + assert.Nil(t, error) + assert.Equal(t, "mockToken", resp) + +} + +// FOR INTEGRATION TESTING PURPOSES ONLY +// expects 3 env variables to be set to work +// mapuk = public key for mongodb atlas +// maprk = private key for mongodb atlas +// maOrgId = orgId to be testsed +func TestMain(t *testing.T) { + + publicKey := os.Getenv("mapuk") + privateKey := os.Getenv("maprk") + orgId := os.Getenv("maorgid") + if publicKey == "" || privateKey == "" || orgId == "" { + t.Skip("Skipping integration test.") + } + + assert.NotNil(t, publicKey) + assert.NotNil(t, privateKey) + assert.NotNil(t, orgId) + + client := &http.Client{ + Transport: &digest.Transport{ + Username: publicKey, + Password: privateKey, + }, + } + + // Define the layout that matches the format of the date string + layout := "2006-01-02" + endTime, _ := time.Parse(layout, "2024-07-01") + startTime, _ := time.Parse(layout, "2023-12-01") + url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" + resp, err := createCostExplorerQueryToken(orgId, startTime, endTime, client, url) + + assert.NotEmpty(t, resp) + assert.Nil(t, err) + +} + +func TestErrorFromServer(t *testing.T) { + + client := &ClientMock{} + // Define the layout that matches the format of the date string + layout := "2006-01-02" + endTime, _ := time.Parse(layout, "2024-07-01") + startTime, _ := time.Parse(layout, "2023-12-01") + orgId := "1" + url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" + _, err := createCostExplorerQueryToken(orgId, startTime, endTime, client, url) + + assert.NotEmpty(t, err) + +} + +func TestCallToCreateCostExplorerQueryBadMessage(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`this is not json`)) + })) + defer server.Close() + + mockClient := &http.Client{} + // Define the layout that matches the format of the date string + layout := "2006-01-02" + endTime, _ := time.Parse(layout, "2024-07-01") + startTime, _ := time.Parse(layout, "2023-12-01") + _, error := createCostExplorerQueryToken("myOrg", startTime, endTime, mockClient, server.URL) + assert.NotEmpty(t, error) + +} diff --git a/mongodb-atlas/go.mod b/mongodb-atlas/go.mod new file mode 100644 index 0000000..53647ca --- /dev/null +++ b/mongodb-atlas/go.mod @@ -0,0 +1,30 @@ +module github.com/opencost/opencost-plugins/mongodb-atlas + +go 1.22.5 + +require github.com/icholy/digest v0.1.23 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/zerolog v1.26.1 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.8.1 // indirect + github.com/stretchr/testify v1.9.0 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/mongodb-atlas/go.sum b/mongodb-atlas/go.sum new file mode 100644 index 0000000..6fe7cc4 --- /dev/null +++ b/mongodb-atlas/go.sum @@ -0,0 +1,604 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/icholy/digest v0.1.23 h1:4hX2pIloP0aDx7RJW0JewhPPy3R8kU+vWKdxPsCCGtY= +github.com/icholy/digest v0.1.23/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c h1:g+AKEjkyTLzo3O6nu2iuCxy+baoXAiPSr3l3bn8J4Ho= +github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c/go.mod h1:c1he7ogYA3J/m2BNbWD5FDLRTpUwuG8CGIyOkDV+i+s= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= +github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/mongodb-atlas/plugin/costexplorerquery.go b/mongodb-atlas/plugin/costexplorerquery.go new file mode 100644 index 0000000..ce98329 --- /dev/null +++ b/mongodb-atlas/plugin/costexplorerquery.go @@ -0,0 +1,16 @@ +package plugin + +type CreateCostExplorerQueryPayload struct { + Clusters []string `json:"clusters"` + EndDate string `json:"endDate"` + GroupBy string `json:"groupBy"` + IncludePartialMatches bool `json:"includePartialMatches"` + Organizations []string `json:"organizations"` + Projects []string `json:"projects"` + Services []string `json:"services"` + StartDate string `json:"startDate"` +} + +type CreateCostExplorerQueryResponse struct { + Token string `json:"token"` +} From 09bb7a30be3137a2b85bf4d08eccbb193ea73eab Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Sun, 1 Sep 2024 19:20:54 -0400 Subject: [PATCH 02/28] boilerplate for plugin interface (#1) Merge Alex's PR to continue work on branch Signed-off-by: sajit --- common/config/config_helpers.go | 22 ++++ common/go.mod | 3 + datadog/cmd/main/main.go | 19 +--- datadog/go.mod | 5 +- mongodb-atlas/cmd/main.go | 149 ++++++++++++++++++++++++++-- mongodb-atlas/config/atlasconfig.go | 32 ++++++ mongodb-atlas/go.mod | 41 +++++++- mongodb-atlas/go.sum | 86 ++++++++++++++++ 8 files changed, 330 insertions(+), 27 deletions(-) create mode 100644 common/config/config_helpers.go create mode 100644 common/go.mod create mode 100644 mongodb-atlas/config/atlasconfig.go diff --git a/common/config/config_helpers.go b/common/config/config_helpers.go new file mode 100644 index 0000000..e974d3f --- /dev/null +++ b/common/config/config_helpers.go @@ -0,0 +1,22 @@ +package config + +import ( + "fmt" + "os" +) + +func GetConfigFilePath() (string, error) { + // plugins expect exactly 2 args: the executable itself, + // and a path to the config file to use + // all config for the plugin must come through the config file + if len(os.Args) != 2 { + return "", fmt.Errorf("plugins require 2 args: the plugin itself, and the full path to its config file. Got %d args", len(os.Args)) + } + + _, err := os.Stat(os.Args[1]) + if err != nil { + return "", fmt.Errorf("error reading config file at %s: %v", os.Args[1], err) + } + + return os.Args[1], nil +} diff --git a/common/go.mod b/common/go.mod new file mode 100644 index 0000000..27eab33 --- /dev/null +++ b/common/go.mod @@ -0,0 +1,3 @@ +module github.com/opencost/opencost-plugins/common + +go 1.22.2 diff --git a/datadog/cmd/main/main.go b/datadog/cmd/main/main.go index 95e7b6e..3dde4ff 100644 --- a/datadog/cmd/main/main.go +++ b/datadog/cmd/main/main.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/hashicorp/go-plugin" + commonconfig "github.com/opencost/opencost-plugins/common/config" datadogplugin "github.com/opencost/opencost-plugins/datadog/datadogplugin" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/model/pb" @@ -89,7 +90,7 @@ func (d *DatadogCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Cust func main() { - configFile, err := getConfigFilePath() + configFile, err := commonconfig.GetConfigFilePath() if err != nil { log.Fatalf("error opening config file: %v", err) } @@ -587,22 +588,6 @@ func getDatadogConfig(configFilePath string) (*datadogplugin.DatadogConfig, erro return &result, nil } -func getConfigFilePath() (string, error) { - // plugins expect exactly 2 args: the executable itself, - // and a path to the config file to use - // all config for the plugin must come through the config file - if len(os.Args) != 2 { - return "", fmt.Errorf("plugins require 2 args: the plugin itself, and the full path to its config file. Got %d args", len(os.Args)) - } - - _, err := os.Stat(os.Args[1]) - if err != nil { - return "", fmt.Errorf("error reading config file at %s: %v", os.Args[1], err) - } - - return os.Args[1], nil -} - func scrapeDatadogPrices(url string) (*datadogplugin.PricingInformation, error) { // Send a GET request to the URL response, err := http.Get(url) diff --git a/datadog/go.mod b/datadog/go.mod index f6deebc..31273df 100644 --- a/datadog/go.mod +++ b/datadog/go.mod @@ -1,6 +1,8 @@ module github.com/opencost/opencost-plugins/datadog -go 1.21.6 +go 1.22.2 + +replace github.com/opencost/opencost-plugins/common => ../common require ( github.com/DataDog/datadog-api-client-go/v2 v2.23.0 @@ -36,6 +38,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect + github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/rs/zerolog v1.32.0 // indirect diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go index 28681cd..00b683b 100644 --- a/mongodb-atlas/cmd/main.go +++ b/mongodb-atlas/cmd/main.go @@ -8,22 +8,159 @@ import ( "net/http" "time" - "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" + "github.com/hashicorp/go-plugin" + "github.com/icholy/digest" + commonconfig "github.com/opencost/opencost-plugins/common/config" + atlasconfig "github.com/opencost/opencost-plugins/mongodb-atlas/config" + atlasplugin "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" ) +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "mongodb-atlas", +} + +const costExplorerFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage" + func main() { - fmt.Println("Initialize plugin") + fmt.Println("Initializing Mongo plugin") + + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + atlasConfig, err := atlasconfig.GetAtlasConfig(configFile) + if err != nil { + log.Fatalf("error building Atlas config: %v", err) + } + log.SetLogLevel(atlasConfig.LogLevel) + + // as per https://www.mongodb.com/docs/atlas/api/atlas-admin-api-ref/, + // atlas admin APIs have a limit of 100 requests per minute + rateLimiter := rate.NewLimiter(1.1, 2) + atlasCostSrc := AtlasCostSource{ + rateLimiter: rateLimiter, + orgID: atlasConfig.OrgID, + } + atlasCostSrc.atlasClient = getAtlasClient(*atlasConfig) + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &atlasCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) } +func getAtlasClient(atlasConfig atlasconfig.AtlasConfig) HTTPClient { + return &http.Client{ + Transport: &digest.Transport{ + Username: atlasConfig.PublicKey, + Password: atlasConfig.PrivateKey, + }, + } +} + +// Implementation of CustomCostSource +type AtlasCostSource struct { + orgID string + rateLimiter *rate.Limiter + atlasClient HTTPClient +} + type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } +func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + for _, target := range targets { + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching atlas costs for window %v", target) + result, err := a.getAtlasCostsForWindow(&target) + if err != nil { + log.Errorf("error getting costs for window %v: %v", target, err) + errResp := pb.CustomCostResponse{} + errResp.Errors = append(errResp.Errors, fmt.Sprintf("error getting costs for window %v: %v", target, err)) + results = append(results, &errResp) + } else { + results = append(results, result) + } + } + + return results +} + +func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.CustomCostResponse, error) { + + // get the token + token, err := createCostExplorerQueryToken(a.orgID, *win.Start(), *win.End(), a.atlasClient) + if err != nil { + log.Errorf("error getting token: %v", err) + return nil, err + } + + // get the costs + costs, err := getCosts(a.atlasClient, token) + if err != nil { + log.Errorf("error getting costs: %v", err) + return nil, err + } + + resp := pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v1"}, + CostSource: "data_storage", + Domain: "mongodb-atlas", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: costs, + } + return &resp, nil +} + +func getCosts(client HTTPClient, token string) ([]*pb.CustomCost, error) { + // TODO - take the token, call the usage API, downlad the usage, parse it into the CustomCost struct + return nil, nil +} + // pass list of orgs , start date, end date func createCostExplorerQueryToken(org string, startDate time.Time, endDate time.Time, - client HTTPClient, url string) (string, error) { + client HTTPClient) (string, error) { // Define the layout for the desired format layout := "2006-01-02" @@ -31,7 +168,7 @@ func createCostExplorerQueryToken(org string, startDate time.Time, endDate time. startDateString := startDate.Format(layout) endDateString := endDate.Format(layout) - payload := plugin.CreateCostExplorerQueryPayload{ + payload := atlasplugin.CreateCostExplorerQueryPayload{ EndDate: endDateString, StartDate: startDateString, @@ -39,7 +176,7 @@ func createCostExplorerQueryToken(org string, startDate time.Time, endDate time. } payloadJson, _ := json.Marshal(payload) - request, _ := http.NewRequest("POST", url, bytes.NewBuffer(payloadJson)) + request, _ := http.NewRequest("POST", fmt.Sprintf(costExplorerFmt, org), bytes.NewBuffer(payloadJson)) request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") @@ -55,7 +192,7 @@ func createCostExplorerQueryToken(org string, startDate time.Time, endDate time. body, _ := io.ReadAll(response.Body) //fmt.Println("response Body:", string(body)) - var createCostExplorerQueryResponse plugin.CreateCostExplorerQueryResponse + var createCostExplorerQueryResponse atlasplugin.CreateCostExplorerQueryResponse respUnmarshalError := json.Unmarshal([]byte(body), &createCostExplorerQueryResponse) if respUnmarshalError != nil { msg := fmt.Sprintf("createCostExplorerQueryToken: error unmarshalling response: %v", respUnmarshalError) diff --git a/mongodb-atlas/config/atlasconfig.go b/mongodb-atlas/config/atlasconfig.go new file mode 100644 index 0000000..98cad03 --- /dev/null +++ b/mongodb-atlas/config/atlasconfig.go @@ -0,0 +1,32 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +type AtlasConfig struct { + PublicKey string `json:"atlas_public_key"` + PrivateKey string `json:"atlas_private_key"` + OrgID string `json:"atlas_org_id"` + LogLevel string `json:"atlas_plugin_log_level"` +} + +func GetAtlasConfig(configFilePath string) (*AtlasConfig, error) { + var result AtlasConfig + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for Atlas config @ %s: %v", configFilePath, err) + } + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error marshaling json into Atlas config %v", err) + } + + if result.LogLevel == "" { + result.LogLevel = "info" + } + + return &result, nil +} diff --git a/mongodb-atlas/go.mod b/mongodb-atlas/go.mod index 53647ca..bf22ec6 100644 --- a/mongodb-atlas/go.mod +++ b/mongodb-atlas/go.mod @@ -2,16 +2,38 @@ module github.com/opencost/opencost-plugins/mongodb-atlas go 1.22.5 +replace github.com/opencost/opencost-plugins/common => ../common + require github.com/icholy/digest v0.1.23 require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/goccy/go-json v0.9.11 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-plugin v1.6.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.5 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oklog/run v1.1.0 // indirect + github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 // indirect + github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/zerolog v1.26.1 // indirect @@ -22,9 +44,22 @@ require ( github.com/spf13/viper v1.8.1 // indirect github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.6.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.25.3 // indirect + k8s.io/apimachinery v0.25.3 // indirect + k8s.io/klog/v2 v2.80.0 // indirect + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/mongodb-atlas/go.sum b/mongodb-atlas/go.sum index 6fe7cc4..1cda2a6 100644 --- a/mongodb-atlas/go.sum +++ b/mongodb-atlas/go.sum @@ -66,6 +66,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -73,7 +76,14 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -104,6 +114,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -120,6 +132,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -142,11 +156,18 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI= +github.com/hashicorp/go-plugin v1.6.1/go.mod h1:XPHFku2tFo3o3QKFgSYo+cghcUhw1NA1hZyMK0PWAw0= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -161,11 +182,15 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icholy/digest v0.1.23 h1:4hX2pIloP0aDx7RJW0JewhPPy3R8kU+vWKdxPsCCGtY= github.com/icholy/digest v0.1.23/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -178,11 +203,22 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -191,11 +227,21 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c h1:g+AKEjkyTLzo3O6nu2iuCxy+baoXAiPSr3l3bn8J4Ho= github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c/go.mod h1:c1he7ogYA3J/m2BNbWD5FDLRTpUwuG8CGIyOkDV+i+s= +github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 h1:PDYQw0cygJ8ehn/AObpRVru4Cg718aGrDJQis4XfHWg= +github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830/go.mod h1:c1he7ogYA3J/m2BNbWD5FDLRTpUwuG8CGIyOkDV+i+s= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -214,6 +260,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= @@ -231,6 +278,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= @@ -272,6 +320,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -334,6 +384,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -373,6 +425,7 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -399,10 +452,17 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -414,9 +474,13 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -542,7 +606,10 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -563,6 +630,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -575,9 +644,13 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -599,6 +672,19 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= +k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= +k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= +k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.80.0 h1:lyJt0TWMPaGoODa8B8bUuxgHS3W/m/bNr2cca3brA/g= +k8s.io/klog/v2 v2.80.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= From 5fe974f90ffc8b209fa045c34b09127e81e71d51 Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 2 Sep 2024 07:50:38 -0400 Subject: [PATCH 03/28] Added tests for atlasconfig.go Signed-off-by: sajit --- mongodb-atlas/config/atlasconfig_test.go | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 mongodb-atlas/config/atlasconfig_test.go diff --git a/mongodb-atlas/config/atlasconfig_test.go b/mongodb-atlas/config/atlasconfig_test.go new file mode 100644 index 0000000..06c8027 --- /dev/null +++ b/mongodb-atlas/config/atlasconfig_test.go @@ -0,0 +1,76 @@ +package config + +import ( + "os" + "testing" +) + +// Unit tests for the GetAtlasConfig function +func TestGetAtlasConfig(t *testing.T) { + // Test: Valid configuration file + // t.Run("Valid configuration file", func(t *testing.T) { + // configFilePath := "test_valid_config.json" + // // Create a temporary valid JSON file + // validConfig := `{"log_level": "debug"}` + // err := os.WriteFile(configFilePath, []byte(validConfig), 0644) + // if err != nil { + // t.Fatalf("failed to create temporary config file: %v", err) + // } + // defer os.Remove(configFilePath) + + // config, err := GetAtlasConfig(configFilePath) + // if err != nil { + // t.Fatalf("expected no error, but got: %v", err) + // } + // fmt.Println(config, configFilePath) + // if config.LogLevel != "debug" { + // t.Errorf("expected log level to be 'debug', but got: %s", config.LogLevel) + // } + // }) + + // Test: Invalid file path + t.Run("Invalid file path", func(t *testing.T) { + configFilePath := "invalid_path.json" + _, err := GetAtlasConfig(configFilePath) + if err == nil { + t.Errorf("expected an error, but got none") + } + }) + + // Test: Invalid JSON format + t.Run("Invalid JSON format", func(t *testing.T) { + configFilePath := "test_invalid_json.json" + // Create a temporary invalid JSON file + invalidConfig := `{"log_level": "debug"` + err := os.WriteFile(configFilePath, []byte(invalidConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) + + _, err = GetAtlasConfig(configFilePath) + if err == nil { + t.Errorf("expected an error, but got none") + } + }) + + // Test: Default log level when missing + t.Run("Default log level when missing", func(t *testing.T) { + configFilePath := "test_missing_log_level.json" + // Create a temporary JSON file without log_level + missingLogLevelConfig := `{}` + err := os.WriteFile(configFilePath, []byte(missingLogLevelConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) + + config, err := GetAtlasConfig(configFilePath) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + if config.LogLevel != "info" { + t.Errorf("expected log level to be 'info', but got: %s", config.LogLevel) + } + }) +} From 7e00bcaafa8ba2a2824966bb95447f5e989b4567 Mon Sep 17 00:00:00 2001 From: sajit Date: Fri, 27 Sep 2024 09:26:40 -0400 Subject: [PATCH 04/28] Get costs Signed-off-by: sajit --- mongodb-atlas/cmd/main.go | 40 +++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go index 00b683b..5197112 100644 --- a/mongodb-atlas/cmd/main.go +++ b/mongodb-atlas/cmd/main.go @@ -32,6 +32,7 @@ var handshakeConfig = plugin.HandshakeConfig{ } const costExplorerFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage" +const costExplorerQueryFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage/%s" func main() { fmt.Println("Initializing Mongo plugin") @@ -133,7 +134,7 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust } // get the costs - costs, err := getCosts(a.atlasClient, token) + costs, err := getCosts(a.atlasClient, a.orgID, token) if err != nil { log.Errorf("error getting costs: %v", err) return nil, err @@ -153,9 +154,40 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust return &resp, nil } -func getCosts(client HTTPClient, token string) ([]*pb.CustomCost, error) { +func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, error) { + request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerQueryFmt, org, token) + + request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") + request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") + + response, error := client.Do(request) + statusCode := response.StatusCode + //102 status code means processing - so repeat call 5 times to see if we get a response + for count := 1; count < 5 && statusCode == 102; count++ { + // Sleep for 5 seconds before the next request + time.Sleep(5 * time.Second) + response, err := client.Do(request) + statusCode := response.StatusCode + + + } + + if error != nil { + msg := fmt.Sprintf("getCostExplorerUsage: error from server: %v", error) + log.Errorf(msg) + return nil, fmt.Errorf(msg) + + } + //fake it for now + cost := pb.CustomCost{ + Metadata: "HI", + Zone: "US-EAST", + AccountName: org, + ChargeCategory: "DU" + } + //TODO convert response to // TODO - take the token, call the usage API, downlad the usage, parse it into the CustomCost struct - return nil, nil + return cost, nil } // pass list of orgs , start date, end date @@ -191,7 +223,7 @@ func createCostExplorerQueryToken(org string, startDate time.Time, endDate time. defer response.Body.Close() body, _ := io.ReadAll(response.Body) - //fmt.Println("response Body:", string(body)) + log.Debugf("response Body:", string(body)) var createCostExplorerQueryResponse atlasplugin.CreateCostExplorerQueryResponse respUnmarshalError := json.Unmarshal([]byte(body), &createCostExplorerQueryResponse) if respUnmarshalError != nil { From 5d26705999e6185b256fa553a25a07bef04b1738 Mon Sep 17 00:00:00 2001 From: sajit Date: Thu, 3 Oct 2024 22:07:48 -0400 Subject: [PATCH 05/28] Signed-off-by: Sajit Kunnumkal Signed-off-by: sajit --- .gitmessage.txt | 2 + .vscode/launch.json | 17 ++ mongodb-atlas/cmd/main.go | 55 +++++-- mongodb-atlas/cmd/main_test.go | 192 ++++++++++++++-------- mongodb-atlas/plugin/costexplorerquery.go | 13 ++ 5 files changed, 197 insertions(+), 82 deletions(-) create mode 100644 .gitmessage.txt create mode 100644 .vscode/launch.json diff --git a/.gitmessage.txt b/.gitmessage.txt new file mode 100644 index 0000000..f9a3bed --- /dev/null +++ b/.gitmessage.txt @@ -0,0 +1,2 @@ +#Commit Message template +Signed-off-by: Sajit Kunnumkal diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..abba819 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "args": ["/Users/sajit/Desktop/atlas-config.json"] + } + ] +} \ No newline at end of file diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go index 5197112..b9dddd9 100644 --- a/mongodb-atlas/cmd/main.go +++ b/mongodb-atlas/cmd/main.go @@ -127,7 +127,7 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.CustomCostResponse, error) { // get the token - token, err := createCostExplorerQueryToken(a.orgID, *win.Start(), *win.End(), a.atlasClient) + token, err := CreateCostExplorerQueryToken(a.orgID, *win.Start(), *win.End(), a.atlasClient) if err != nil { log.Errorf("error getting token: %v", err) return nil, err @@ -155,7 +155,7 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust } func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, error) { - request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerQueryFmt, org, token) + request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerQueryFmt, org, token), nil) request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") @@ -166,11 +166,10 @@ func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er for count := 1; count < 5 && statusCode == 102; count++ { // Sleep for 5 seconds before the next request time.Sleep(5 * time.Second) - response, err := client.Do(request) - statusCode := response.StatusCode - - - } + response, _ := client.Do(request) + statusCode = response.StatusCode + + } if error != nil { msg := fmt.Sprintf("getCostExplorerUsage: error from server: %v", error) @@ -178,20 +177,40 @@ func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er return nil, fmt.Errorf(msg) } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + log.Debugf("response Body: %s", string(body)) + var costResponse atlasplugin.CostResponse + respUnmarshalError := json.Unmarshal([]byte(body), &costResponse) + if respUnmarshalError != nil { + msg := fmt.Sprintf("getCost: error unmarshalling response: %v", respUnmarshalError) + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + //sample + //report_data='{"usageDetails":[{"invoiceId":"66d7254246a21a41036ff315","organizationId":"66d7254246a21a41036ff2e9","organizationName":"Kubecost","service":"Data Transfer","usageAmount":1.33,"usageDate":"2024-09-01"}, + //{"invoiceId":"66d7254246a21a41036ff315","organizationId":"66d7254246a21a41036ff2e9","organizationName":"Kubecost","service":"Clusters","usageAmount":51.19,"usageDate":"2024-09-01"}]}’ + //fake it for now - cost := pb.CustomCost{ - Metadata: "HI", - Zone: "US-EAST", - AccountName: org, - ChargeCategory: "DU" - } - //TODO convert response to - // TODO - take the token, call the usage API, downlad the usage, parse it into the CustomCost struct - return cost, nil + var costs []*pb.CustomCost + // Iterate over the UsageDetails in CostResponse + for _, invoice := range costResponse.UsageDetails { + // Create a new pb.CustomCost for each Invoice + customCost := &pb.CustomCost{ + Id: invoice.InvoiceId, + AccountName: invoice.OrganizationName, + ChargeCategory: invoice.Service, + BilledCost: invoice.UsageAmount, + } + + // Append the customCost pointer to the slice + costs = append(costs, customCost) + } + return costs, nil } // pass list of orgs , start date, end date -func createCostExplorerQueryToken(org string, startDate time.Time, endDate time.Time, +func CreateCostExplorerQueryToken(org string, startDate time.Time, endDate time.Time, client HTTPClient) (string, error) { // Define the layout for the desired format layout := "2006-01-02" @@ -223,7 +242,7 @@ func createCostExplorerQueryToken(org string, startDate time.Time, endDate time. defer response.Body.Close() body, _ := io.ReadAll(response.Body) - log.Debugf("response Body:", string(body)) + log.Debugf("response Body: %s", string(body)) var createCostExplorerQueryResponse atlasplugin.CreateCostExplorerQueryResponse respUnmarshalError := json.Unmarshal([]byte(body), &createCostExplorerQueryResponse) if respUnmarshalError != nil { diff --git a/mongodb-atlas/cmd/main_test.go b/mongodb-atlas/cmd/main_test.go index 5219a2a..da7bc46 100644 --- a/mongodb-atlas/cmd/main_test.go +++ b/mongodb-atlas/cmd/main_test.go @@ -1,42 +1,75 @@ package main import ( - "errors" + "bytes" + "encoding/json" + "fmt" + "io/ioutil" "net/http" - "net/http/httptest" - "os" "testing" "time" - "github.com/icholy/digest" + atlasplugin "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" "github.com/stretchr/testify/assert" ) -type ClientMock struct { +// Mock HTTPClient implementation +type MockHTTPClient struct { + DoFunc func(req *http.Request) (*http.Response, error) } -func (c *ClientMock) Do(req *http.Request) (*http.Response, error) { - // Implement a mock response - return nil, errors.New("Test Error") +// The MockHTTPClient's Do method uses a function defined at runtime to mock various responses +func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { + return m.DoFunc(req) } -func TestCallToCreateCostExplorerQuery(t *testing.T) { +func TestSanity(t *testing.T) { + assert.True(t, true) +} - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"Token":"mockToken"}`)) - })) - defer server.Close() +func TestCreateCostExplorerQueryToken(t *testing.T) { + // Mock data + org := "testOrg" + startDate := time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC) + endDate := time.Date(2023, 9, 30, 0, 0, 0, 0, time.UTC) + expectedToken := "mockToken" - mockClient := &http.Client{} - // Define the layout that matches the format of the date string - layout := "2006-01-02" - endTime, _ := time.Parse(layout, "2024-07-01") - startTime, _ := time.Parse(layout, "2023-12-01") - resp, error := createCostExplorerQueryToken("myOrg", startTime, endTime, mockClient, server.URL) - assert.Nil(t, error) - assert.Equal(t, "mockToken", resp) + // Define the response that the mock client will return + mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ + Token: expectedToken, + } + mockResponseJson, _ := json.Marshal(mockResponse) + + // Create a mock HTTPClient that returns a successful response + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify that the request method and URL are correct + if req.Method != http.MethodPost { + t.Errorf("expected POST request, got %s", req.Method) + } + expectedURL := fmt.Sprintf(costExplorerFmt, org) + if req.URL.String() != expectedURL { + t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) + } + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + }, + } + // Call the function under test + token, err := CreateCostExplorerQueryToken(org, startDate, endDate, mockClient) + + // Assert that the function returned the expected token and no error + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if token != expectedToken { + t.Errorf("expected token %s, got %s", expectedToken, token) + } } // FOR INTEGRATION TESTING PURPOSES ONLY @@ -44,48 +77,65 @@ func TestCallToCreateCostExplorerQuery(t *testing.T) { // mapuk = public key for mongodb atlas // maprk = private key for mongodb atlas // maOrgId = orgId to be testsed -func TestMain(t *testing.T) { - - publicKey := os.Getenv("mapuk") - privateKey := os.Getenv("maprk") - orgId := os.Getenv("maorgid") - if publicKey == "" || privateKey == "" || orgId == "" { - t.Skip("Skipping integration test.") - } - - assert.NotNil(t, publicKey) - assert.NotNil(t, privateKey) - assert.NotNil(t, orgId) - - client := &http.Client{ - Transport: &digest.Transport{ - Username: publicKey, - Password: privateKey, - }, - } - - // Define the layout that matches the format of the date string - layout := "2006-01-02" - endTime, _ := time.Parse(layout, "2024-07-01") - startTime, _ := time.Parse(layout, "2023-12-01") - url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" - resp, err := createCostExplorerQueryToken(orgId, startTime, endTime, client, url) - - assert.NotEmpty(t, resp) - assert.Nil(t, err) - -} +// func TestMain(t *testing.T) { + +// publicKey := os.Getenv("mapuk") +// privateKey := os.Getenv("maprk") +// orgId := os.Getenv("maorgid") +// if publicKey == "" || privateKey == "" || orgId == "" { +// t.Skip("Skipping integration test.") +// } + +// assert.NotNil(t, publicKey) +// assert.NotNil(t, privateKey) +// assert.NotNil(t, orgId) + +// client := &http.Client{ +// Transport: &digest.Transport{ +// Username: publicKey, +// Password: privateKey, +// }, +// } + +// // Define the layout that matches the format of the date string +// layout := "2006-01-02" +// endTime, _ := time.Parse(layout, "2024-07-01") +// startTime, _ := time.Parse(layout, "2023-12-01") +// url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" +// resp, err := createCostExplorerQueryToken(orgId, startTime, endTime, client) + +// assert.NotEmpty(t, resp) +// assert.Nil(t, err) + +// } func TestErrorFromServer(t *testing.T) { - client := &ClientMock{} // Define the layout that matches the format of the date string layout := "2006-01-02" endTime, _ := time.Parse(layout, "2024-07-01") startTime, _ := time.Parse(layout, "2023-12-01") orgId := "1" - url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" - _, err := createCostExplorerQueryToken(orgId, startTime, endTime, client, url) + // Create a mock HTTPClient that returns a successful response + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify that the request method and URL are correct + if req.Method != http.MethodPost { + t.Errorf("expected POST request, got %s", req.Method) + } + expectedURL := fmt.Sprintf(costExplorerFmt, orgId) + if req.URL.String() != expectedURL { + t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) + } + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: ioutil.NopCloser(bytes.NewBufferString("fake")), + }, nil + }, + } + _, err := CreateCostExplorerQueryToken(orgId, startTime, endTime, mockClient) assert.NotEmpty(t, err) @@ -93,18 +143,32 @@ func TestErrorFromServer(t *testing.T) { func TestCallToCreateCostExplorerQueryBadMessage(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(`this is not json`)) - })) - defer server.Close() - - mockClient := &http.Client{} // Define the layout that matches the format of the date string layout := "2006-01-02" endTime, _ := time.Parse(layout, "2024-07-01") startTime, _ := time.Parse(layout, "2023-12-01") - _, error := createCostExplorerQueryToken("myOrg", startTime, endTime, mockClient, server.URL) + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + // Verify that the request method and URL are correct + if req.Method != http.MethodPost { + t.Errorf("expected POST request, got %s", req.Method) + } + expectedURL := fmt.Sprintf(costExplorerFmt, "myOrg") + if req.URL.String() != expectedURL { + t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) + } + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewBufferString("This ain't json")), + }, nil + }, + } + + _, error := CreateCostExplorerQueryToken("myOrg", startTime, endTime, mockClient) assert.NotEmpty(t, error) } + +//tests for getCosts diff --git a/mongodb-atlas/plugin/costexplorerquery.go b/mongodb-atlas/plugin/costexplorerquery.go index ce98329..c145229 100644 --- a/mongodb-atlas/plugin/costexplorerquery.go +++ b/mongodb-atlas/plugin/costexplorerquery.go @@ -14,3 +14,16 @@ type CreateCostExplorerQueryPayload struct { type CreateCostExplorerQueryResponse struct { Token string `json:"token"` } + +type Invoice struct { + InvoiceId string `json:"invoiceId"` + OrganizationId string `json:"organizationId"` + OrganizationName string `json:"organizationName"` + Service string `json:"service"` + UsageAmount float32 `json:"usageAmount"` + UsageDate string `json:"usageDate"` + //"invoiceId":"66d7254246a21a41036ff315","organizationId":"66d7254246a21a41036ff2e9","organizationName":"Kubecost","service":"Clusters","usageAmount":51.19,"usageDate":"2024-09-01"} +} +type CostResponse struct { + UsageDetails []Invoice `json:"usageDetails"` +} From d8f1adb95f6fb41ef2e9e8469f6cea1d282eac63 Mon Sep 17 00:00:00 2001 From: sajit Date: Fri, 4 Oct 2024 16:55:39 -0400 Subject: [PATCH 06/28] Add unit tests Signed-off-by: sajit --- mongodb-atlas/cmd/main.go | 15 ++- mongodb-atlas/cmd/main_test.go | 186 +++++++++++++++++++++++++++++---- 2 files changed, 172 insertions(+), 29 deletions(-) diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go index b9dddd9..5123145 100644 --- a/mongodb-atlas/cmd/main.go +++ b/mongodb-atlas/cmd/main.go @@ -134,7 +134,7 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust } // get the costs - costs, err := getCosts(a.atlasClient, a.orgID, token) + costs, err := GetCosts(a.atlasClient, a.orgID, token) if err != nil { log.Errorf("error getting costs: %v", err) return nil, err @@ -154,7 +154,7 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust return &resp, nil } -func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, error) { +func GetCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, error) { request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerQueryFmt, org, token), nil) request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") @@ -163,7 +163,7 @@ func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er response, error := client.Do(request) statusCode := response.StatusCode //102 status code means processing - so repeat call 5 times to see if we get a response - for count := 1; count < 5 && statusCode == 102; count++ { + for count := 1; count < 5 && statusCode == http.StatusProcessing; count++ { // Sleep for 5 seconds before the next request time.Sleep(5 * time.Second) response, _ := client.Do(request) @@ -171,6 +171,10 @@ func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er } + if statusCode == http.StatusProcessing { + msg := "timeout waiting for response" + return nil, fmt.Errorf(msg) + } if error != nil { msg := fmt.Sprintf("getCostExplorerUsage: error from server: %v", error) log.Errorf(msg) @@ -187,11 +191,6 @@ func getCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er log.Errorf(msg) return nil, fmt.Errorf(msg) } - //sample - //report_data='{"usageDetails":[{"invoiceId":"66d7254246a21a41036ff315","organizationId":"66d7254246a21a41036ff2e9","organizationName":"Kubecost","service":"Data Transfer","usageAmount":1.33,"usageDate":"2024-09-01"}, - //{"invoiceId":"66d7254246a21a41036ff315","organizationId":"66d7254246a21a41036ff2e9","organizationName":"Kubecost","service":"Clusters","usageAmount":51.19,"usageDate":"2024-09-01"}]}’ - - //fake it for now var costs []*pb.CustomCost // Iterate over the UsageDetails in CostResponse for _, invoice := range costResponse.UsageDetails { diff --git a/mongodb-atlas/cmd/main_test.go b/mongodb-atlas/cmd/main_test.go index da7bc46..50c3944 100644 --- a/mongodb-atlas/cmd/main_test.go +++ b/mongodb-atlas/cmd/main_test.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "testing" "time" @@ -23,10 +23,6 @@ func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { return m.DoFunc(req) } -func TestSanity(t *testing.T) { - assert.True(t, true) -} - func TestCreateCostExplorerQueryToken(t *testing.T) { // Mock data org := "testOrg" @@ -55,7 +51,7 @@ func TestCreateCostExplorerQueryToken(t *testing.T) { // Return a mock response with status 200 and mock JSON body return &http.Response{ StatusCode: http.StatusOK, - Body: ioutil.NopCloser(bytes.NewBuffer(mockResponseJson)), + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), }, nil }, } @@ -119,19 +115,11 @@ func TestErrorFromServer(t *testing.T) { // Create a mock HTTPClient that returns a successful response mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { - // Verify that the request method and URL are correct - if req.Method != http.MethodPost { - t.Errorf("expected POST request, got %s", req.Method) - } - expectedURL := fmt.Sprintf(costExplorerFmt, orgId) - if req.URL.String() != expectedURL { - t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) - } // Return a mock response with status 200 and mock JSON body return &http.Response{ StatusCode: http.StatusInternalServerError, - Body: ioutil.NopCloser(bytes.NewBufferString("fake")), + Body: io.NopCloser(bytes.NewBufferString("fake")), }, nil }, } @@ -147,13 +135,62 @@ func TestCallToCreateCostExplorerQueryBadMessage(t *testing.T) { layout := "2006-01-02" endTime, _ := time.Parse(layout, "2024-07-01") startTime, _ := time.Parse(layout, "2023-12-01") + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString("This ain't json")), + }, nil + }, + } + + _, error := CreateCostExplorerQueryToken("myOrg", startTime, endTime, mockClient) + assert.NotEmpty(t, error) + +} + +// tests for getCosts +func TestGetCostsMultipleInvoices(t *testing.T) { + costResponse := atlasplugin.CostResponse{ + UsageDetails: []atlasplugin.Invoice{ + { + InvoiceId: "INV001", + OrganizationId: "ORG123", + OrganizationName: "Acme Corp", + Service: "Compute", + UsageAmount: 120.50, + UsageDate: "2024-10-01", + }, + { + InvoiceId: "INV002", + OrganizationId: "ORG124", + OrganizationName: "Beta Corp", + Service: "Storage", + UsageAmount: 75.75, + UsageDate: "2024-10-02", + }, + { + InvoiceId: "INV003", + OrganizationId: "ORG125", + OrganizationName: "Gamma Inc", + Service: "Networking", + UsageAmount: 50.00, + UsageDate: "2024-10-03", + }, + }, + } + + mockResponseJson, _ := json.Marshal(costResponse) + mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { // Verify that the request method and URL are correct - if req.Method != http.MethodPost { - t.Errorf("expected POST request, got %s", req.Method) + if req.Method != http.MethodGet { + t.Errorf("expected GET request, got %s", req.Method) } - expectedURL := fmt.Sprintf(costExplorerFmt, "myOrg") + expectedURL := fmt.Sprintf(costExplorerQueryFmt, "myOrg", "t1") if req.URL.String() != expectedURL { t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) } @@ -161,14 +198,121 @@ func TestCallToCreateCostExplorerQueryBadMessage(t *testing.T) { // Return a mock response with status 200 and mock JSON body return &http.Response{ StatusCode: http.StatusOK, - Body: ioutil.NopCloser(bytes.NewBufferString("This ain't json")), + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), }, nil }, } + costs, err := GetCosts(mockClient, "myOrg", "t1") + assert.Nil(t, err) + assert.Equal(t, 3, len(costs)) - _, error := CreateCostExplorerQueryToken("myOrg", startTime, endTime, mockClient) + for i, invoice := range costResponse.UsageDetails { + assert.Equal(t, invoice.InvoiceId, costs[i].Id) + assert.Equal(t, invoice.OrganizationName, costs[i].AccountName) + assert.Equal(t, invoice.Service, costs[i].ChargeCategory) + assert.Equal(t, invoice.UsageAmount, costs[i].BilledCost) + } +} + +func TestGetCostErrorFromServer(t *testing.T) { + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: io.NopCloser(bytes.NewBufferString("")), + }, nil + }, + } + costs, err := GetCosts(mockClient, "myOrg", "t1") + + assert.NotEmpty(t, err) + assert.Nil(t, costs) + +} + +func TestGetCostsBadMessage(t *testing.T) { + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Body: io.NopCloser(bytes.NewBufferString("No Jason No")), + }, nil + }, + } + + _, error := GetCosts(mockClient, "myOrg", "t1") assert.NotEmpty(t, error) } -//tests for getCosts +func TestRepeatCallTill200(t *testing.T) { + + var count = 0 + costResponse := atlasplugin.CostResponse{ + UsageDetails: []atlasplugin.Invoice{ + + { + InvoiceId: "INV003", + OrganizationId: "ORG125", + OrganizationName: "Gamma Inc", + Service: "Networking", + UsageAmount: 50.00, + UsageDate: "2024-10-03", + }, + }, + } + + mockResponseJson, _ := json.Marshal(costResponse) + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + count++ + + if count < 5 { + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusProcessing, + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + + } else { + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + + } + + }, + } + + costs, err := GetCosts(mockClient, "myOrg", "t1") + assert.Nil(t, err) + assert.Equal(t, 1, len(costs)) +} + +func TestStuckInProcessing(t *testing.T) { + + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + // Return a mock response with status 200 and mock JSON body + return &http.Response{ + StatusCode: http.StatusProcessing, + Body: io.NopCloser(bytes.NewBufferString("")), + }, nil + + }, + } + + costs, err := GetCosts(mockClient, "myOrg", "t1") + assert.NotNil(t, err) + assert.Nil(t, costs) +} From cfd8a561a11f88c00589bdc8c3a10987eb915d86 Mon Sep 17 00:00:00 2001 From: sajit Date: Fri, 4 Oct 2024 22:19:47 -0400 Subject: [PATCH 07/28] more tests Signed-off-by: sajit --- mongodb-atlas/cmd/main.go | 4 +- mongodb-atlas/cmd/main_test.go | 132 +++++++++++++++++++++++++++++---- 2 files changed, 118 insertions(+), 18 deletions(-) diff --git a/mongodb-atlas/cmd/main.go b/mongodb-atlas/cmd/main.go index 5123145..02fe7cf 100644 --- a/mongodb-atlas/cmd/main.go +++ b/mongodb-atlas/cmd/main.go @@ -162,8 +162,8 @@ func GetCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er response, error := client.Do(request) statusCode := response.StatusCode - //102 status code means processing - so repeat call 5 times to see if we get a response - for count := 1; count < 5 && statusCode == http.StatusProcessing; count++ { + //102 status code means processing - so repeat call 2 times to see if we get a response + for count := 1; count < 2 && statusCode == http.StatusProcessing; count++ { // Sleep for 5 seconds before the next request time.Sleep(5 * time.Second) response, _ := client.Do(request) diff --git a/mongodb-atlas/cmd/main_test.go b/mongodb-atlas/cmd/main_test.go index 50c3944..44d0b86 100644 --- a/mongodb-atlas/cmd/main_test.go +++ b/mongodb-atlas/cmd/main_test.go @@ -10,6 +10,11 @@ import ( "time" atlasplugin "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/stretchr/testify/assert" ) @@ -254,27 +259,13 @@ func TestGetCostsBadMessage(t *testing.T) { func TestRepeatCallTill200(t *testing.T) { var count = 0 - costResponse := atlasplugin.CostResponse{ - UsageDetails: []atlasplugin.Invoice{ - - { - InvoiceId: "INV003", - OrganizationId: "ORG125", - OrganizationName: "Gamma Inc", - Service: "Networking", - UsageAmount: 50.00, - UsageDate: "2024-10-03", - }, - }, - } - - mockResponseJson, _ := json.Marshal(costResponse) + mockResponseJson := getCostResponseMock() mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { count++ - if count < 5 { + if count < 2 { // Return a mock response with status 200 and mock JSON body return &http.Response{ StatusCode: http.StatusProcessing, @@ -298,6 +289,25 @@ func TestRepeatCallTill200(t *testing.T) { assert.Equal(t, 1, len(costs)) } +func getCostResponseMock() []byte { + costResponse := atlasplugin.CostResponse{ + UsageDetails: []atlasplugin.Invoice{ + + { + InvoiceId: "INV003", + OrganizationId: "ORG125", + OrganizationName: "Gamma Inc", + Service: "Networking", + UsageAmount: 50.00, + UsageDate: "2024-10-03", + }, + }, + } + + mockResponseJson, _ := json.Marshal(costResponse) + return mockResponseJson +} + func TestStuckInProcessing(t *testing.T) { mockClient := &MockHTTPClient{ @@ -316,3 +326,93 @@ func TestStuckInProcessing(t *testing.T) { assert.NotNil(t, err) assert.Nil(t, costs) } + +func TestGetAtlasCostsForWindow(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + costResponse := getCostResponseMock() + if req.Method == http.MethodGet { + //return costs + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(costResponse)), + }, nil + } else { + // Define the response that the mock client will return + mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ + Token: "fake", + } + mockResponseJson, _ := json.Marshal(mockResponse) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + } + + }, + } + atlasCostSource := AtlasCostSource{ + orgID: "myOrg", + atlasClient: mockClient, + } + // Define the start and end time for the window + startTime := time.Now().Add(-24 * time.Hour) // 24 hours ago + endTime := time.Now() // Now + + // Create a new Window instance + window := opencost.NewWindow(&startTime, &endTime) + resp, error := atlasCostSource.getAtlasCostsForWindow(&window) + assert.Nil(t, error) + assert.True(t, resp != nil) + assert.Equal(t, "data_storage", resp.CostSource) + assert.Equal(t, "mongodb-atlas", resp.Domain) + assert.Equal(t, "v1", resp.Version) + assert.Equal(t, "USD", resp.Currency) + assert.Equal(t, 1, len(resp.Costs)) +} + +func TestGetCosts(t *testing.T) { + mockClient := &MockHTTPClient{ + DoFunc: func(req *http.Request) (*http.Response, error) { + + costResponse := getCostResponseMock() + if req.Method == http.MethodGet { + //return costs + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(costResponse)), + }, nil + } else { + // Define the response that the mock client will return + mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ + Token: "fake", + } + mockResponseJson, _ := json.Marshal(mockResponse) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), + }, nil + } + + }, + } + atlasCostSource := AtlasCostSource{ + orgID: "myOrg", + atlasClient: mockClient, + } + // Define the start and end time for the window + startTime := time.Now().Add(-24 * time.Hour) // 24 hours ago + endTime := time.Now() + + customCostRequest := pb.CustomCostRequest{ + Start: timestamppb.New(startTime), + End: timestamppb.New(endTime), + Resolution: durationpb.New(time.Hour), // Example resolution: 1 hour + } // Now + + resp := atlasCostSource.GetCustomCosts(&customCostRequest) + + assert.Equal(t, 1, len(resp)) + +} From bdf1210baacbdc79686a417b8877eca8c6bfdaf0 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Fri, 11 Oct 2024 21:43:16 -0400 Subject: [PATCH 08/28] migrate mongo plugin to use harness (#4) Signed-off-by: sajit --- .github/workflows/run-integration-tests.yaml | 66 ++++++ .github/workflows/update-manifest.yaml | 47 ++++ .gitignore | 3 +- .vscode/launch.json | 17 -- README.md | 3 - justfile | 17 +- .../common}/config/config_helpers.go | 0 {common => pkg/common}/go.mod | 0 .../plugins/datadog}/cmd/main/main.go | 162 ++++++++++--- .../plugins/datadog}/cmd/main/main_test.go | 6 +- .../datadog/cmd/validator/main/main.go | 219 ++++++++++++++++++ .../datadog}/datadogplugin/datadogconfig.go | 0 .../datadog}/datadogplugin/ddpricing.go | 0 {datadog => pkg/plugins/datadog}/go.mod | 7 +- {datadog => pkg/plugins/datadog}/go.sum | 2 + .../plugins/datadog}/tests/datadog_test.go | 0 .../plugins/mongodb-atlas/cmd/main}/main.go | 6 +- .../mongodb-atlas/cmd/main}/main_test.go | 2 +- .../mongodb-atlas}/config/atlasconfig.go | 0 .../mongodb-atlas}/config/atlasconfig_test.go | 0 .../plugins/mongodb-atlas}/go.mod | 4 +- .../plugins/mongodb-atlas}/go.sum | 0 .../plugin/costexplorerquery.go | 0 {test => pkg/test}/go.mod | 6 +- {test => pkg/test}/go.sum | 10 + pkg/test/pkg/executor/main/main.go | 188 +++++++++++++++ {test => pkg/test}/pkg/harness/harness.go | 0 27 files changed, 691 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/run-integration-tests.yaml create mode 100644 .github/workflows/update-manifest.yaml delete mode 100644 .vscode/launch.json rename {common => pkg/common}/config/config_helpers.go (100%) rename {common => pkg/common}/go.mod (100%) rename {datadog => pkg/plugins/datadog}/cmd/main/main.go (82%) rename {datadog => pkg/plugins/datadog}/cmd/main/main_test.go (89%) create mode 100644 pkg/plugins/datadog/cmd/validator/main/main.go rename {datadog => pkg/plugins/datadog}/datadogplugin/datadogconfig.go (100%) rename {datadog => pkg/plugins/datadog}/datadogplugin/ddpricing.go (100%) rename {datadog => pkg/plugins/datadog}/go.mod (90%) rename {datadog => pkg/plugins/datadog}/go.sum (98%) rename {datadog => pkg/plugins/datadog}/tests/datadog_test.go (100%) rename {mongodb-atlas/cmd => pkg/plugins/mongodb-atlas/cmd/main}/main.go (97%) rename {mongodb-atlas/cmd => pkg/plugins/mongodb-atlas/cmd/main}/main_test.go (99%) rename {mongodb-atlas => pkg/plugins/mongodb-atlas}/config/atlasconfig.go (100%) rename {mongodb-atlas => pkg/plugins/mongodb-atlas}/config/atlasconfig_test.go (100%) rename {mongodb-atlas => pkg/plugins/mongodb-atlas}/go.mod (95%) rename {mongodb-atlas => pkg/plugins/mongodb-atlas}/go.sum (100%) rename {mongodb-atlas => pkg/plugins/mongodb-atlas}/plugin/costexplorerquery.go (100%) rename {test => pkg/test}/go.mod (87%) rename {test => pkg/test}/go.sum (92%) create mode 100644 pkg/test/pkg/executor/main/main.go rename {test => pkg/test}/pkg/harness/harness.go (100%) diff --git a/.github/workflows/run-integration-tests.yaml b/.github/workflows/run-integration-tests.yaml new file mode 100644 index 0000000..98386b3 --- /dev/null +++ b/.github/workflows/run-integration-tests.yaml @@ -0,0 +1,66 @@ +name: Run Integration Tests + +on: + push: + branches: + - main + schedule: + - cron: "0 5 * * *" + +jobs: + integration-test-runner: + runs-on: ubuntu-latest + steps: + - uses: oNaiPs/secrets-to-env-action@v1 + with: + secrets: ${{ toJSON(secrets) }} + - uses: actions/checkout@v4 + with: + path: ./ + ref: main + - name: Install just + uses: extractions/setup-just@v2 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "stable" + - name: prep workspace + run: | + just init-workspace + - name: run integration tests + run: | + just integration-test-all-plugins + + + notify-fail: + needs: [integration-test-runner] + runs-on: ubuntu-latest + if: failure() + steps: + - name: Slack notify + id: slack + uses: slackapi/slack-github-action@v1 + with: + payload: | + { + "workflow": "${{github.workflow}}", + "message": "Plugin Integration tests have failed! Please check the logs for more information." + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} + notify-success: + needs: [integration-test-runner] + runs-on: ubuntu-latest + if: success() + steps: + - name: Slack notify + id: slack + uses: slackapi/slack-github-action@v1 + with: + payload: | + { + "workflow": "${{github.workflow}}", + "message": "Plugin Integration tests have passed! :tada:" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} \ No newline at end of file diff --git a/.github/workflows/update-manifest.yaml b/.github/workflows/update-manifest.yaml new file mode 100644 index 0000000..f877045 --- /dev/null +++ b/.github/workflows/update-manifest.yaml @@ -0,0 +1,47 @@ +name: Update Manifest + +on: + push: + branches: + - main + +jobs: + update-manifest: + runs-on: ubuntu-latest + steps: + - name: echo actor + run: echo "actor is ${{ github.actor }}" + - uses: actions/checkout@v4 + with: + path: ./ + ssh-key: ${{ secrets.GH_SSH_KEY}} + persist-credentials: true + - name: update current plugin list + run: | + plugins=$(ls pkg/plugins | tr -s ' ' | sort) + echo "got plugins: $plugins" + echo '# this manifest contains the name of every currently implemented plugin. it can be pulled via https://github.com/opencost/opencost-plugins/raw/main/manifest to get an up to date list of current plugins. + ' > manifest + echo "$plugins" >> manifest + echo "manifest updated" + echo "manifest contents:" + cat manifest + + - name: Commit changes + run: | + git config --local user.email "github-actions[bot]@noreply.example.com" + git config --local user.name "github-actions[bot]" + git add manifest || true + git commit -m "update manifest" || true + + - name: update manifest + uses: ad-m/github-push-action@master + with: + ssh: true + directory: . + repository: opencost/opencost-plugins + branch: main + + + + \ No newline at end of file diff --git a/.gitignore b/.gitignore index bd922fd..cf1836e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build *.out -mongodb-atlas/scripts \ No newline at end of file +mongodb-atlas/scripts +.vscode \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index abba819..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - { - "name": "Launch Package", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${fileDirname}", - "args": ["/Users/sajit/Desktop/atlas-config.json"] - } - ] -} \ No newline at end of file diff --git a/README.md b/README.md index 10fa38c..336ece2 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,6 @@ Once the configuration is designed, it's time to write the plugin. Within ` ") + os.Exit(1) + } + + dailyProtobufFilePath := os.Args[1] + + // read in the protobuf file + data, err := os.ReadFile(dailyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading daily protobuf file: %v\n", err) + os.Exit(1) + } + + dailyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) + + // second arg is the path to the hourly protobuf file + hourlyProtobufFilePath := os.Args[2] + + data, err = os.ReadFile(hourlyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading hourly protobuf file: %v\n", err) + os.Exit(1) + } + + // read in the protobuf file + hourlyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) + + // validate the custom cost response data + isvalid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isvalid { + os.Exit(1) + } else { + fmt.Println("Validation successful") + } +} + +func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { + if len(respDaily) == 0 { + log.Errorf("no daily response received from datadog plugin") + return false + } + + if len(respHourly) == 0 { + log.Errorf("no hourly response received from datadog plugin") + return false + } + + var multiErr error + + // parse the response and look for errors + for _, resp := range respDaily { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) + } + } + + for _, resp := range respHourly { + if resp.Errors != nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors)) + } + } + + // check if any errors occurred + if multiErr != nil { + log.Errorf("Errors occurred during plugin testing for datadog: %v", multiErr) + return false + } + + dbmCostsInRange := 0 + //verify that the returned costs are non zero + for _, resp := range respDaily { + var costSum float32 + for _, cost := range resp.Costs { + costSum += cost.GetListCost() + if cost.GetListCost() > 100 { + log.Errorf("daily cost returned by plugin datadog for %v is greater than 100", cost) + return false + } + + //as of 10/2024, dbm hosts cost $84 a month or about $2.70. confirm that + // range + if cost.GetResourceName() == "dbm_host_count" { + // filter out recent costs since those might not be full days worth + if cost.GetListCost() > 2.5 && cost.GetListCost() < 3.0 { + dbmCostsInRange++ + } + } + } + if costSum == 0 { + log.Errorf("daily costs returned by datadog plugin are zero") + return false + } + + } + + if dbmCostsInRange == 0 { + log.Errorf("no dbm costs in expected range found in daily costs") + return false + } + + seenCosts := map[string]bool{} + for _, resp := range respHourly { + for _, cost := range resp.Costs { + seenCosts[cost.GetResourceName()] = true + if cost.GetListCost() == 0 { + log.Errorf("hourly cost returned by plugin datadog is zero") + return false + } + } + } + + expectedCosts := []string{ + "agent_host_count", + "logs_indexed_events_15_day_count", + "container_count_excl_agent", + "agent_container", + "dbm_host_count", + } + + for _, cost := range expectedCosts { + if !seenCosts[cost] { + log.Errorf("hourly cost %s not found in plugin datadog response", cost) + return false + } + } + + if len(seenCosts) != len(expectedCosts) { + log.Errorf("hourly costs returned by plugin datadog do not equal expected costs") + log.Errorf("seen costs: %v", seenCosts) + log.Errorf("expected costs: %v", expectedCosts) + + log.Errorf("response: %v", respHourly) + return false + } + + // verify the domain matches the plugin name + for _, resp := range respDaily { + if resp.Domain != "datadog" { + log.Errorf("daily domain returned by plugin datadog does not match plugin name") + return false + } + } + + seenCosts = map[string]bool{} + for _, resp := range respHourly { + for _, cost := range resp.Costs { + seenCosts[cost.GetResourceName()] = true + if cost.GetListCost() == 0 { + log.Errorf("hourly cost returned by plugin datadog is zero") + return false + } + if cost.GetListCost() > 100 { + log.Errorf("hourly cost returned by plugin datadog for %v is greater than 100", cost) + return false + } + } + } + + for _, cost := range expectedCosts { + if !seenCosts[cost] { + log.Errorf("daily cost %s not found in plugin datadog response", cost) + return false + } + } + return true +} + +func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + protoResps := make([]*pb.CustomCostResponse, len(raw)) + for i, r := range raw { + p := &pb.CustomCostResponse{} + if err := protojson.Unmarshal(r, p); err != nil { + return nil, err + } + protoResps[i] = p + } + + return protoResps, nil +} diff --git a/datadog/datadogplugin/datadogconfig.go b/pkg/plugins/datadog/datadogplugin/datadogconfig.go similarity index 100% rename from datadog/datadogplugin/datadogconfig.go rename to pkg/plugins/datadog/datadogplugin/datadogconfig.go diff --git a/datadog/datadogplugin/ddpricing.go b/pkg/plugins/datadog/datadogplugin/ddpricing.go similarity index 100% rename from datadog/datadogplugin/ddpricing.go rename to pkg/plugins/datadog/datadogplugin/ddpricing.go diff --git a/datadog/go.mod b/pkg/plugins/datadog/go.mod similarity index 90% rename from datadog/go.mod rename to pkg/plugins/datadog/go.mod index 31273df..10c175f 100644 --- a/datadog/go.mod +++ b/pkg/plugins/datadog/go.mod @@ -1,13 +1,15 @@ -module github.com/opencost/opencost-plugins/datadog +module github.com/opencost/opencost-plugins/pkg/plugins/datadog go 1.22.2 -replace github.com/opencost/opencost-plugins/common => ../common +replace github.com/opencost/opencost-plugins/pkg/common => ../../common require ( github.com/DataDog/datadog-api-client-go/v2 v2.23.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/hashicorp/go-plugin v1.6.0 + github.com/opencost/opencost-plugins/datadog v0.0.0-20240429172518-a50cd1290864 + github.com/opencost/opencost-plugins/pkg/common v0.0.0-00010101000000-000000000000 github.com/opencost/opencost-plugins/test v0.0.0-20240307142929-df4df8ee69fa github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a golang.org/x/time v0.5.0 @@ -38,7 +40,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/rs/zerolog v1.32.0 // indirect diff --git a/datadog/go.sum b/pkg/plugins/datadog/go.sum similarity index 98% rename from datadog/go.sum rename to pkg/plugins/datadog/go.sum index 39f4793..4137a78 100644 --- a/datadog/go.sum +++ b/pkg/plugins/datadog/go.sum @@ -82,6 +82,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/opencost/opencost-plugins/datadog v0.0.0-20240429172518-a50cd1290864 h1:RfjDGBV2GfsB2jgJ1+0N+TeSysg080tGxAyd9pz/QIc= +github.com/opencost/opencost-plugins/datadog v0.0.0-20240429172518-a50cd1290864/go.mod h1:8zKr+wkfcFJeS955K4wyQ8FiSdNQHSl+ZZajs5tUkyk= github.com/opencost/opencost-plugins/test v0.0.0-20240307142929-df4df8ee69fa h1:hzsLLTOech6pPdWMtQT/D9pa8eZu52VWKqnPdH4mMZQ= github.com/opencost/opencost-plugins/test v0.0.0-20240307142929-df4df8ee69fa/go.mod h1:6B39vhLt/C97HxMoTZehNF9cf9z7qRd9EzKUYKgvPg4= github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a h1:m6sesjHd7phuhoWhrCXrzLKHJbAdlH0Q07Uvpbgl4G0= diff --git a/datadog/tests/datadog_test.go b/pkg/plugins/datadog/tests/datadog_test.go similarity index 100% rename from datadog/tests/datadog_test.go rename to pkg/plugins/datadog/tests/datadog_test.go diff --git a/mongodb-atlas/cmd/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go similarity index 97% rename from mongodb-atlas/cmd/main.go rename to pkg/plugins/mongodb-atlas/cmd/main/main.go index 02fe7cf..1b914b1 100644 --- a/mongodb-atlas/cmd/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -11,8 +11,8 @@ import ( "github.com/hashicorp/go-plugin" "github.com/icholy/digest" commonconfig "github.com/opencost/opencost-plugins/common/config" - atlasconfig "github.com/opencost/opencost-plugins/mongodb-atlas/config" - atlasplugin "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" + atlasconfig "github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas/config" + atlasplugin "github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas/plugin" "github.com/opencost/opencost/core/pkg/log" "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" @@ -35,7 +35,7 @@ const costExplorerFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/ const costExplorerQueryFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage/%s" func main() { - fmt.Println("Initializing Mongo plugin") + log.Debug("Initializing Mongo plugin") configFile, err := commonconfig.GetConfigFilePath() if err != nil { diff --git a/mongodb-atlas/cmd/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go similarity index 99% rename from mongodb-atlas/cmd/main_test.go rename to pkg/plugins/mongodb-atlas/cmd/main/main_test.go index 44d0b86..e3fc2d0 100644 --- a/mongodb-atlas/cmd/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - atlasplugin "github.com/opencost/opencost-plugins/mongodb-atlas/plugin" + atlasplugin "github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas/plugin" "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" "google.golang.org/protobuf/types/known/durationpb" diff --git a/mongodb-atlas/config/atlasconfig.go b/pkg/plugins/mongodb-atlas/config/atlasconfig.go similarity index 100% rename from mongodb-atlas/config/atlasconfig.go rename to pkg/plugins/mongodb-atlas/config/atlasconfig.go diff --git a/mongodb-atlas/config/atlasconfig_test.go b/pkg/plugins/mongodb-atlas/config/atlasconfig_test.go similarity index 100% rename from mongodb-atlas/config/atlasconfig_test.go rename to pkg/plugins/mongodb-atlas/config/atlasconfig_test.go diff --git a/mongodb-atlas/go.mod b/pkg/plugins/mongodb-atlas/go.mod similarity index 95% rename from mongodb-atlas/go.mod rename to pkg/plugins/mongodb-atlas/go.mod index bf22ec6..46173a2 100644 --- a/mongodb-atlas/go.mod +++ b/pkg/plugins/mongodb-atlas/go.mod @@ -1,8 +1,8 @@ -module github.com/opencost/opencost-plugins/mongodb-atlas +module github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas go 1.22.5 -replace github.com/opencost/opencost-plugins/common => ../common +replace github.com/opencost/opencost-plugins/common => ../../common require github.com/icholy/digest v0.1.23 diff --git a/mongodb-atlas/go.sum b/pkg/plugins/mongodb-atlas/go.sum similarity index 100% rename from mongodb-atlas/go.sum rename to pkg/plugins/mongodb-atlas/go.sum diff --git a/mongodb-atlas/plugin/costexplorerquery.go b/pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go similarity index 100% rename from mongodb-atlas/plugin/costexplorerquery.go rename to pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go diff --git a/test/go.mod b/pkg/test/go.mod similarity index 87% rename from test/go.mod rename to pkg/test/go.mod index c797de3..aeff58c 100644 --- a/test/go.mod +++ b/pkg/test/go.mod @@ -1,4 +1,4 @@ -module github.com/opencost/opencost-plugins/test +module github.com/opencost/opencost-plugins/pkg/test go 1.21.6 @@ -6,14 +6,18 @@ require ( github.com/hashicorp/go-hclog v1.6.2 github.com/hashicorp/go-plugin v1.6.0 github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a + github.com/spf13/cobra v1.8.1 ) require ( github.com/fatih/color v1.16.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/test/go.sum b/pkg/test/go.sum similarity index 92% rename from test/go.sum rename to pkg/test/go.sum index 593d430..a7f4103 100644 --- a/test/go.sum +++ b/pkg/test/go.sum @@ -1,6 +1,7 @@ github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -17,14 +18,20 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -65,6 +72,7 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -75,6 +83,8 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= diff --git a/pkg/test/pkg/executor/main/main.go b/pkg/test/pkg/executor/main/main.go new file mode 100644 index 0000000..b9c6a14 --- /dev/null +++ b/pkg/test/pkg/executor/main/main.go @@ -0,0 +1,188 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/hashicorp/go-multierror" + harness "github.com/opencost/opencost-plugins/pkg/test/pkg/harness" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/spf13/cobra" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func main() { + var plugins []string + + var rootCmd = &cobra.Command{ + Use: "plugin-harness", + Short: "A test harness for opencost plugins", + Long: `This program will invoke each plugin in turn, and will confirm no errors, and that the returned costs are non-zero.`, + Run: func(cmd *cobra.Command, args []string) { + log.Info("running opencost plugin integration test harness") + log.Info("this program will invoke each plugin in turn, and then will call a validator to confirm the results.") + log.Info("it is up to plugin implementors to ensure that their plugins edge cases are covered by unit tests.") + log.Info("this harness requires the JSON config for each plugin to be present in secret env vars") + + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("error getting current working directory: %s", err) + } + log.Infof("current working directory: %s", cwd) + var validationErrors error + + // for each plugin given via a flag + for _, plugin := range plugins { + log.Infof("Testing plugin: %s", plugin) + + // write the config in PLUGIN_NAME_CONFIG out to a file + envVarName := fmt.Sprintf("%s_CONFIG", strings.ReplaceAll(strings.ToUpper(plugin), "-", "_")) + config := os.Getenv(envVarName) + if len(config) == 0 { + log.Fatalf("missing config for plugin %s", plugin) + } + + // write the config to a file + configDir := os.TempDir() + + file, err := os.CreateTemp(configDir, fmt.Sprintf("%s_config.json", plugin)) + if err != nil { + log.Fatalf("error creating temp file for plugin %s: %s", plugin, err) + } + defer os.RemoveAll(file.Name()) + + _, err = file.WriteString(config) + if err != nil { + log.Fatalf("error writing config for plugin %s: %s", plugin, err) + } + + // request usage for last week in daily increments + windowStart := time.Now().AddDate(0, 0, -7).Truncate(24 * time.Hour) + windowEnd := time.Now().Add(24 * time.Hour).Truncate(24 * time.Hour) + // invoke plugin via harness + + pluginPath := cwd + "/pkg/plugins/" + plugin + respDaily := getResponse(pluginPath, file.Name(), windowStart, windowEnd, 24*time.Hour) + + // request usage for 2 days ago in hourly increments + windowStart = time.Now().AddDate(0, 0, -2).Truncate(24 * time.Hour) + windowEnd = time.Now().AddDate(0, 0, -1).Truncate(24 * time.Hour) + // invoke plugin via harness + respHourly := getResponse(pluginPath, file.Name(), windowStart, windowEnd, 1*time.Hour) + + // call validator if implemented + validator := validatorPath(pluginPath) + if validator != "" { + // write hourly cost response to a file + hourlyBytes, err := marshal(respHourly) + if err != nil { + log.Fatalf("error marshalling hourly response for plugin %s: %s", plugin, err) + } + hourlyFile, err := os.CreateTemp("", fmt.Sprintf("%s_hourly_response_*.pb", plugin)) + if err != nil { + log.Fatalf("error creating temp file for hourly response for plugin %s: %s", plugin, err) + } + + _, err = hourlyFile.Write(hourlyBytes) + if err != nil { + log.Fatalf("error writing hourly response for plugin %s: %s", plugin, err) + } + + // write daily cost response to a file + dailyBytes, err := marshal(respDaily) + if err != nil { + log.Fatalf("error marshalling daily response for plugin %s: %s", plugin, err) + } + dailyFile, err := os.CreateTemp("", fmt.Sprintf("%s_daily_response_*.pb", plugin)) + if err != nil { + log.Fatalf("error creating temp file for daily response for plugin %s: %s", plugin, err) + } + + _, err = dailyFile.Write(dailyBytes) + if err != nil { + log.Fatalf("error writing daily response for plugin %s: %s", plugin, err) + } + + err = invokeValidator(validator, hourlyFile.Name(), dailyFile.Name()) + if err != nil { + validationErrors = multierror.Append(validationErrors, fmt.Errorf("error testing plugin %s: %w", plugin, err)) + } + + } else { + log.Infof("no validator found for plugin %s. Consider implementing a validator to improve the quality of the integration tests", plugin) + + } + + } + + if validationErrors != nil { + log.Fatalf("TESTS FAILED - validation errors: %s", validationErrors) + } + }, + } + + rootCmd.Flags().StringSliceVarP(&plugins, "plugins", "p", []string{}, "List of plugins to test (comma-separated)") + + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + } +} + +func invokeValidator(validatorPath, hourlyPath, dailyPath string) error { + // invoke validator + + // Create the command with the given arguments + cmd := exec.Command("go", "run", validatorPath, dailyPath, hourlyPath) + + // Run the command and capture the output + output, err := cmd.CombinedOutput() + if err != nil { + log.Errorf("error running validator command: %s\nOutput: %s", err, output) + return fmt.Errorf("error running validator command: %s, output: %s", err, output) + } + + // Print the output of the command + fmt.Printf("Validator output:\n%s\n", output) + return nil +} + +func marshal(protoResps []*pb.CustomCostResponse) ([]byte, error) { + raw := make([]json.RawMessage, len(protoResps)) + for i, p := range protoResps { + r, err := protojson.Marshal(p) + if err != nil { + return nil, err + } + raw[i] = r + } + + return json.Marshal(raw) +} + +func validatorPath(plugin string) string { + path := plugin + "/cmd/validator/main/main.go" + if _, err := os.Stat(path); os.IsNotExist(err) { + return "" + } + return path +} + +func getResponse(pluginPath, pathToConfigFile string, windowStart, windowEnd time.Time, step time.Duration) []*pb.CustomCostResponse { + + // invoke plugin via harness + pluginFile := pluginPath + "/cmd/main/main.go" + + req := pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(step), + } + return harness.InvokePlugin(pathToConfigFile, pluginFile, &req) +} diff --git a/test/pkg/harness/harness.go b/pkg/test/pkg/harness/harness.go similarity index 100% rename from test/pkg/harness/harness.go rename to pkg/test/pkg/harness/harness.go From 24aed3e0dc91f448c8e8aad34620682690f151a1 Mon Sep 17 00:00:00 2001 From: sajit Date: Fri, 11 Oct 2024 21:50:29 -0400 Subject: [PATCH 09/28] fix atlas_config_test Signed-off-by: sajit --- .../mongodb-atlas/config/atlasconfig_test.go | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/config/atlasconfig_test.go b/pkg/plugins/mongodb-atlas/config/atlasconfig_test.go index 06c8027..b47be6b 100644 --- a/pkg/plugins/mongodb-atlas/config/atlasconfig_test.go +++ b/pkg/plugins/mongodb-atlas/config/atlasconfig_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "testing" ) @@ -8,25 +9,25 @@ import ( // Unit tests for the GetAtlasConfig function func TestGetAtlasConfig(t *testing.T) { // Test: Valid configuration file - // t.Run("Valid configuration file", func(t *testing.T) { - // configFilePath := "test_valid_config.json" - // // Create a temporary valid JSON file - // validConfig := `{"log_level": "debug"}` - // err := os.WriteFile(configFilePath, []byte(validConfig), 0644) - // if err != nil { - // t.Fatalf("failed to create temporary config file: %v", err) - // } - // defer os.Remove(configFilePath) + t.Run("Valid configuration file", func(t *testing.T) { + configFilePath := "test_valid_config.json" + // Create a temporary valid JSON file + validConfig := `{"atlas_plugin_log_level": "debug"}` + err := os.WriteFile(configFilePath, []byte(validConfig), 0644) + if err != nil { + t.Fatalf("failed to create temporary config file: %v", err) + } + defer os.Remove(configFilePath) - // config, err := GetAtlasConfig(configFilePath) - // if err != nil { - // t.Fatalf("expected no error, but got: %v", err) - // } - // fmt.Println(config, configFilePath) - // if config.LogLevel != "debug" { - // t.Errorf("expected log level to be 'debug', but got: %s", config.LogLevel) - // } - // }) + config, err := GetAtlasConfig(configFilePath) + if err != nil { + t.Fatalf("expected no error, but got: %v", err) + } + fmt.Println(config, configFilePath) + if config.LogLevel != "debug" { + t.Errorf("expected log level to be 'debug', but got: %s", config.LogLevel) + } + }) // Test: Invalid file path t.Run("Invalid file path", func(t *testing.T) { From 0f305cf052e3ba3ce983fdcfdb6dfad1217d8f78 Mon Sep 17 00:00:00 2001 From: sajit Date: Sun, 13 Oct 2024 21:24:35 -0400 Subject: [PATCH 10/28] have a working version. Need to change the function signature Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 166 +++++----- .../mongodb-atlas/cmd/main/main_test.go | 288 +++++++----------- .../mongodb-atlas/plugin/costexplorerquery.go | 37 +++ 3 files changed, 226 insertions(+), 265 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index 1b914b1..1a17a2a 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "encoding/json" "fmt" "io" @@ -31,8 +30,7 @@ var handshakeConfig = plugin.HandshakeConfig{ MagicCookieValue: "mongodb-atlas", } -const costExplorerFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage" -const costExplorerQueryFmt = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/billing/costExplorer/usage/%s" +const costExplorerPendingInvoices = "https://cloud.mongodb.com/api/atlas/v2/orgs/%s/invoices/pending" func main() { log.Debug("Initializing Mongo plugin") @@ -90,9 +88,40 @@ type HTTPClient interface { Do(req *http.Request) (*http.Response, error) } +func validateRequest(req *pb.CustomCostRequest) []string { + var errors []string + now := time.Now() + // 1. Check if resolution is less than a day + if req.Resolution.AsDuration() < 24*time.Hour { + errors = append(errors, "Resolution should be at least one day.") + } + // Get the start of the current month + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + + // 2. Check if start time is before the start of the current month + if req.Start.AsTime().Before(currentMonthStart) { + errors = append(errors, "Start date cannot be before the current month. Historical costs not currently supported") + } + + // 3. Check if end time is before the start of the current month + if req.End.AsTime().Before(currentMonthStart) { + errors = append(errors, "End date cannot be before the current month. Historical costs not currently supported") + } + + return errors +} func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { results := []*pb.CustomCostResponse{} + requestErrors := validateRequest(req) + if len(requestErrors) > 0 { + errResp := pb.CustomCostResponse{ + Errors: requestErrors, + } + results = append(results, &errResp) + return results + } + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) if err != nil { log.Errorf("error getting windows: %v", err) @@ -102,6 +131,16 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom results = append(results, &errResp) return results } + costs, err := GetPendingInvoices(a.orgID, a.atlasClient) + if err != nil { + log.Errorf("Error fetching invoices: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error fetching invoices: %v", err)}, + } + results = append(results, &errResp) + return results + + } for _, target := range targets { if target.Start().After(time.Now().UTC()) { @@ -110,7 +149,7 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom } log.Debugf("fetching atlas costs for window %v", target) - result, err := a.getAtlasCostsForWindow(&target) + result, err := a.getAtlasCostsForWindow(&target, costs) if err != nil { log.Errorf("error getting costs for window %v: %v", target, err) errResp := pb.CustomCostResponse{} @@ -124,22 +163,34 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom return results } -func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.CustomCostResponse, error) { +func filterLineItemsByWindow(win *opencost.Window, lineItems []*pb.CustomCost) []*pb.CustomCost { + var filteredItems []*pb.CustomCost - // get the token - token, err := CreateCostExplorerQueryToken(a.orgID, *win.Start(), *win.End(), a.atlasClient) - if err != nil { - log.Errorf("error getting token: %v", err) - return nil, err - } + // Iterate over each line item + for _, item := range lineItems { + // Parse StartDate and EndDate from strings to time.Time + startDate, err1 := time.Parse("2006-01-02", item.StartDate) // Assuming date format is YYYY-MM-DD + endDate, err2 := time.Parse("2006-01-02", item.EndDate) // Same format assumption - // get the costs - costs, err := GetCosts(a.atlasClient, a.orgID, token) - if err != nil { - log.Errorf("error getting costs: %v", err) - return nil, err + if err1 != nil || err2 != nil { + // If parsing fails, skip this item + continue + } + + // Check if the item's StartDate >= win.start and EndDate <= win.end + if (win.start == nil || !startDate.Before(*win.start)) && (win.end == nil || !endDate.After(*win.end)) { + filteredItems = append(filteredItems, item) + } } + return filteredItems +} + +func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window, lineItems []*pb.CustomCost) (*pb.CustomCostResponse, error) { + + //filter responses between + costsInWindow := filterLineItemsByWindow(win, lineItems) + resp := pb.CustomCostResponse{ Metadata: map[string]string{"api_client_version": "v1"}, CostSource: "data_storage", @@ -149,57 +200,42 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window) (*pb.Cust Start: timestamppb.New(*win.Start()), End: timestamppb.New(*win.End()), Errors: []string{}, - Costs: costs, + Costs: costsInWindow, } return &resp, nil } -func GetCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, error) { - request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerQueryFmt, org, token), nil) +func GetPendingInvoices(org string, client HTTPClient) ([]*pb.CustomCost, error) { + request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerPendingInvoices, org), nil) request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") response, error := client.Do(request) - statusCode := response.StatusCode - //102 status code means processing - so repeat call 2 times to see if we get a response - for count := 1; count < 2 && statusCode == http.StatusProcessing; count++ { - // Sleep for 5 seconds before the next request - time.Sleep(5 * time.Second) - response, _ := client.Do(request) - statusCode = response.StatusCode - - } - - if statusCode == http.StatusProcessing { - msg := "timeout waiting for response" - return nil, fmt.Errorf(msg) - } if error != nil { - msg := fmt.Sprintf("getCostExplorerUsage: error from server: %v", error) + msg := fmt.Sprintf("getPending Invoices: error from server: %v", error) log.Errorf(msg) return nil, fmt.Errorf(msg) } + defer response.Body.Close() body, _ := io.ReadAll(response.Body) log.Debugf("response Body: %s", string(body)) - var costResponse atlasplugin.CostResponse - respUnmarshalError := json.Unmarshal([]byte(body), &costResponse) + var pendingInvoicesResponse atlasplugin.PendingInvoice + respUnmarshalError := json.Unmarshal([]byte(body), &pendingInvoicesResponse) if respUnmarshalError != nil { - msg := fmt.Sprintf("getCost: error unmarshalling response: %v", respUnmarshalError) + msg := fmt.Sprintf("pendingInvoices: error unmarshalling response: %v", respUnmarshalError) log.Errorf(msg) return nil, fmt.Errorf(msg) } var costs []*pb.CustomCost // Iterate over the UsageDetails in CostResponse - for _, invoice := range costResponse.UsageDetails { - // Create a new pb.CustomCost for each Invoice + for _, lineItem := range pendingInvoicesResponse.LineItems { + // Create a new pb.CustomCost for each LineItem + log.Debugf("Line item %v", lineItem) customCost := &pb.CustomCost{ - Id: invoice.InvoiceId, - AccountName: invoice.OrganizationName, - ChargeCategory: invoice.Service, - BilledCost: invoice.UsageAmount, + //TODO get mapping } // Append the customCost pointer to the slice @@ -207,47 +243,3 @@ func GetCosts(client HTTPClient, org string, token string) ([]*pb.CustomCost, er } return costs, nil } - -// pass list of orgs , start date, end date -func CreateCostExplorerQueryToken(org string, startDate time.Time, endDate time.Time, - client HTTPClient) (string, error) { - // Define the layout for the desired format - layout := "2006-01-02" - - // Convert the time.Time object to a string in yyyy-mm-dd format - startDateString := startDate.Format(layout) - endDateString := endDate.Format(layout) - - payload := atlasplugin.CreateCostExplorerQueryPayload{ - - EndDate: endDateString, - StartDate: startDateString, - Organizations: []string{org}, - } - payloadJson, _ := json.Marshal(payload) - - request, _ := http.NewRequest("POST", fmt.Sprintf(costExplorerFmt, org), bytes.NewBuffer(payloadJson)) - - request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") - request.Header.Set("Content-Type", "application/vnd.atlas.2023-01-01+json") - - response, error := client.Do(request) - if error != nil { - msg := fmt.Sprintf("createCostExplorerQueryToken: error from server: %v", error) - log.Errorf(msg) - return "", fmt.Errorf(msg) - - } - defer response.Body.Close() - - body, _ := io.ReadAll(response.Body) - log.Debugf("response Body: %s", string(body)) - var createCostExplorerQueryResponse atlasplugin.CreateCostExplorerQueryResponse - respUnmarshalError := json.Unmarshal([]byte(body), &createCostExplorerQueryResponse) - if respUnmarshalError != nil { - msg := fmt.Sprintf("createCostExplorerQueryToken: error unmarshalling response: %v", respUnmarshalError) - log.Errorf(msg) - return "", fmt.Errorf(msg) - } - return createCostExplorerQueryResponse.Token, nil -} diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go index e3fc2d0..6055e39 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -28,51 +28,6 @@ func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { return m.DoFunc(req) } -func TestCreateCostExplorerQueryToken(t *testing.T) { - // Mock data - org := "testOrg" - startDate := time.Date(2023, 9, 1, 0, 0, 0, 0, time.UTC) - endDate := time.Date(2023, 9, 30, 0, 0, 0, 0, time.UTC) - expectedToken := "mockToken" - - // Define the response that the mock client will return - mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ - Token: expectedToken, - } - mockResponseJson, _ := json.Marshal(mockResponse) - - // Create a mock HTTPClient that returns a successful response - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - // Verify that the request method and URL are correct - if req.Method != http.MethodPost { - t.Errorf("expected POST request, got %s", req.Method) - } - expectedURL := fmt.Sprintf(costExplorerFmt, org) - if req.URL.String() != expectedURL { - t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) - } - - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), - }, nil - }, - } - - // Call the function under test - token, err := CreateCostExplorerQueryToken(org, startDate, endDate, mockClient) - - // Assert that the function returned the expected token and no error - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if token != expectedToken { - t.Errorf("expected token %s, got %s", expectedToken, token) - } -} - // FOR INTEGRATION TESTING PURPOSES ONLY // expects 3 env variables to be set to work // mapuk = public key for mongodb atlas @@ -110,84 +65,46 @@ func TestCreateCostExplorerQueryToken(t *testing.T) { // } -func TestErrorFromServer(t *testing.T) { - - // Define the layout that matches the format of the date string - layout := "2006-01-02" - endTime, _ := time.Parse(layout, "2024-07-01") - startTime, _ := time.Parse(layout, "2023-12-01") - orgId := "1" - // Create a mock HTTPClient that returns a successful response - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusInternalServerError, - Body: io.NopCloser(bytes.NewBufferString("fake")), - }, nil - }, - } - _, err := CreateCostExplorerQueryToken(orgId, startTime, endTime, mockClient) - - assert.NotEmpty(t, err) - -} - -func TestCallToCreateCostExplorerQueryBadMessage(t *testing.T) { - - // Define the layout that matches the format of the date string - layout := "2006-01-02" - endTime, _ := time.Parse(layout, "2024-07-01") - startTime, _ := time.Parse(layout, "2023-12-01") - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBufferString("This ain't json")), - }, nil - }, - } - - _, error := CreateCostExplorerQueryToken("myOrg", startTime, endTime, mockClient) - assert.NotEmpty(t, error) - -} - // tests for getCosts -func TestGetCostsMultipleInvoices(t *testing.T) { - costResponse := atlasplugin.CostResponse{ - UsageDetails: []atlasplugin.Invoice{ +func TestGetCostsPendingInvoices(t *testing.T) { + pendingInvoiceResponse := atlasplugin.PendingInvoice{ + AmountBilledCents: 0, + AmountPaidCents: 0, + Created: "2024-10-01T02:00:26Z", + CreditsCents: 0, + EndDate: "2024-11-01T00:00:00Z", + Id: "66fb726b79b56205f9376437", + LineItems: []atlasplugin.LineItem{ { - InvoiceId: "INV001", - OrganizationId: "ORG123", - OrganizationName: "Acme Corp", - Service: "Compute", - UsageAmount: 120.50, - UsageDate: "2024-10-01", - }, - { - InvoiceId: "INV002", - OrganizationId: "ORG124", - OrganizationName: "Beta Corp", - Service: "Storage", - UsageAmount: 75.75, - UsageDate: "2024-10-02", + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: "2024-10-11T00:00:00Z", + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 6.035e-07, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: "2024-10-10T00:00:00Z", + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, }, + }, + Links: []atlasplugin.Link{ { - InvoiceId: "INV003", - OrganizationId: "ORG125", - OrganizationName: "Gamma Inc", - Service: "Networking", - UsageAmount: 50.00, - UsageDate: "2024-10-03", + Href: "https://cloud.mongodb.com/api/atlas/v2/orgs/66d7254246a21a41036ff2e9", + Rel: "self", }, }, + OrgId: "66d7254246a21a41036ff2e9", + SalesTaxCents: 0, + StartDate: "2024-10-01T00:00:00Z", + StartingBalanceCents: 0, + StatusName: "PENDING", + SubTotalCents: 0, + Updated: "2024-10-01T02:00:26Z", } - mockResponseJson, _ := json.Marshal(costResponse) + mockResponseJson, _ := json.Marshal(pendingInvoiceResponse) mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { @@ -195,7 +112,7 @@ func TestGetCostsMultipleInvoices(t *testing.T) { if req.Method != http.MethodGet { t.Errorf("expected GET request, got %s", req.Method) } - expectedURL := fmt.Sprintf(costExplorerQueryFmt, "myOrg", "t1") + expectedURL := fmt.Sprintf(costExplorerPendingInvoices, "myOrg") if req.URL.String() != expectedURL { t.Errorf("expected URL %s, got %s", expectedURL, req.URL.String()) } @@ -207,16 +124,17 @@ func TestGetCostsMultipleInvoices(t *testing.T) { }, nil }, } - costs, err := GetCosts(mockClient, "myOrg", "t1") + costs, err := GetPendingInvoices("myOrg", mockClient) assert.Nil(t, err) - assert.Equal(t, 3, len(costs)) + assert.Equal(t, 1, len(costs)) - for i, invoice := range costResponse.UsageDetails { - assert.Equal(t, invoice.InvoiceId, costs[i].Id) - assert.Equal(t, invoice.OrganizationName, costs[i].AccountName) - assert.Equal(t, invoice.Service, costs[i].ChargeCategory) - assert.Equal(t, invoice.UsageAmount, costs[i].BilledCost) - } + //TODO + // for i, invoice := range costResponse.UsageDetails { + // assert.Equal(t, invoice.InvoiceId, costs[i].Id) + // assert.Equal(t, invoice.OrganizationName, costs[i].AccountName) + // assert.Equal(t, invoice.Service, costs[i].ChargeCategory) + // assert.Equal(t, invoice.UsageAmount, costs[i].BilledCost) + // } } func TestGetCostErrorFromServer(t *testing.T) { @@ -231,7 +149,7 @@ func TestGetCostErrorFromServer(t *testing.T) { }, nil }, } - costs, err := GetCosts(mockClient, "myOrg", "t1") + costs, err := GetPendingInvoices("myOrg", mockClient) assert.NotEmpty(t, err) assert.Nil(t, costs) @@ -251,44 +169,12 @@ func TestGetCostsBadMessage(t *testing.T) { }, } - _, error := GetCosts(mockClient, "myOrg", "t1") + _, error := GetPendingInvoices("myOrd", mockClient) assert.NotEmpty(t, error) } -func TestRepeatCallTill200(t *testing.T) { - - var count = 0 - mockResponseJson := getCostResponseMock() - - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - count++ - - if count < 2 { - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusProcessing, - Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), - }, nil - - } else { - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), - }, nil - - } - - }, - } - - costs, err := GetCosts(mockClient, "myOrg", "t1") - assert.Nil(t, err) - assert.Equal(t, 1, len(costs)) -} - +// TODO delete this func getCostResponseMock() []byte { costResponse := atlasplugin.CostResponse{ UsageDetails: []atlasplugin.Invoice{ @@ -308,25 +194,6 @@ func getCostResponseMock() []byte { return mockResponseJson } -func TestStuckInProcessing(t *testing.T) { - - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - - // Return a mock response with status 200 and mock JSON body - return &http.Response{ - StatusCode: http.StatusProcessing, - Body: io.NopCloser(bytes.NewBufferString("")), - }, nil - - }, - } - - costs, err := GetCosts(mockClient, "myOrg", "t1") - assert.NotNil(t, err) - assert.Nil(t, costs) -} - func TestGetAtlasCostsForWindow(t *testing.T) { mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { @@ -416,3 +283,68 @@ func TestGetCosts(t *testing.T) { assert.Equal(t, 1, len(resp)) } + +func TestValidateRequest(t *testing.T) { + // Get current time and first day of the current month + now := time.Now() + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + + tests := []struct { + name string + req *pb.CustomCostRequest + expectedErrors []string + }{ + { + name: "Valid request", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart.Add(5 * time.Hour)), // Start in current month + End: timestamppb.New(currentMonthStart.Add(48 * time.Hour)), // End in current month + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution + }, + expectedErrors: []string{}, + }, + { + name: "Resolution less than a day", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart.Add(5 * time.Hour)), // Start in current month + End: timestamppb.New(currentMonthStart.Add(48 * time.Hour)), // End in current month + Resolution: durationpb.New(12 * time.Hour), // 12 hours resolution (error) + }, + expectedErrors: []string{"Resolution should be at least one day."}, + }, + { + name: "Start date before current month", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart.Add(-48 * time.Hour)), // Start before current month (error) + End: timestamppb.New(currentMonthStart.Add(48 * time.Hour)), // End in current month + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution + }, + expectedErrors: []string{"Start date cannot be before the current month. Historical costs not currently supported"}, + }, + { + name: "End date before current month", + req: &pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart.Add(5 * time.Hour)), // Start in current month + End: timestamppb.New(currentMonthStart.Add(-48 * time.Hour)), // End before current month (error) + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution + }, + expectedErrors: []string{"End date cannot be before the current month. Historical costs not currently supported"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + errors := validateRequest(tt.req) + + if len(errors) != len(tt.expectedErrors) { + t.Errorf("Expected %d errors, got %d", len(tt.expectedErrors), len(errors)) + } + + for i, err := range tt.expectedErrors { + if errors[i] != err { + t.Errorf("Expected error %q, got %q", err, errors[i]) + } + } + }) + } +} diff --git a/pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go b/pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go index c145229..aada164 100644 --- a/pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go +++ b/pkg/plugins/mongodb-atlas/plugin/costexplorerquery.go @@ -27,3 +27,40 @@ type Invoice struct { type CostResponse struct { UsageDetails []Invoice `json:"usageDetails"` } + +type PendingInvoice struct { + AmountBilledCents int32 `json:"amountBilledCents"` + AmountPaidCents int32 `json:"amountPaidCents"` + Created string `json:"created"` + CreditsCents int32 `json:"creditCents"` + Id string `json:"id"` + EndDate string `json:"endDate"` + LineItems []LineItem `json:"lineItems"` + Links []Link `json:"links"` + OrgId string `json:"orgId"` + SalesTaxCents int32 `json:"salesTaxCents"` + StartDate string `json:"startDate"` + StartingBalanceCents int32 `json:"startingBalanceCents"` + StatusName string `json:"statusName"` + SubTotalCents int32 `json:"subtotalCents"` + Updated string `json:"updated"` +} + +type Link struct { + Href string `json:"href"` + Rel string `json:"rel"` +} + +type LineItem struct { + ClusterName string `json:"clusterName"` + Created string `json:"created"` + EndDate string `json:"endDate"` + GroupId string `json:"groupId"` + GroupName string `json:"groupName"` + Quantity float32 `json:"quantity"` + SKU string `json:"sku"` + StartDate string `json:"startDate"` + TotalPriceCents int32 `json:"totalPriceCents"` + Unit string `json:"unit"` + UnitPriceDollars float32 `json:"unitPriceDollars"` +} From 9013769f36bbe9d0b1dcb789cee329fa7e388186 Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 14 Oct 2024 14:08:37 -0400 Subject: [PATCH 11/28] working unit tests Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 60 +++++--- .../mongodb-atlas/cmd/main/main_test.go | 142 ++++++++++++------ pkg/plugins/mongodb-atlas/go.mod | 20 +-- pkg/plugins/mongodb-atlas/go.sum | 21 ++- 4 files changed, 160 insertions(+), 83 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index 1a17a2a..3d2dcc1 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -18,6 +18,7 @@ import ( ocplugin "github.com/opencost/opencost/core/pkg/plugin" "golang.org/x/time/rate" "google.golang.org/protobuf/types/known/timestamppb" + "k8s.io/apimachinery/pkg/util/uuid" ) // handshakeConfigs are used to just do a basic handshake between @@ -131,7 +132,9 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom results = append(results, &errResp) return results } - costs, err := GetPendingInvoices(a.orgID, a.atlasClient) + + lineItems, err := GetPendingInvoices(a.orgID, a.atlasClient) + if err != nil { log.Errorf("Error fetching invoices: %v", err) errResp := pb.CustomCostResponse{ @@ -149,7 +152,7 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom } log.Debugf("fetching atlas costs for window %v", target) - result, err := a.getAtlasCostsForWindow(&target, costs) + result, err := a.getAtlasCostsForWindow(&target, lineItems) if err != nil { log.Errorf("error getting costs for window %v: %v", target, err) errResp := pb.CustomCostResponse{} @@ -163,32 +166,58 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom return results } -func filterLineItemsByWindow(win *opencost.Window, lineItems []*pb.CustomCost) []*pb.CustomCost { +func filterLineItemsByWindow(win *opencost.Window, lineItems []atlasplugin.LineItem) []*pb.CustomCost { var filteredItems []*pb.CustomCost + winStartUTC := win.Start().UTC() + winEndUTC := win.End().UTC() + log.Debugf("Item window %s %s", winStartUTC, winEndUTC) // Iterate over each line item for _, item := range lineItems { // Parse StartDate and EndDate from strings to time.Time - startDate, err1 := time.Parse("2006-01-02", item.StartDate) // Assuming date format is YYYY-MM-DD - endDate, err2 := time.Parse("2006-01-02", item.EndDate) // Same format assumption + startDate, err1 := time.Parse("2006-01-02T15:04:05Z07:00", item.StartDate) // Assuming date format is "2006-01-02T15:04:05Z07:00" + endDate, err2 := time.Parse("2006-01-02T15:04:05Z07:00", item.EndDate) // Same format assumption if err1 != nil || err2 != nil { // If parsing fails, skip this item continue } + // // Iterate over the UsageDetails in CostResponse + // for _, lineItem := range pendingInvoicesResponse.LineItems { + // Create a new pb.CustomCost for each LineItem + //log.Debugf("Line item %v", item) + customCost := &pb.CustomCost{ + + AccountName: item.GroupName, + ChargeCategory: "Usage", + Description: fmt.Sprintf("Usage for %s", item.SKU), + ResourceName: item.SKU, + Id: string(uuid.NewUUID()), + ProviderId: fmt.Sprintf("%s %s %s", item.GroupId, item.ClusterName, item.SKU), + BilledCost: float32(item.TotalPriceCents / 100), + ListCost: item.Quantity * item.UnitPriceDollars, + ListUnitPrice: item.UnitPriceDollars, + UsageQuantity: item.Quantity, + UsageUnit: item.Unit, + } + log.Debugf("Line Item %s %s", startDate.UTC(), endDate.UTC()) // Check if the item's StartDate >= win.start and EndDate <= win.end - if (win.start == nil || !startDate.Before(*win.start)) && (win.end == nil || !endDate.After(*win.end)) { - filteredItems = append(filteredItems, item) + if (startDate.UTC().After(winStartUTC) || startDate.UTC().Equal(winStartUTC)) && + (endDate.UTC().Before(winEndUTC) || endDate.UTC().Equal(winEndUTC)) { + // // Append the customCost pointer to the slice + filteredItems = append(filteredItems, customCost) } } return filteredItems + } -func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window, lineItems []*pb.CustomCost) (*pb.CustomCostResponse, error) { +func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window, lineItems []atlasplugin.LineItem) (*pb.CustomCostResponse, error) { //filter responses between + costsInWindow := filterLineItemsByWindow(win, lineItems) resp := pb.CustomCostResponse{ @@ -205,7 +234,7 @@ func (a *AtlasCostSource) getAtlasCostsForWindow(win *opencost.Window, lineItems return &resp, nil } -func GetPendingInvoices(org string, client HTTPClient) ([]*pb.CustomCost, error) { +func GetPendingInvoices(org string, client HTTPClient) ([]atlasplugin.LineItem, error) { request, _ := http.NewRequest("GET", fmt.Sprintf(costExplorerPendingInvoices, org), nil) request.Header.Set("Accept", "application/vnd.atlas.2023-01-01+json") @@ -229,17 +258,6 @@ func GetPendingInvoices(org string, client HTTPClient) ([]*pb.CustomCost, error) log.Errorf(msg) return nil, fmt.Errorf(msg) } - var costs []*pb.CustomCost - // Iterate over the UsageDetails in CostResponse - for _, lineItem := range pendingInvoicesResponse.LineItems { - // Create a new pb.CustomCost for each LineItem - log.Debugf("Line item %v", lineItem) - customCost := &pb.CustomCost{ - //TODO get mapping - } - // Append the customCost pointer to the slice - costs = append(costs, customCost) - } - return costs, nil + return pendingInvoicesResponse.LineItems, nil } diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go index 6055e39..4f33bb7 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -195,41 +195,47 @@ func getCostResponseMock() []byte { } func TestGetAtlasCostsForWindow(t *testing.T) { - mockClient := &MockHTTPClient{ - DoFunc: func(req *http.Request) (*http.Response, error) { - - costResponse := getCostResponseMock() - if req.Method == http.MethodGet { - //return costs - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(costResponse)), - }, nil - } else { - // Define the response that the mock client will return - mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ - Token: "fake", - } - mockResponseJson, _ := json.Marshal(mockResponse) - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), - }, nil - } - }, - } atlasCostSource := AtlasCostSource{ - orgID: "myOrg", - atlasClient: mockClient, + orgID: "myOrg", } // Define the start and end time for the window - startTime := time.Now().Add(-24 * time.Hour) // 24 hours ago - endTime := time.Now() // Now + day1 := time.Date(2024, time.October, 12, 0, 0, 0, 0, time.UTC) // Now + + day2 := time.Date(2024, time.October, 13, 0, 0, 0, 0, time.UTC) + day3 := time.Date(2024, time.October, 14, 0, 0, 0, 0, time.UTC) // Now + lineItems := []atlasplugin.LineItem{ + { + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: day3.Format("2006-01-02T15:04:05Z07:00"), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 6.035e-07, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: day2.Format("2006-01-02T15:04:05Z07:00"), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.02, + }, + { + ClusterName: "kubecost-mongo-dev-1", + Created: "2024-10-11T02:57:56Z", + EndDate: day2.Format("2006-01-02T15:04:05Z07:00"), + GroupId: "66d7254246a21a41036ff33e", + GroupName: "Project 0", + Quantity: 0.0555, + SKU: "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + StartDate: day1.Add(-24 * time.Hour).Format("2006-01-02T15:04:05Z07:00"), + TotalPriceCents: 0, + Unit: "GB", + UnitPriceDollars: 0.03, + }, + } // Create a new Window instance - window := opencost.NewWindow(&startTime, &endTime) - resp, error := atlasCostSource.getAtlasCostsForWindow(&window) + window := opencost.NewWindow(&day2, &day3) + resp, error := atlasCostSource.getAtlasCostsForWindow(&window, lineItems) assert.Nil(t, error) assert.True(t, resp != nil) assert.Equal(t, "data_storage", resp.CostSource) @@ -244,23 +250,12 @@ func TestGetCosts(t *testing.T) { DoFunc: func(req *http.Request) (*http.Response, error) { costResponse := getCostResponseMock() - if req.Method == http.MethodGet { - //return costs - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(costResponse)), - }, nil - } else { - // Define the response that the mock client will return - mockResponse := atlasplugin.CreateCostExplorerQueryResponse{ - Token: "fake", - } - mockResponseJson, _ := json.Marshal(mockResponse) - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), - }, nil - } + + //return costs + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBuffer(costResponse)), + }, nil }, } @@ -348,3 +343,58 @@ func TestValidateRequest(t *testing.T) { }) } } + +func TestFilterInvoicesOnWindow(t *testing.T) { + // Setup test data + //day3.Format("2006-01-02T15:04:05Z07:00") + windowStart := time.Date(2024, time.October, 1, 0, 0, 0, 0, time.UTC) + windowEnd := time.Date(2024, time.October, 31, 0, 0, 0, 0, time.UTC) + window := opencost.NewWindow(&windowStart, &windowEnd) + + lineItems := []atlasplugin.LineItem{ + {StartDate: "2024-10-05T00:00:00Z", EndDate: "2024-10-10T00:00:00Z", UnitPriceDollars: 1.0, GroupName: "kubecost0", + SKU: "0", ClusterName: "cluster-0", GroupId: "A", TotalPriceCents: 45, Quantity: 2, Unit: "GB"}, // Within window + {StartDate: "2024-09-01T00:00:00Z", EndDate: "2024-09-30T00:00:00Z"}, // Before window + {StartDate: "2024-11-01T00:00:00Z", EndDate: "2024-11-10T00:00:00Z"}, // After window + {StartDate: "2024-10-01T00:00:00Z", EndDate: "2024-10-31T00:00:00Z", UnitPriceDollars: 5}, // Exactly matching the window + {StartDate: "2024-10-15T00:00:00Z", EndDate: "2024-10-20T00:00:00Z", UnitPriceDollars: 2.45}, // Fully within window + {StartDate: "2024-09-25T00:00:00Z", EndDate: "2024-10-13T00:00:00Z"}, // Partially in window + {StartDate: "2024-10-12T00:00:00Z", EndDate: "2024-11-01T00:00:00Z"}, // Partially in window + } + + filteredItems := filterLineItemsByWindow(&window, lineItems) + + // Verify results + assert.Equal(t, 3, len(filteredItems), "Expected 3 line items to be filtered") + + //Check if the filtered items are the correct ones + expectedFilteredDates := []pb.CustomCost{ + { + ListUnitPrice: 1.0, + }, + { + ListUnitPrice: 5, + }, + { + ListUnitPrice: 2.45, + }, + } + + for i, item := range filteredItems { + assert.Equal(t, expectedFilteredDates[i].ListUnitPrice, item.ListUnitPrice, "Unit price mismatch") + + } + //assert mapping to CustomCost object + + assert.Equal(t, lineItems[0].GroupName, filteredItems[0].AccountName, "accout name mismatch") + assert.Equal(t, "Usage", filteredItems[0].ChargeCategory) + assert.Equal(t, "Usage for 0", filteredItems[0].Description) + assert.Equal(t, "0", filteredItems[0].ResourceName) + assert.NotNil(t, filteredItems[0].Id) + assert.NotNil(t, filteredItems[0].ProviderId) + + assert.InDelta(t, lineItems[0].TotalPriceCents/100, filteredItems[0].BilledCost, 0.01) + assert.InDelta(t, filteredItems[0].ListCost, lineItems[0].Quantity*lineItems[0].UnitPriceDollars, 0.01) + assert.Equal(t, lineItems[0].Quantity, filteredItems[0].UsageQuantity) + assert.Equal(t, filteredItems[0].UsageUnit, lineItems[0].Unit) +} diff --git a/pkg/plugins/mongodb-atlas/go.mod b/pkg/plugins/mongodb-atlas/go.mod index 46173a2..aed5aea 100644 --- a/pkg/plugins/mongodb-atlas/go.mod +++ b/pkg/plugins/mongodb-atlas/go.mod @@ -4,7 +4,16 @@ go 1.22.5 replace github.com/opencost/opencost-plugins/common => ../../common -require github.com/icholy/digest v0.1.23 +require ( + github.com/hashicorp/go-plugin v1.6.1 + github.com/icholy/digest v0.1.23 + github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 + github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 + github.com/stretchr/testify v1.9.0 + golang.org/x/time v0.6.0 + google.golang.org/protobuf v1.34.2 + k8s.io/apimachinery v0.25.3 +) require ( github.com/davecgh/go-spew v1.1.1 // indirect @@ -14,12 +23,11 @@ require ( github.com/goccy/go-json v0.9.11 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-plugin v1.6.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -31,8 +39,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/opencost/opencost-plugins/common v0.0.0-00010101000000-000000000000 // indirect - github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -42,22 +48,18 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.8.1 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/grpc v1.66.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.25.3 // indirect - k8s.io/apimachinery v0.25.3 // indirect k8s.io/klog/v2 v2.80.0 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect diff --git a/pkg/plugins/mongodb-atlas/go.sum b/pkg/plugins/mongodb-atlas/go.sum index 1cda2a6..77b2838 100644 --- a/pkg/plugins/mongodb-atlas/go.sum +++ b/pkg/plugins/mongodb-atlas/go.sum @@ -45,6 +45,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -150,6 +152,8 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -188,6 +192,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icholy/digest v0.1.23 h1:4hX2pIloP0aDx7RJW0JewhPPy3R8kU+vWKdxPsCCGtY= github.com/icholy/digest v0.1.23/go.mod h1:QNrsSGQ5v7v9cReDI0+eyjsXGUoRSUZQHeQ5C4XLa0Y= +github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -200,6 +206,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -233,10 +241,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c h1:g+AKEjkyTLzo3O6nu2iuCxy+baoXAiPSr3l3bn8J4Ho= -github.com/opencost/opencost/core v0.0.0-20240827181822-a4065411ba4c/go.mod h1:c1he7ogYA3J/m2BNbWD5FDLRTpUwuG8CGIyOkDV+i+s= github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830 h1:PDYQw0cygJ8ehn/AObpRVru4Cg718aGrDJQis4XfHWg= github.com/opencost/opencost/core v0.0.0-20240829194822-b82370afd830/go.mod h1:c1he7ogYA3J/m2BNbWD5FDLRTpUwuG8CGIyOkDV+i+s= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -459,8 +467,6 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -472,8 +478,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -606,7 +610,6 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= @@ -648,6 +651,8 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -688,3 +693,5 @@ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 011320b1203e385e62ead23293ea6945a386ceb4 Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 14 Oct 2024 14:50:46 -0400 Subject: [PATCH 12/28] unit tests Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 2 +- .../mongodb-atlas/cmd/main/main_test.go | 85 ++++++++++--------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index 3d2dcc1..214e586 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -97,7 +97,7 @@ func validateRequest(req *pb.CustomCostRequest) []string { errors = append(errors, "Resolution should be at least one day.") } // Get the start of the current month - currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) // 2. Check if start time is before the start of the current month if req.Start.AsTime().Before(currentMonthStart) { diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go index 4f33bb7..b7a0e70 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -124,17 +124,16 @@ func TestGetCostsPendingInvoices(t *testing.T) { }, nil }, } - costs, err := GetPendingInvoices("myOrg", mockClient) + lineItems, err := GetPendingInvoices("myOrg", mockClient) assert.Nil(t, err) - assert.Equal(t, 1, len(costs)) - - //TODO - // for i, invoice := range costResponse.UsageDetails { - // assert.Equal(t, invoice.InvoiceId, costs[i].Id) - // assert.Equal(t, invoice.OrganizationName, costs[i].AccountName) - // assert.Equal(t, invoice.Service, costs[i].ChargeCategory) - // assert.Equal(t, invoice.UsageAmount, costs[i].BilledCost) - // } + assert.Equal(t, 1, len(lineItems)) + + for _, invoice := range pendingInvoiceResponse.LineItems { + assert.Equal(t, "kubecost-mongo-dev-1", invoice.ClusterName) + assert.Equal(t, "66d7254246a21a41036ff33e", invoice.GroupId) + assert.Equal(t, "Project 0", invoice.GroupName) + //TODO add more asserts on the fields + } } func TestGetCostErrorFromServer(t *testing.T) { @@ -174,26 +173,6 @@ func TestGetCostsBadMessage(t *testing.T) { } -// TODO delete this -func getCostResponseMock() []byte { - costResponse := atlasplugin.CostResponse{ - UsageDetails: []atlasplugin.Invoice{ - - { - InvoiceId: "INV003", - OrganizationId: "ORG125", - OrganizationName: "Gamma Inc", - Service: "Networking", - UsageAmount: 50.00, - UsageDate: "2024-10-03", - }, - }, - } - - mockResponseJson, _ := json.Marshal(costResponse) - return mockResponseJson -} - func TestGetAtlasCostsForWindow(t *testing.T) { atlasCostSource := AtlasCostSource{ @@ -246,15 +225,37 @@ func TestGetAtlasCostsForWindow(t *testing.T) { } func TestGetCosts(t *testing.T) { + pendingInvoiceResponse := atlasplugin.PendingInvoice{ + AmountBilledCents: 0, + AmountPaidCents: 0, + Created: "2024-10-01T02:00:26Z", + CreditsCents: 0, + EndDate: "2024-11-01T00:00:00Z", + Id: "66fb726b79b56205f9376437", + LineItems: []atlasplugin.LineItem{}, + Links: []atlasplugin.Link{ + { + Href: "https://cloud.mongodb.com/api/atlas/v2/orgs/66d7254246a21a41036ff2e9", + Rel: "self", + }, + }, + OrgId: "66d7254246a21a41036ff2e9", + SalesTaxCents: 0, + StartDate: "2024-10-01T00:00:00Z", + StartingBalanceCents: 0, + StatusName: "PENDING", + SubTotalCents: 0, + Updated: "2024-10-01T02:00:26Z", + } + + mockResponseJson, _ := json.Marshal(pendingInvoiceResponse) mockClient := &MockHTTPClient{ DoFunc: func(req *http.Request) (*http.Response, error) { - costResponse := getCostResponseMock() - //return costs return &http.Response{ StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewBuffer(costResponse)), + Body: io.NopCloser(bytes.NewBuffer(mockResponseJson)), }, nil }, @@ -264,19 +265,21 @@ func TestGetCosts(t *testing.T) { atlasClient: mockClient, } // Define the start and end time for the window - startTime := time.Now().Add(-24 * time.Hour) // 24 hours ago - endTime := time.Now() + now := time.Now() + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) customCostRequest := pb.CustomCostRequest{ - Start: timestamppb.New(startTime), - End: timestamppb.New(endTime), - Resolution: durationpb.New(time.Hour), // Example resolution: 1 hour - } // Now + Start: timestamppb.New(currentMonthStart), // Start in current month + End: timestamppb.New(currentMonthStart.Add(48 * time.Hour)), // End in current month + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution - resp := atlasCostSource.GetCustomCosts(&customCostRequest) + } - assert.Equal(t, 1, len(resp)) + resp := atlasCostSource.GetCustomCosts(&customCostRequest) + assert.Equal(t, 2, len(resp)) + assert.True(t, len(resp[0].Costs) == 0) + assert.True(t, len(resp[1].Costs) == 0) } func TestValidateRequest(t *testing.T) { From 493bc4efc65cf79fdcf186b4844dd5cd22487c0a Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 14 Oct 2024 14:57:37 -0400 Subject: [PATCH 13/28] optional integration test Signed-off-by: sajit --- .../mongodb-atlas/cmd/main/main_test.go | 74 +++++++++++-------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go index b7a0e70..77dbb28 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -6,9 +6,11 @@ import ( "fmt" "io" "net/http" + "os" "testing" "time" + "github.com/icholy/digest" atlasplugin "github.com/opencost/opencost-plugins/pkg/plugins/mongodb-atlas/plugin" "github.com/opencost/opencost/core/pkg/model/pb" "github.com/opencost/opencost/core/pkg/opencost" @@ -33,37 +35,47 @@ func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { // mapuk = public key for mongodb atlas // maprk = private key for mongodb atlas // maOrgId = orgId to be testsed -// func TestMain(t *testing.T) { - -// publicKey := os.Getenv("mapuk") -// privateKey := os.Getenv("maprk") -// orgId := os.Getenv("maorgid") -// if publicKey == "" || privateKey == "" || orgId == "" { -// t.Skip("Skipping integration test.") -// } - -// assert.NotNil(t, publicKey) -// assert.NotNil(t, privateKey) -// assert.NotNil(t, orgId) - -// client := &http.Client{ -// Transport: &digest.Transport{ -// Username: publicKey, -// Password: privateKey, -// }, -// } - -// // Define the layout that matches the format of the date string -// layout := "2006-01-02" -// endTime, _ := time.Parse(layout, "2024-07-01") -// startTime, _ := time.Parse(layout, "2023-12-01") -// url := "https://cloud.mongodb.com/api/atlas/v2/orgs/" + orgId + "/billing/costExplorer/usage" -// resp, err := createCostExplorerQueryToken(orgId, startTime, endTime, client) - -// assert.NotEmpty(t, resp) -// assert.Nil(t, err) - -// } +func TestMain(t *testing.T) { + + publicKey := os.Getenv("mapuk") + privateKey := os.Getenv("maprk") + orgId := os.Getenv("maorgid") + if publicKey == "" || privateKey == "" || orgId == "" { + t.Skip("Skipping integration test.") + } + + assert.NotNil(t, publicKey) + assert.NotNil(t, privateKey) + assert.NotNil(t, orgId) + + client := &http.Client{ + Transport: &digest.Transport{ + + Username: publicKey, + Password: privateKey, + }, + } + + atlasCostSource := AtlasCostSource{ + orgID: "myOrg", + atlasClient: client, + } + // Define the start and end time for the window + now := time.Now() + currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + + customCostRequest := pb.CustomCostRequest{ + Start: timestamppb.New(currentMonthStart), // Start in current month + End: timestamppb.New(currentMonthStart.Add(24 * time.Hour)), // End in current month + Resolution: durationpb.New(24 * time.Hour), // 1 day resolution + + } + + resp := atlasCostSource.GetCustomCosts(&customCostRequest) + + assert.NotEmpty(t, resp) + +} // tests for getCosts func TestGetCostsPendingInvoices(t *testing.T) { From 2581ed93e783462e42a50736eb10da52aff911ab Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Wed, 2 Oct 2024 13:05:11 -0400 Subject: [PATCH 14/28] Fixing DCO error Signed-off-by: sajit Signed-off-by: Alex Meijer Signed-off-by: sajit --- .github/workflows/run-integration-tests.yaml | 2 +- justfile | 3 --- pkg/plugins/datadog/cmd/main/main.go | 5 +++-- .../datadog/cmd/validator/main/main.go | 20 +++++++++++++++---- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-integration-tests.yaml b/.github/workflows/run-integration-tests.yaml index 98386b3..0182601 100644 --- a/.github/workflows/run-integration-tests.yaml +++ b/.github/workflows/run-integration-tests.yaml @@ -63,4 +63,4 @@ jobs: "message": "Plugin Integration tests have passed! :tada:" } env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} diff --git a/justfile b/justfile index 77e5615..6278326 100644 --- a/justfile +++ b/justfile @@ -22,9 +22,6 @@ integration-test-all-plugins: echo "pluginPaths: {{pluginPaths}}" {{commonenv}} go run pkg/test/pkg/executor/main/main.go --plugins={{pluginPaths}} -integration-test-plugin pluginName: - {{commonenv}} go run pkg/test/pkg/executor/main/main.go --plugins={{pluginName}} - clean: rm -rf ./build diff --git a/pkg/plugins/datadog/cmd/main/main.go b/pkg/plugins/datadog/cmd/main/main.go index 7ae5b2e..d3253c0 100644 --- a/pkg/plugins/datadog/cmd/main/main.go +++ b/pkg/plugins/datadog/cmd/main/main.go @@ -677,7 +677,7 @@ func scrapeDatadogPrices(url string) (*datadogplugin.PricingInformation, error) } response.Body.Close() res := datadogplugin.DatadogProJSON{} - r := regexp.MustCompile(`var productDetailData = \s*(.*?)\s*;`) + r := regexp.MustCompile(`var productDetailData = \s*(.*?)\s*};`) log.Tracef("got response: %s", string(b)) matches := r.FindAllStringSubmatch(string(b), -1) if len(matches) != 1 { @@ -688,7 +688,8 @@ func scrapeDatadogPrices(url string) (*datadogplugin.PricingInformation, error) } log.Tracef("matches[0][1]:" + matches[0][1]) - err = json.Unmarshal([]byte(matches[0][1]), &res) + // add back in the closing curly brace that was used to pattern match + err = json.Unmarshal([]byte(matches[0][1]+"}"), &res) if err != nil { errTry = err log.Errorf("failed to read pricing page body: %v", err) diff --git a/pkg/plugins/datadog/cmd/validator/main/main.go b/pkg/plugins/datadog/cmd/validator/main/main.go index 2bd9f39..515f421 100644 --- a/pkg/plugins/datadog/cmd/validator/main/main.go +++ b/pkg/plugins/datadog/cmd/validator/main/main.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "time" "github.com/hashicorp/go-multierror" "github.com/opencost/opencost/core/pkg/log" @@ -105,9 +106,17 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { dbmCostsInRange := 0 //verify that the returned costs are non zero for _, resp := range respDaily { + if len(resp.Costs) == 0 && resp.Start.AsTime().After(time.Now().Truncate(24*time.Hour).Add(-1*time.Minute)) { + log.Debugf("today's daily costs returned by plugin datadog are empty, skipping: %v", resp) + continue + } var costSum float32 for _, cost := range resp.Costs { costSum += cost.GetListCost() + + if cost.GetListCost() == 0 { + log.Debugf("got zero cost for %v", cost) + } if cost.GetListCost() > 100 { log.Errorf("daily cost returned by plugin datadog for %v is greater than 100", cost) return false @@ -136,13 +145,16 @@ func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { seenCosts := map[string]bool{} for _, resp := range respHourly { + var costSum float32 for _, cost := range resp.Costs { seenCosts[cost.GetResourceName()] = true - if cost.GetListCost() == 0 { - log.Errorf("hourly cost returned by plugin datadog is zero") - return false - } + costSum += cost.GetListCost() } + if costSum == 0 { + log.Errorf("hourly cost returned by plugin datadog is zero") + return false + } + } expectedCosts := []string{ From 7e562b1b06dca58918e8d44f0f579eda0dc9cd4e Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Wed, 2 Oct 2024 14:20:21 -0400 Subject: [PATCH 15/28] checkout current branch for now Signed-off-by: Alex Meijer Signed-off-by: sajit --- .github/workflows/run-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-integration-tests.yaml b/.github/workflows/run-integration-tests.yaml index 0182601..6d82714 100644 --- a/.github/workflows/run-integration-tests.yaml +++ b/.github/workflows/run-integration-tests.yaml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: path: ./ - ref: main + #ref: main - name: Install just uses: extractions/setup-just@v2 - name: Install Go From dbaf3f6ea873770a637c86e9c040be0d6602f508 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Wed, 2 Oct 2024 14:58:03 -0400 Subject: [PATCH 16/28] JPHM Signed-off-by: sajit prep for testing, fix build Signed-off-by: Alex Meijer --- .github/workflows/run-integration-tests.yaml | 2 +- tools/build-plugins | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run-integration-tests.yaml b/.github/workflows/run-integration-tests.yaml index 6d82714..0182601 100644 --- a/.github/workflows/run-integration-tests.yaml +++ b/.github/workflows/run-integration-tests.yaml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: path: ./ - #ref: main + ref: main - name: Install just uses: extractions/setup-just@v2 - name: Install Go diff --git a/tools/build-plugins b/tools/build-plugins index 0d3ae1b..c797981 100755 --- a/tools/build-plugins +++ b/tools/build-plugins @@ -26,26 +26,26 @@ fi echo $(pwd) plugin_name=$(basename $(dirname $gomod)) echo "plugin name: $plugin_name" -cd $plugin_name -GOOS=linux GOARCH=arm64 go build -x -o "../build/$plugin_name.ocplugin.linux.arm64" \ +cd pkg/plugins/$plugin_name +GOOS=linux GOARCH=arm64 go build -o "../../../build/$plugin_name.ocplugin.linux.arm64" \ -ldflags \ "-X github.com/opencost/opencost/pkg/version.Version=$VERSION \ -X github.com/opencost/opencost/pkg/version.GitCommit=$COMMIT" \ ./cmd/main -GOOS=linux GOARCH=amd64 go build -x -o "../build/$plugin_name.ocplugin.linux.amd64" \ +GOOS=linux GOARCH=amd64 go build -o "../../../build/$plugin_name.ocplugin.linux.amd64" \ -ldflags \ "-X github.com/opencost/opencost/pkg/version.Version=$VERSION \ -X github.com/opencost/opencost/pkg/version.GitCommit=$COMMIT" \ ./cmd/main -GOOS=darwin GOARCH=arm64 go build -x -o "../build/$plugin_name.ocplugin.darwin.arm64" \ +GOOS=darwin GOARCH=arm64 go build -o "../../../build/$plugin_name.ocplugin.darwin.arm64" \ -ldflags \ "-X github.com/opencost/opencost/pkg/version.Version=$VERSION \ -X github.com/opencost/opencost/pkg/version.GitCommit=$COMMIT" \ ./cmd/main -GOOS=darwin GOARCH=amd64 go build -x -o "../build/$plugin_name.ocplugin.darwin.amd64" \ +GOOS=darwin GOARCH=amd64 go build -o "../../../build/$plugin_name.ocplugin.darwin.amd64" \ -ldflags \ "-X github.com/opencost/opencost/pkg/version.Version=$VERSION \ -X github.com/opencost/opencost/pkg/version.GitCommit=$COMMIT" \ From 2cde7378a0a893df0609daa7bafcc160141f5575 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Wed, 2 Oct 2024 16:30:28 -0400 Subject: [PATCH 17/28] repro on branch Signed-off-by: Alex Meijer --- pkg/test/pkg/executor/main/main.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/test/pkg/executor/main/main.go b/pkg/test/pkg/executor/main/main.go index b9c6a14..ee57d2b 100644 --- a/pkg/test/pkg/executor/main/main.go +++ b/pkg/test/pkg/executor/main/main.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/exec" - "strings" "time" "github.com/hashicorp/go-multierror" @@ -43,12 +42,13 @@ func main() { log.Infof("Testing plugin: %s", plugin) // write the config in PLUGIN_NAME_CONFIG out to a file - envVarName := fmt.Sprintf("%s_CONFIG", strings.ReplaceAll(strings.ToUpper(plugin), "-", "_")) - config := os.Getenv(envVarName) - if len(config) == 0 { - log.Fatalf("missing config for plugin %s", plugin) - } + // envVarName := fmt.Sprintf("%s_CONFIG", strings.ReplaceAll(strings.ToUpper(plugin), "-", "_")) + // config := os.Getenv(envVarName) + // if len(config) == 0 { + // log.Fatalf("missing config for plugin %s", plugin) + // } + config := "{}" // write the config to a file configDir := os.TempDir() From f74a1f025339ecdce7af2a324d7e8be80ccbfe5c Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Wed, 2 Oct 2024 16:35:40 -0400 Subject: [PATCH 18/28] use branch Signed-off-by: Alex Meijer Signed-off-by: sajit --- .github/workflows/run-integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-integration-tests.yaml b/.github/workflows/run-integration-tests.yaml index 0182601..6d82714 100644 --- a/.github/workflows/run-integration-tests.yaml +++ b/.github/workflows/run-integration-tests.yaml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 with: path: ./ - ref: main + #ref: main - name: Install just uses: extractions/setup-just@v2 - name: Install Go From 9f5de8c5cacad132b98742bcec44c974c5fb7250 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Thu, 3 Oct 2024 08:32:20 -0400 Subject: [PATCH 19/28] us PAT to commit Signed-off-by: Alex Meijer --- .github/workflows/update-manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-manifest.yaml b/.github/workflows/update-manifest.yaml index f877045..a4c7164 100644 --- a/.github/workflows/update-manifest.yaml +++ b/.github/workflows/update-manifest.yaml @@ -44,4 +44,4 @@ jobs: - \ No newline at end of file + From 7d679a75a521e4e59dacb3839aacb2a7985f80d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Oct 2024 13:31:30 +0000 Subject: [PATCH 20/28] update manifest Signed-off-by: sajit --- manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/manifest b/manifest index 9e011e8..12b0019 100644 --- a/manifest +++ b/manifest @@ -1,3 +1,4 @@ + # this manifest contains the name of every currently implemented plugin. it can be pulled via https://github.com/opencost/opencost-plugins/raw/main/manifest to get an up to date list of current plugins. datadog From 1b9f1f7dd6b3298d6d6d16805b66f82166bdfaff Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Oct 2024 14:11:05 +0000 Subject: [PATCH 21/28] update manifest Signed-off-by: sajit --- manifest | 1 - 1 file changed, 1 deletion(-) diff --git a/manifest b/manifest index 12b0019..9e011e8 100644 --- a/manifest +++ b/manifest @@ -1,4 +1,3 @@ - # this manifest contains the name of every currently implemented plugin. it can be pulled via https://github.com/opencost/opencost-plugins/raw/main/manifest to get an up to date list of current plugins. datadog From 448825e915a251c3021a1b28b652e4eba4eaa7fc Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Thu, 10 Oct 2024 14:47:58 -0400 Subject: [PATCH 22/28] OpenAI Plugin MVP (#38) * OpenAI Plugin MVP Signed-off-by: Alex Meijer * code review fixes Signed-off-by: Alex Meijer * add MAINTAINERS Signed-off-by: Alex Meijer * only attempt to read if no error Signed-off-by: Alex Meijer --------- Signed-off-by: Alex Meijer Signed-off-by: sajit --- MAINTAINERS.md | 10 + .../openai/cmd/main/getcustomcosts_test.go | 53 +++ pkg/plugins/openai/cmd/main/main.go | 361 ++++++++++++++++++ pkg/plugins/openai/cmd/validator/main/main.go | 163 ++++++++ pkg/plugins/openai/go.mod | 68 ++++ pkg/plugins/openai/go.sum | 197 ++++++++++ .../openai/openaiplugin/openaibilling.go | 22 ++ .../openai/openaiplugin/openaiconfig.go | 6 + .../openai/openaiplugin/openaiusage.go | 26 ++ 9 files changed, 906 insertions(+) create mode 100644 MAINTAINERS.md create mode 100644 pkg/plugins/openai/cmd/main/getcustomcosts_test.go create mode 100644 pkg/plugins/openai/cmd/main/main.go create mode 100644 pkg/plugins/openai/cmd/validator/main/main.go create mode 100644 pkg/plugins/openai/go.mod create mode 100644 pkg/plugins/openai/go.sum create mode 100644 pkg/plugins/openai/openaiplugin/openaibilling.go create mode 100644 pkg/plugins/openai/openaiplugin/openaiconfig.go create mode 100644 pkg/plugins/openai/openaiplugin/openaiusage.go diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..f37671b --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,10 @@ +# OpenCost Plugins Committers and Maintainers + +Official list of OpenCost Plugins Maintainers + +## Maintainers + +| Maintainer | GitHub ID | Affiliation | Email | +| --------------- | --------- | ----------- | ----------- | +| Nik Willwerth | @nik-kc | Kubecost | | +| Alex Meijer | @ameijer | Kubecost | | \ No newline at end of file diff --git a/pkg/plugins/openai/cmd/main/getcustomcosts_test.go b/pkg/plugins/openai/cmd/main/getcustomcosts_test.go new file mode 100644 index 0000000..2b111df --- /dev/null +++ b/pkg/plugins/openai/cmd/main/getcustomcosts_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + "testing" + "time" + + openaiplugin "github.com/opencost/opencost-plugins/pkg/plugins/openai/openaiplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/util/timeutil" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGetCustomCosts(t *testing.T) { + // read necessary env vars. If any are missing, log warning and skip test + oaiApiKey := os.Getenv("OAI_API_KEY") + if oaiApiKey == "" { + log.Warnf("OAI_API_KEY undefined, skipping test") + t.Skip() + return + } + + //set up config + config := openaiplugin.OpenAIConfig{ + APIKey: oaiApiKey, + } + + rateLimiter := rate.NewLimiter(1, 5) + oaiCostSrc := OpenAICostSource{ + rateLimiter: rateLimiter, + config: &config, + } + + windowStart := time.Date(2024, 10, 9, 0, 0, 0, 0, time.UTC) + // query for qty 2 of 1 hour windows + windowEnd := time.Date(2024, 10, 10, 0, 0, 0, 0, time.UTC) + + req := &pb.CustomCostRequest{ + Start: timestamppb.New(windowStart), + End: timestamppb.New(windowEnd), + Resolution: durationpb.New(timeutil.Day), + } + + log.SetLogLevel("debug") + resp := oaiCostSrc.GetCustomCosts(req) + + if len(resp) == 0 { + t.Fatalf("empty response") + } +} diff --git a/pkg/plugins/openai/cmd/main/main.go b/pkg/plugins/openai/cmd/main/main.go new file mode 100644 index 0000000..b8646d5 --- /dev/null +++ b/pkg/plugins/openai/cmd/main/main.go @@ -0,0 +1,361 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" + + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/google/uuid" + "github.com/hashicorp/go-plugin" + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + openaiplugin "github.com/opencost/opencost-plugins/pkg/plugins/openai/openaiplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" + "github.com/opencost/opencost/core/pkg/util/timeutil" +) + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "openai", +} + +const openAIUsageURLFmt = "https://api.openai.com/v1/usage?date=%s" +const openAIBillingURLFmt = "https://api.openai.com/v1/dashboard/billing/usage/export?exclude_project_costs=false&file_format=json&new_endpoint=true&project_id&start_date=%s&end_date=%s" +const openAIAPIDateFormat = "2006-01-02" + +// Implementation of CustomCostSource +type OpenAICostSource struct { + rateLimiter *rate.Limiter + config *openaiplugin.OpenAIConfig +} + +func (d *OpenAICostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + if req.Resolution.AsDuration() != timeutil.Day { + log.Infof("openai plugin only supports daily resolution") + return results + } + + for _, target := range targets { + // don't allow future request + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching Open AI costs for window %v", target) + result := d.getOpenAICostsForWindow(target) + results = append(results, result) + } + + return results +} + +func main() { + + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + oaiConfig, err := getOpenAIConfig(configFile) + if err != nil { + log.Fatalf("error building OpenAI config: %v", err) + } + log.SetLogLevel(oaiConfig.LogLevel) + // rate limit to 1 request per second + rateLimiter := rate.NewLimiter(0.5, 1) + oaiCostSrc := OpenAICostSource{ + rateLimiter: rateLimiter, + config: oaiConfig, + } + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &oaiCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) +} + +func boilerplateOpenAICustomCost(win opencost.Window) pb.CustomCostResponse { + return pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v1"}, + CostSource: "AI", + Domain: "openai", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: []*pb.CustomCost{}, + } +} +func (d *OpenAICostSource) getOpenAICostsForWindow(window opencost.Window) *pb.CustomCostResponse { + ccResp := boilerplateOpenAICustomCost(window) + + oaiTokenUsages, err := d.getOpenAITokenUsages(*window.Start()) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error getting OpenAI token usages: %v", err)) + } + + oaiBilling, err := d.getOpenAIBilling(*window.Start(), *window.End()) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error getting OpenAI billing data: %v", err)) + } + + customCosts, err := getCustomCostsFromUsageAndBilling(oaiTokenUsages, oaiBilling) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error converting API responses into custom costs: %v", err)) + } + ccResp.Costs = customCosts + + return &ccResp +} + +func getCustomCostsFromUsageAndBilling(usage *openaiplugin.OpenAIUsage, billing *openaiplugin.OpenAIBilling) ([]*pb.CustomCost, error) { + customCosts := []*pb.CustomCost{} + + tokenMap := buildTokenMap(usage) + for _, billingEntry := range billing.Data { + tokenMapKey := strings.ReplaceAll(strings.ToLower(billingEntry.Name), "-", "") + tokenMapKey = strings.ReplaceAll(tokenMapKey, " ", "") + tokenMapKey = strings.ReplaceAll(tokenMapKey, "_", "") + + tokenCount, ok := tokenMap[tokenMapKey] + if !ok { + log.Debugf("no token usage found for %s", billingEntry.Name) + tokenCount = -1 + } + + extendedAttrs := pb.CustomCostExtendedAttributes{ + AccountId: &billingEntry.OrganizationID, + SubAccountId: &billingEntry.ProjectID, + } + customCost := pb.CustomCost{ + BilledCost: float32(billingEntry.CostInMajor), + AccountName: billingEntry.OrganizationName, + ChargeCategory: "Usage", + Description: fmt.Sprintf("OpenAI usage for model %s", billingEntry.Name), + ResourceName: billingEntry.Name, + ResourceType: "AI Model", + Id: uuid.New().String(), + ProviderId: fmt.Sprintf("%s/%s/%s", billingEntry.OrganizationID, billingEntry.ProjectID, billingEntry.Name), + UsageQuantity: float32(tokenCount), + UsageUnit: "tokens - All snapshots, all projects", + ExtendedAttributes: &extendedAttrs, + } + + customCosts = append(customCosts, &customCost) + } + + return customCosts, nil +} + +var snapshotRe = regexp.MustCompile(`-\d{4}-\d{2}-\d{2}|-`) + +func buildTokenMap(usage *openaiplugin.OpenAIUsage) map[string]int { + tokenMap := make(map[string]int) + if usage == nil { + return tokenMap + } + for _, usageData := range usage.Data { + key := snapshotRe.ReplaceAllString(usageData.SnapshotID, "") + key = strings.ToLower(key) + if _, ok := tokenMap[key]; !ok { + tokenMap[key] = 0 + } + + tokenMap[key] += (usageData.NGeneratedTokensTotal + usageData.NContextTokensTotal) + } + return tokenMap +} + +func (d *OpenAICostSource) getOpenAIBilling(start time.Time, end time.Time) (*openaiplugin.OpenAIBilling, error) { + client := &http.Client{} + openAIBillingURL := fmt.Sprintf(openAIBillingURLFmt, start.Format(openAIAPIDateFormat), end.Format(openAIAPIDateFormat)) + log.Debugf("fetching OpenAI billing data from %s", openAIBillingURL) + var errReq error + var resp *http.Response + for i := 0; i < 3; i++ { + err := d.rateLimiter.Wait(context.Background()) + if err != nil { + log.Warnf("error waiting for rate limiter: %v", err) + return nil, fmt.Errorf("error waiting for rate limiter: %v", err) + } + var req *http.Request + req, errReq = http.NewRequest("GET", openAIBillingURL, nil) + if errReq != nil { + log.Warnf("error creating billing export request: %v", errReq) + log.Warnf("retrying request after 30s") + time.Sleep(30 * time.Second) + continue + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", d.config.APIKey)) + + resp, errReq = client.Do(req) + if errReq != nil { + log.Warnf("error doing billing export request: %v", errReq) + log.Warnf("retrying requestafter 30s") + time.Sleep(30 * time.Second) + continue + } + + if resp.StatusCode != http.StatusOK { + bodyBytes, err := io.ReadAll(resp.Body) + bodyString := "" + if err != nil { + log.Warnf("error reading body of non-200 response: %v", err) + } else { + bodyString = string(bodyBytes) + } + + errReq = fmt.Errorf("received non-200 response for billing export request: %d", resp.StatusCode) + log.Warnf("got non-200 response for billing export request: %d, body is: %s", resp.StatusCode, bodyString) + log.Warnf("retrying request after 30s") + time.Sleep(30 * time.Second) + continue + } else { + errReq = nil + } + // request was successful, break out of loop + break + } + + if errReq != nil { + return nil, fmt.Errorf("error making request after retries: %v", errReq) + } + var billingData openaiplugin.OpenAIBilling + if err := json.NewDecoder(resp.Body).Decode(&billingData); err != nil { + return nil, fmt.Errorf("error decoding billing export response: %v", err) + } + resp.Body.Close() + for i := range billingData.Data { + asFloat, err := strconv.ParseFloat(billingData.Data[i].CostInMajorStr, 64) + if err != nil { + return nil, fmt.Errorf("error parsing cost: %v", err) + } + billingData.Data[i].CostInMajor = asFloat + } + + return &billingData, nil +} + +func (d *OpenAICostSource) getOpenAITokenUsages(targetTime time.Time) (*openaiplugin.OpenAIUsage, error) { + client := &http.Client{} + + openAIUsageURL := fmt.Sprintf(openAIUsageURLFmt, targetTime.Format(openAIAPIDateFormat)) + log.Debugf("fetching OpenAI usage data from %s", openAIUsageURL) + var errReq error + var resp *http.Response + for i := 0; i < 3; i++ { + errReq = nil + err := d.rateLimiter.Wait(context.Background()) + if err != nil { + log.Warnf("error waiting for rate limiter: %v", err) + return nil, fmt.Errorf("error waiting for rate limiter: %v", err) + } + var req *http.Request + req, errReq = http.NewRequest("GET", openAIUsageURL, nil) + if errReq != nil { + log.Warnf("error creating usage request: %v", errReq) + log.Warnf("retrying request after 30s") + time.Sleep(30 * time.Second) + continue + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", d.config.APIKey)) + + resp, errReq = client.Do(req) + if errReq != nil { + log.Warnf("error doing token request: %v", errReq) + log.Warnf("retrying request after 30s") + time.Sleep(30 * time.Second) + continue + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errReq = fmt.Errorf("received non-200 response for token usage request: %d", resp.StatusCode) + bodyBytes, err := io.ReadAll(resp.Body) + bodyString := "" + if err != nil { + log.Warnf("error reading body of non-200 response: %v", err) + } else { + bodyString = string(bodyBytes) + } + log.Warnf("got non-200 response for token usage request: %d, body is: %s", resp.StatusCode, bodyString) + log.Warnf("retrying request after 30s") + time.Sleep(30 * time.Second) + continue + } else { + errReq = nil + } + // request was successful, break out of loop + break + } + + if errReq != nil { + return nil, fmt.Errorf("error making request after retries: %v", errReq) + } + + var usageData openaiplugin.OpenAIUsage + if err := json.NewDecoder(resp.Body).Decode(&usageData); err != nil { + return nil, fmt.Errorf("error decoding response: %v", err) + } + + return &usageData, nil +} + +func getOpenAIConfig(configFilePath string) (*openaiplugin.OpenAIConfig, error) { + var result openaiplugin.OpenAIConfig + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for openai config @ %s: %v", configFilePath, err) + } + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error marshaling json into openai config %v", err) + } + + if result.LogLevel == "" { + result.LogLevel = "info" + } + + return &result, nil +} diff --git a/pkg/plugins/openai/cmd/validator/main/main.go b/pkg/plugins/openai/cmd/validator/main/main.go new file mode 100644 index 0000000..9a22f96 --- /dev/null +++ b/pkg/plugins/openai/cmd/validator/main/main.go @@ -0,0 +1,163 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/hashicorp/go-multierror" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/encoding/protojson" +) + +// the validator is designed to allow plugin implementors to validate their plugin information +// as called by the central test harness. +// this avoids having to ask folks to re-implement the test harness over again for each plugin + +// the integration test harness provides a path to a protobuf file for each window +// the validator can then read that in and further validate the response data +// using the domain knowledge of each plugin author +func main() { + + // first arg is the path to the daily protobuf file + if len(os.Args) < 3 { + fmt.Println("Usage: validator ") + os.Exit(1) + } + + dailyProtobufFilePath := os.Args[1] + + // read in the protobuf file + data, err := os.ReadFile(dailyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading daily protobuf file: %v\n", err) + os.Exit(1) + } + + dailyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) + + // second arg is the path to the hourly protobuf file + hourlyProtobufFilePath := os.Args[2] + + data, err = os.ReadFile(hourlyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading hourly protobuf file: %v\n", err) + os.Exit(1) + } + + // read in the protobuf file + hourlyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) + + // validate the custom cost response data + isvalid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isvalid { + os.Exit(1) + } else { + fmt.Println("Validation successful") + } +} + +func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { + if len(respDaily) == 0 { + log.Errorf("no daily response received from openai plugin") + return false + } + + var multiErr error + + // parse the response and look for errors + for _, resp := range respDaily { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) + } + } + + // check if any errors occurred + if multiErr != nil { + log.Errorf("Errors occurred during plugin testing for open ai: %v", multiErr) + return false + } + seenCosts := map[string]bool{} + var costSum float32 + //verify that the returned costs are non zero + for _, resp := range respDaily { + if len(resp.Costs) == 0 && resp.Start.AsTime().After(time.Now().Truncate(24*time.Hour).Add(-1*time.Minute)) { + log.Debugf("today's daily costs returned by plugin openai are empty, skipping: %v", resp) + continue + } + + for _, cost := range resp.Costs { + costSum += cost.GetBilledCost() + seenCosts[cost.GetResourceName()] = true + if cost.GetBilledCost() == 0 { + log.Debugf("got zero cost for %v", cost) + } + if cost.GetBilledCost() > 1 { + log.Errorf("daily cost returned by plugin openai for %v is greater than 1", cost) + return false + } + } + + } + if costSum == 0 { + log.Errorf("daily costs returned by openai plugin are zero") + return false + } + expectedCosts := []string{ + "GPT-4o mini", + "GPT-4o", + "Other models", + } + + for _, cost := range expectedCosts { + if !seenCosts[cost] { + log.Errorf("daily cost %s not found in plugin openai response", cost) + return false + } + } + + // verify the domain matches the plugin name + for _, resp := range respDaily { + if resp.Domain != "openai" { + log.Errorf("daily domain returned by plugin openai does not match plugin name") + return false + } + } + + if len(seenCosts) < len(expectedCosts)-1 || len(seenCosts) > len(expectedCosts)+1 { + log.Errorf("daily costs returned by openai plugin are very different than expected") + return false + } + return true +} + +func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + protoResps := make([]*pb.CustomCostResponse, len(raw)) + for i, r := range raw { + p := &pb.CustomCostResponse{} + if err := protojson.Unmarshal(r, p); err != nil { + return nil, err + } + protoResps[i] = p + } + + return protoResps, nil +} diff --git a/pkg/plugins/openai/go.mod b/pkg/plugins/openai/go.mod new file mode 100644 index 0000000..a1d15ed --- /dev/null +++ b/pkg/plugins/openai/go.mod @@ -0,0 +1,68 @@ +module github.com/opencost/opencost-plugins/pkg/plugins/openai + +go 1.23 + +toolchain go1.23.1 + +replace github.com/opencost/opencost-plugins/pkg/common => ../../common + +require ( + github.com/google/uuid v1.6.0 + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-plugin v1.6.0 + github.com/opencost/opencost-plugins/pkg/common v0.0.0-00010101000000-000000000000 + github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a + golang.org/x/time v0.5.0 + google.golang.org/protobuf v1.33.0 +) + +require ( + github.com/fatih/color v1.16.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-hclog v1.6.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oklog/run v1.1.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/rs/zerolog v1.32.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.18.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 // indirect + google.golang.org/grpc v1.62.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.29.2 // indirect + k8s.io/apimachinery v0.29.2 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) diff --git a/pkg/plugins/openai/go.sum b/pkg/plugins/openai/go.sum new file mode 100644 index 0000000..d7c91be --- /dev/null +++ b/pkg/plugins/openai/go.sum @@ -0,0 +1,197 @@ +github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= +github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-hclog v1.6.2 h1:NOtoftovWkDheyUM/8JW3QMiXyxJK3uHRK7wV04nD2I= +github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= +github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a h1:m6sesjHd7phuhoWhrCXrzLKHJbAdlH0Q07Uvpbgl4G0= +github.com/opencost/opencost/core v0.0.0-20240307141548-816f98c9051a/go.mod h1:9o1Jfz3nuxVYRmlGk4xo84XZxoQk/LHqPd+Kvo1YIZ4= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 h1:IR+hp6ypxjH24bkMfEJ0yHR21+gwPWdV+/IBrPQyn3k= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/plugins/openai/openaiplugin/openaibilling.go b/pkg/plugins/openai/openaiplugin/openaibilling.go new file mode 100644 index 0000000..e767e9a --- /dev/null +++ b/pkg/plugins/openai/openaiplugin/openaibilling.go @@ -0,0 +1,22 @@ +package openaiplugin + +// OpenAIBilling represents the structure of the response JSON +type OpenAIBilling struct { + Object string `json:"object"` + Data []BillingData `json:"data"` +} + +// BillingData represents the individual Billing data entries +type BillingData struct { + Timestamp float64 `json:"timestamp"` + Currency string `json:"currency"` + Name string `json:"name"` + Cost float64 `json:"cost"` + OrganizationID string `json:"organization_id"` + ProjectID string `json:"project_id"` + ProjectName string `json:"project_name"` + OrganizationName string `json:"organization_name"` + CostInMajorStr string `json:"cost_in_major"` + CostInMajor float64 `json:"-"` + Date string `json:"date"` +} diff --git a/pkg/plugins/openai/openaiplugin/openaiconfig.go b/pkg/plugins/openai/openaiplugin/openaiconfig.go new file mode 100644 index 0000000..44c6a4e --- /dev/null +++ b/pkg/plugins/openai/openaiplugin/openaiconfig.go @@ -0,0 +1,6 @@ +package openaiplugin + +type OpenAIConfig struct { + APIKey string `json:"openai_api_key"` + LogLevel string `json:"log_level"` +} diff --git a/pkg/plugins/openai/openaiplugin/openaiusage.go b/pkg/plugins/openai/openaiplugin/openaiusage.go new file mode 100644 index 0000000..d541a40 --- /dev/null +++ b/pkg/plugins/openai/openaiplugin/openaiusage.go @@ -0,0 +1,26 @@ +package openaiplugin + +type OpenAIUsage struct { + Object string `json:"object"` + Data []UsageData `json:"data"` +} + +type UsageData struct { + OrganizationID string `json:"organization_id"` + OrganizationName string `json:"organization_name"` + AggregationTimestamp int `json:"aggregation_timestamp"` + NRequests int `json:"n_requests"` + Operation string `json:"operation"` + SnapshotID string `json:"snapshot_id"` + NContextTokensTotal int `json:"n_context_tokens_total"` + NGeneratedTokensTotal int `json:"n_generated_tokens_total"` + Email *string `json:"email"` + APIKeyID *string `json:"api_key_id"` + APIKeyName *string `json:"api_key_name"` + APIKeyRedacted *string `json:"api_key_redacted"` + APIKeyType *string `json:"api_key_type"` + ProjectID *string `json:"project_id"` + ProjectName *string `json:"project_name"` + RequestType string `json:"request_type"` + NCachedContextTokensTotal int `json:"n_cached_context_tokens_total"` +} From c8a4a4401fcd71863ab20d0bf6a76d41e66154df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 10 Oct 2024 18:48:16 +0000 Subject: [PATCH 23/28] update manifest Signed-off-by: sajit --- manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/manifest b/manifest index 9e011e8..c1d98eb 100644 --- a/manifest +++ b/manifest @@ -1,3 +1,4 @@ # this manifest contains the name of every currently implemented plugin. it can be pulled via https://github.com/opencost/opencost-plugins/raw/main/manifest to get an up to date list of current plugins. datadog +openai From 9ff32f3106135fa134d149229675034a93d688b6 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Thu, 10 Oct 2024 16:35:05 -0400 Subject: [PATCH 24/28] fix harness to support multiple plugins (#39) Signed-off-by: Alex Meijer Signed-off-by: sajit --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 6278326..2620144 100644 --- a/justfile +++ b/justfile @@ -2,7 +2,7 @@ commonenv := "CGO_ENABLED=0" version := `./tools/image-tag` commit := `git rev-parse --short HEAD` -pluginPaths := `find ./pkg/plugins -type f -iname "go.mod" -print0 | xargs -0 dirname | xargs basename | tr ' ' ','` +pluginPaths := `find ./pkg/plugins -type f -iname "go.mod" -print0 | xargs -0 dirname | xargs -I{} basename {} | tr ' ' ',' | tr '\n' ','` default: just --list From b29c8b96bc6f5a76ceae5cff68cfec2bb8a89459 Mon Sep 17 00:00:00 2001 From: Alex Meijer Date: Mon, 14 Oct 2024 16:09:57 -0400 Subject: [PATCH 25/28] implement validator, small bugfix (#6) * implement validator, small bugfix Signed-off-by: Alex Meijer * additional testing Signed-off-by: Alex Meijer --------- Signed-off-by: Alex Meijer Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 2 +- .../mongodb-atlas/cmd/validator/main/main.go | 181 ++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 pkg/plugins/mongodb-atlas/cmd/validator/main/main.go diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index 214e586..d7de989 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -194,7 +194,7 @@ func filterLineItemsByWindow(win *opencost.Window, lineItems []atlasplugin.LineI ResourceName: item.SKU, Id: string(uuid.NewUUID()), ProviderId: fmt.Sprintf("%s %s %s", item.GroupId, item.ClusterName, item.SKU), - BilledCost: float32(item.TotalPriceCents / 100), + BilledCost: float32(item.TotalPriceCents) / 100.0, ListCost: item.Quantity * item.UnitPriceDollars, ListUnitPrice: item.UnitPriceDollars, UsageQuantity: item.Quantity, diff --git a/pkg/plugins/mongodb-atlas/cmd/validator/main/main.go b/pkg/plugins/mongodb-atlas/cmd/validator/main/main.go new file mode 100644 index 0000000..0ac11a7 --- /dev/null +++ b/pkg/plugins/mongodb-atlas/cmd/validator/main/main.go @@ -0,0 +1,181 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/hashicorp/go-multierror" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "google.golang.org/protobuf/encoding/protojson" +) + +// the validator is designed to allow plugin implementors to validate their plugin information +// as called by the central test harness. +// this avoids having to ask folks to re-implement the test harness over again for each plugin + +// the integration test harness provides a path to a protobuf file for each window +// the validator can then read that in and further validate the response data +// using the domain knowledge of each plugin author +func main() { + + // first arg is the path to the daily protobuf file + if len(os.Args) < 3 { + fmt.Println("Usage: validator ") + os.Exit(1) + } + + dailyProtobufFilePath := os.Args[1] + + // read in the protobuf file + data, err := os.ReadFile(dailyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading daily protobuf file: %v\n", err) + os.Exit(1) + } + + dailyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling daily protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d daily custom cost responses\n", len(dailyCustomCostResponses)) + + // second arg is the path to the hourly protobuf file + hourlyProtobufFilePath := os.Args[2] + + data, err = os.ReadFile(hourlyProtobufFilePath) + if err != nil { + fmt.Printf("Error reading hourly protobuf file: %v\n", err) + os.Exit(1) + } + + // read in the protobuf file + hourlyCustomCostResponses, err := Unmarshal(data) + if err != nil { + fmt.Printf("Error unmarshalling hourly protobuf data: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Successfully unmarshalled %d hourly custom cost responses\n", len(hourlyCustomCostResponses)) + + // validate the custom cost response data + isvalid := validate(dailyCustomCostResponses, hourlyCustomCostResponses) + if !isvalid { + os.Exit(1) + } else { + fmt.Println("Validation successful") + } +} + +func validate(respDaily, respHourly []*pb.CustomCostResponse) bool { + if len(respDaily) == 0 { + log.Errorf("no daily response received from mongodb-atlas plugin") + return false + } + + if len(respHourly) == 0 { + log.Errorf("no hourly response received from mongodb-atlas plugin") + return false + } + + var multiErr error + + // parse the response and look for errors + for _, resp := range respDaily { + if len(resp.Errors) > 0 { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in daily response: %v", resp.Errors)) + } + } + + for _, resp := range respHourly { + if resp.Errors != nil { + multiErr = multierror.Append(multiErr, fmt.Errorf("errors occurred in hourly response: %v", resp.Errors)) + } + } + + // check if any errors occurred + if multiErr != nil { + log.Errorf("Errors occurred during plugin testing for mongodb-atlas: %v", multiErr) + return false + } + + seenCosts := map[string]bool{} + nonZeroBilledCosts := 0 + //verify that the returned costs are non zero + for _, resp := range respDaily { + for _, cost := range resp.Costs { + seenCosts[cost.GetResourceName()] = true + if !strings.Contains(cost.GetResourceName(), "FREE") && cost.GetListCost() == 0 { + log.Errorf("daily list cost returned by plugin mongodb-atlas is zero for cost: %v", cost) + return false + } + if cost.GetListCost() >= 0.01 && !strings.Contains(cost.GetResourceName(), "FREE") && cost.GetBilledCost() == 0 { + log.Errorf("daily billed cost returned by plugin mongodb-atlas is zero for cost: %v", cost) + return false + } + if cost.GetBilledCost() > 0 { + nonZeroBilledCosts++ + } + } + } + + if nonZeroBilledCosts == 0 { + log.Errorf("no non-zero billed costs returned by plugin mongodb-atlas") + return false + } + expectedCosts := []string{ + "ATLAS_AWS_DATA_TRANSFER_DIFFERENT_REGION", + "ATLAS_AWS_DATA_TRANSFER_INTERNET", + "ATLAS_AWS_DATA_TRANSFER_SAME_REGION", + "ATLAS_AWS_INSTANCE_M10", + "ATLAS_NDS_AWS_PIT_RESTORE_STORAGE", + "ATLAS_NDS_AWS_PIT_RESTORE_STORAGE_FREE_TIER", + } + + for _, cost := range expectedCosts { + if !seenCosts[cost] { + log.Errorf("hourly cost %s not found in plugin mongodb-atlas response", cost) + return false + } + } + + if len(seenCosts) != len(expectedCosts) { + log.Errorf("hourly costs returned by plugin mongodb-atlas do not equal expected costs") + log.Errorf("seen costs: %v", seenCosts) + log.Errorf("expected costs: %v", expectedCosts) + + log.Errorf("response: %v", respHourly) + return false + } + + // verify the domain matches the plugin name + for _, resp := range respDaily { + if resp.Domain != "mongodb-atlas" { + log.Errorf("daily domain returned by plugin mongodb-atlas does not match plugin name") + return false + } + } + + return true +} + +func Unmarshal(data []byte) ([]*pb.CustomCostResponse, error) { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + protoResps := make([]*pb.CustomCostResponse, len(raw)) + for i, r := range raw { + p := &pb.CustomCostResponse{} + if err := protojson.Unmarshal(r, p); err != nil { + return nil, err + } + protoResps[i] = p + } + + return protoResps, nil +} From 27c8385059dfb2fc62e580533c8c2178edd3a2f8 Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 14 Oct 2024 16:40:27 -0400 Subject: [PATCH 26/28] fixing tests; remove .gitmessage.txt from source control Signed-off-by: sajit --- .gitmessage.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .gitmessage.txt diff --git a/.gitmessage.txt b/.gitmessage.txt deleted file mode 100644 index f9a3bed..0000000 --- a/.gitmessage.txt +++ /dev/null @@ -1,2 +0,0 @@ -#Commit Message template -Signed-off-by: Sajit Kunnumkal From b81730a5825beea104e5d31d1486d68185dfeeb9 Mon Sep 17 00:00:00 2001 From: sajit Date: Mon, 14 Oct 2024 16:56:03 -0400 Subject: [PATCH 27/28] fix tests Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 2 +- pkg/plugins/mongodb-atlas/cmd/main/main_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index d7de989..fd03a58 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -193,7 +193,7 @@ func filterLineItemsByWindow(win *opencost.Window, lineItems []atlasplugin.LineI Description: fmt.Sprintf("Usage for %s", item.SKU), ResourceName: item.SKU, Id: string(uuid.NewUUID()), - ProviderId: fmt.Sprintf("%s %s %s", item.GroupId, item.ClusterName, item.SKU), + ProviderId: fmt.Sprintf("%s/%s/%s", item.GroupId, item.ClusterName, item.SKU), BilledCost: float32(item.TotalPriceCents) / 100.0, ListCost: item.Quantity * item.UnitPriceDollars, ListUnitPrice: item.UnitPriceDollars, diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go index 77dbb28..0cbf719 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main_test.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main_test.go @@ -407,8 +407,9 @@ func TestFilterInvoicesOnWindow(t *testing.T) { assert.Equal(t, "0", filteredItems[0].ResourceName) assert.NotNil(t, filteredItems[0].Id) assert.NotNil(t, filteredItems[0].ProviderId) + assert.Equal(t, "A/cluster-0/0", filteredItems[0].ProviderId) - assert.InDelta(t, lineItems[0].TotalPriceCents/100, filteredItems[0].BilledCost, 0.01) + assert.InDelta(t, float32(lineItems[0].TotalPriceCents)/100.0, filteredItems[0].BilledCost, 0.01) assert.InDelta(t, filteredItems[0].ListCost, lineItems[0].Quantity*lineItems[0].UnitPriceDollars, 0.01) assert.Equal(t, lineItems[0].Quantity, filteredItems[0].UsageQuantity) assert.Equal(t, filteredItems[0].UsageUnit, lineItems[0].Unit) From 52e0d61c189d546e23c5b4d0f18115e8fd434333 Mon Sep 17 00:00:00 2001 From: sajit Date: Wed, 16 Oct 2024 09:45:54 -0400 Subject: [PATCH 28/28] address PR comments: change error to warnings Signed-off-by: Sajit Kunnumkal Signed-off-by: sajit --- pkg/plugins/mongodb-atlas/cmd/main/main.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/plugins/mongodb-atlas/cmd/main/main.go b/pkg/plugins/mongodb-atlas/cmd/main/main.go index fd03a58..a58c8e2 100644 --- a/pkg/plugins/mongodb-atlas/cmd/main/main.go +++ b/pkg/plugins/mongodb-atlas/cmd/main/main.go @@ -94,19 +94,25 @@ func validateRequest(req *pb.CustomCostRequest) []string { now := time.Now() // 1. Check if resolution is less than a day if req.Resolution.AsDuration() < 24*time.Hour { - errors = append(errors, "Resolution should be at least one day.") + var resolutionMessage = "Resolution should be at least one day." + log.Warnf(resolutionMessage) + errors = append(errors, resolutionMessage) } // Get the start of the current month currentMonthStart := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) // 2. Check if start time is before the start of the current month if req.Start.AsTime().Before(currentMonthStart) { - errors = append(errors, "Start date cannot be before the current month. Historical costs not currently supported") + var startDateMessage = "Start date cannot be before the current month. Historical costs not currently supported" + log.Warnf(startDateMessage) + errors = append(errors, startDateMessage) } // 3. Check if end time is before the start of the current month if req.End.AsTime().Before(currentMonthStart) { - errors = append(errors, "End date cannot be before the current month. Historical costs not currently supported") + var endDateMessage = "End date cannot be before the current month. Historical costs not currently supported" + log.Warnf(endDateMessage) + errors = append(errors, endDateMessage) } return errors @@ -116,10 +122,7 @@ func (a *AtlasCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.Custom requestErrors := validateRequest(req) if len(requestErrors) > 0 { - errResp := pb.CustomCostResponse{ - Errors: requestErrors, - } - results = append(results, &errResp) + //return empty response return results }