diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..93884a9 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +AZURE_CREDS=$(vault read -format=json azure/creds/azure-deployer) && export AZURE_CLIENT_ID=$(echo $AZURE_CREDS | jq .data.client_id -r) && export AZURE_CLIENT_SECRET=$(echo $AZURE_CREDS | jq .data.client_secret -r) diff --git a/cmd/prometheus-aws-discovery/main.go b/cmd/prometheus-aws-discovery/main.go index 9db02e5..61bcfe0 100644 --- a/cmd/prometheus-aws-discovery/main.go +++ b/cmd/prometheus-aws-discovery/main.go @@ -1,17 +1,16 @@ package main import ( + "encoding/csv" "flag" - "fmt" "os" + "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/daspawnw/prometheus-aws-discovery/pkg/awsdiscovery" + "github.com/daspawnw/prometheus-aws-discovery/pkg/azurediscovery" "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" "github.com/daspawnw/prometheus-aws-discovery/pkg/output" - "github.com/daspawnw/prometheus-aws-discovery/pkg/outputfile" - "github.com/daspawnw/prometheus-aws-discovery/pkg/outputkubernetes" + log "github.com/sirupsen/logrus" ) @@ -23,66 +22,102 @@ func init() { } func main() { - tagPrefix := flag.String("tagprefix", "prom/scrape", "Prefix used for tag key to filter for exporter") - outputType := flag.String("output", "kubernetes", "Allowed Values {kubernetes|file}") - filePath := flag.String("file-path", "", "Target file path to write file to") - kubeconfig := flag.String("kube-config", "", "Path to a kubeconfig file") - namespace := flag.String("kube-namespace", "", "Namespace where to create or update configmap (If in cluster and no namespace is provided it tries to detect namespace from incluster config otherwise it uses 'default' namespace)") - configmapName := flag.String("kube-configmap-name", "", "Name of the configmap to create or update with discovery output") - configmapKey := flag.String("kube-configmap-key", "", "Name of configmap key to set discovery output to") + var outputType string + var iaasCSV string + tagPrefix := false + var tag string + var filePath string + var kubeconfig string + var namespace string + var configmapName string + var configmapKey string + var subscrID string + + flag.StringVar(&tag, "tagprefix", "prom/scrape", "Tag-Key-Prefix used for exporter config (V1-Tag)") + flag.StringVar(&tag, "tag", "", "Tag-Key used to look for exporter data (V2-Tag)") + flag.StringVar(&outputType, "output", "kubernetes", "Allowed Values {kubernetes|file}") + flag.StringVar(&filePath, "file-path", "", "Target file path for sd_config file output") + flag.StringVar(&kubeconfig, "kube-config", "", "Path to a kubeconfig file") + flag.StringVar(&namespace, "kube-namespace", "", "Namespace where to create or update configmap (If in cluster and no namespace is provided it tries to detect namespace from incluster config otherwise it uses 'default' namespace)") + flag.StringVar(&configmapName, "kube-configmap-name", "", "Name of the configmap to create or update with discovery output") + flag.StringVar(&configmapKey, "kube-configmap-key", "", "Name of configmap key to set discovery output to") + flag.StringVar(&iaasCSV, "iaas", "aws", "CSV of Clouds to check [aws/azure] e.g. aws,azure ") + flag.StringVar(&subscrID, "azure-subscr", "", "Azure Subscription ID to look for VMs") verbose := flag.Bool("verbose", false, "Print verbose log messages") printVersion := flag.Bool("version", false, "Print version") flag.Parse() + flag.Visit(func(f *flag.Flag) { + if f.Name == "tagprefix" { + tagPrefix = true + } + }) if *verbose { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.InfoLevel) } - if *printVersion { log.Info("Version: " + Version) os.Exit(0) } - validateArg("outputtype", *outputType, []string{"kubernetes", "file"}) + log.Info("Infra " + iaasCSV) + infraReader := csv.NewReader(strings.NewReader(iaasCSV)) + records, err := infraReader.ReadAll() - awsSession := session.New() - awsConfig := &aws.Config{} - ec2Client := ec2.New(awsSession, awsConfig) - - log.Info("Start discovery of ec2 instances") - d := discovery.NewDiscovery(ec2Client, *tagPrefix) - instances, err := d.Discover() if err != nil { - panic(err) + log.Fatal(err) } - var o output.Output - if *outputType == "kubernetes" { + switch outputType { + case "stdout": + log.Info("Configured stdout as output target") + o = output.OutputStdOut{} + case "file": + log.Info("Configured file as output target") + o = output.OutputFile{FilePath: filePath} + default: log.Info("Configured kubernetes as output target") - k8s, err := outputkubernetes.NewOutputKubernetes(*kubeconfig, *namespace, *configmapName, *configmapKey) + k8s, err := output.NewOutputKubernetes(kubeconfig, namespace, configmapName, configmapKey) if err != nil { panic(err) } - o = k8s - } else { - log.Info("Configured file as output target") - o = outputfile.OutputFile{FilePath: *filePath} } - e := o.Write(*instances) - if e != nil { - panic(e) + validateArg("outputtype", outputType, []string{"kubernetes", "file", "stdout"}) + + for _, runInfra := range records[0] { + log.Info(runInfra) + clients := []discovery.DiscoveryClient{} + switch runInfra { + case "aws": + log.Info("starting aws discovery") + clients = append(clients, &awsdiscovery.DiscoveryClientAWS{ + TagPrefix: tagPrefix, + Tag: tag, + }) + + case "azure": + log.Info("starting azure discovery") + if subscrID == "" { + log.Errorf("Azure set as target but no subscription provided. Use --azure-subscr") + os.Exit(1) + } + clients = append(clients, azurediscovery.DiscoveryClientAZURE{ + TagPrefix: tagPrefix, + Tag: tag, + Subscription: subscrID, + }) + + } + getOutput(clients, o) } - log.Info("Wrote output to target") - log.Info("Completed successfully") } - func validateArg(field string, arg string, allowedValues []string) { if !sliceContains(arg, allowedValues) { - log.Error(fmt.Sprintf("Field %v has allowed values %v but got %s", field, allowedValues, arg)) + log.Errorf("Field %v has allowed values %v but got %s", field, allowedValues, arg) os.Exit(1) } } @@ -95,3 +130,23 @@ func sliceContains(arg string, allowedValues []string) bool { } return false } +func getOutput(clients []discovery.DiscoveryClient, output output.Output) { + outputInstances := []discovery.Instance{} + for _, d := range clients { + instances, err := d.GetInstances() + if err != nil { + log.Error(err) + } + outputInstances = append(outputInstances, instances...) + } + log.Debug("Writing output\n") + err := output.Write(outputInstances) + if err != nil { + log.Error(err) + os.Exit(1) + } + + log.Info("Wrote output to target") + log.Info("Completed successfully") + +} diff --git a/go.mod b/go.mod index d5874c1..0d5928c 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,28 @@ module github.com/daspawnw/prometheus-aws-discovery -go 1.13 +go 1.15 require ( - github.com/aws/aws-sdk-go v1.30.9 + github.com/Azure/azure-sdk-for-go v53.4.0+incompatible + github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 + github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/aws/aws-sdk-go v1.38.25 + github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/golang/protobuf v1.3.3 // indirect + github.com/google/martian v2.1.0+incompatible github.com/imdario/mergo v0.3.9 // indirect + github.com/mitchellh/mapstructure v1.4.1 + github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.5.0 + github.com/softlayer/softlayer-go v1.0.3 + github.com/stretchr/testify v1.5.1 // indirect + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect + golang.org/x/sys v0.0.0-20210421221651-33663a62ff08 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect + google.golang.org/appengine v1.6.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/api v0.18.2 k8s.io/apimachinery v0.18.2 k8s.io/client-go v0.18.2 diff --git a/go.sum b/go.sum index c4e69a0..cea15b1 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,71 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/Azure/azure-sdk-for-go v53.3.0+incompatible h1:DFyCwv0VetPlvKYckSGJRYWUSc+NKRDSryVWVvvVkFw= +github.com/Azure/azure-sdk-for-go v53.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v53.4.0+incompatible h1:ky9S0JoJOvO39ncwrYAVMEBiIjFB/WkbZsWkg/nbYx0= +github.com/Azure/azure-sdk-for-go v53.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v1.1.1 h1:4G9tVCqooRY3vDTB2bA1Z01PlSALtnUbji0AfzthUSs= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.11.17 h1:2zCdHwNgRH+St1J+ZMf66xI8aLr/5KMy+wWLH97zwYM= +github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.11 h1:L4/pmq7poLdsy41Bj1FayKvBhayuWRYkx9HU5i4Ybl0= +github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 h1:8DQB8yl7aLQuP+nuR5e2RO6454OvFlSTXXaNHshc16s= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.7/go.mod h1:AkzUsqkrdmNhfP2i54HqINVQopw0CLDnvHpJ88Zz1eI= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/aws/aws-sdk-go v1.30.9 h1:DntpBUKkchINPDbhEzDRin1eEn1TG9TZFlzWPf0i8to= github.com/aws/aws-sdk-go v1.30.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.38.25 h1:aNjeh7+MON05cZPtZ6do+KxVT67jPOSQXANA46gOQao= +github.com/aws/aws-sdk-go v1.38.25/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -43,6 +84,7 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -72,8 +114,12 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -88,6 +134,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -96,6 +146,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e h1:S+/ptYdZtpK/MDstwCyt+ZHdXEpz86RJZ5gyZU4txJY= +github.com/nsf/jsondiff v0.0.0-20210303162244-6ea32392771e/go.mod h1:uFMI8w+ref4v2r9jz+c9i1IfIttS/OkmLfrk1jne5hs= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= @@ -104,11 +156,16 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/softlayer/softlayer-go v1.0.3 h1:9FONm5xzQ9belQtbdryR6gBg4EF6hX6lrjNKi0IvZkU= +github.com/softlayer/softlayer-go v1.0.3/go.mod h1:6HepcfAXROz0Rf63krk5hPZyHT6qyx2MNvYyHof7ik4= +github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e h1:3OgWYFw7jxCZPcvAg+4R8A50GZ+CCkARF10lxu2qDsQ= +github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e/go.mod h1:fKZCUVdirrxrBpwd9wb+lSoVixvpwAu8eHzbQB2tums= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -119,16 +176,25 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -137,9 +203,15 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -149,6 +221,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -157,14 +231,25 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210421221651-33663a62ff08 h1:qyN5bV+96OX8pL78eXDuz6YlDPzCYgdW74H5yE9BoSU= +golang.org/x/sys v0.0.0-20210421221651-33663a62ff08/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= @@ -174,13 +259,22 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -200,6 +294,8 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.18.2 h1:wG5g5ZmSVgm5B+eHMIbI9EGATS2L8Z72rda19RIEgY8= diff --git a/pkg/awsdiscovery/discovery_aws.go b/pkg/awsdiscovery/discovery_aws.go new file mode 100644 index 0000000..c0f9a05 --- /dev/null +++ b/pkg/awsdiscovery/discovery_aws.go @@ -0,0 +1,168 @@ +package awsdiscovery + +import ( + "encoding/json" + "errors" + "regexp" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" + log "github.com/sirupsen/logrus" +) + +type DiscoveryClientAWS struct { + TagPrefix bool + Tag string +} + +var ec2Client ec2iface.EC2API + +func (d DiscoveryClientAWS) SetEC2Client(client ec2iface.EC2API) { + ec2Client = client +} +func (d DiscoveryClientAWS) GetInstances() ([]discovery.Instance, error) { + + ec2Input := ec2.DescribeInstancesInput{ + Filters: d.filter(), + } + if ec2Client == nil { + awsSession := session.New() + awsConfig := &aws.Config{} + ec2Client = ec2.New(awsSession, awsConfig) + } + ec2DescrRes, err := ec2Client.DescribeInstances(&ec2Input) + if err != nil { + log.Error("Failed to load ec2 instances") + return nil, err + } + ec2InstanceList := []*ec2.Instance{} + for _, res := range ec2DescrRes.Reservations { + for _, instance := range res.Instances { + ec2InstanceList = append(ec2InstanceList, instance) + } + } + + return d.parseScrapeConfigs(ec2InstanceList) +} + +func (d DiscoveryClientAWS) parseScrapeConfigs(ec2Instances []*ec2.Instance) ([]discovery.Instance, error) { + endpointCount := 0 + var instances []discovery.Instance + for _, ec2Instance := range ec2Instances { + + log.Debugf("Extract metric endpoint(s) from instance with ip %s", *ec2Instance.PrivateIpAddress) + + metricEndpoints, err := extractMetricEndpoints(ec2Instance.Tags, d.Tag, d.TagPrefix) + if err != nil { + return nil, err + } + endpointCount += len(metricEndpoints) + log.Debugf("Instance with ip %s has %d metric endpoint(s)", *ec2Instance.PrivateIpAddress, len(metricEndpoints)) + + instances = append(instances, discovery.Instance{ + InstanceType: *ec2Instance.InstanceType, + PrivateIP: *ec2Instance.PrivateIpAddress, + Tags: cleanupTagList(ec2Instance.Tags, d.Tag), + Metrics: metricEndpoints, + }) + } + + log.Infof("Discovered %d instance(s) with %d endpoint(s)", len(instances), endpointCount) + return instances, nil +} +func (d DiscoveryClientAWS) filter() []*ec2.Filter { + filters := []*ec2.Filter{ + { + Name: aws.String("tag-key"), + Values: []*string{aws.String(d.Tag + "*")}, + }, + { + Name: aws.String("instance-state-name"), + Values: []*string{aws.String("running")}, + }, + } + + return filters +} + +func cleanupTagList(tags []*ec2.Tag, prefix string) map[string]string { + tagMap := make(map[string]string) + + for _, tag := range tags { + // remove prom/scrape annotation and aws based tags from tag list + if !matchKeyPattern(*tag.Key, prefix) && !strings.Contains(*tag.Key, ":") { + tagMap[*tag.Key] = *tag.Value + } + } + + return tagMap +} + +func extractMetricEndpoints(tags []*ec2.Tag, promTagValue string, prefix bool) ([]discovery.InstanceMetrics, error) { + metrics := []discovery.InstanceMetrics{} + for _, tag := range tags { + if prefix == true { + //TODO Change to multi v2 tag lookup once migration of existing deployments is done + + if matchKeyPattern(*tag.Key, promTagValue) { + parsedMetric, err := parseMetricEndpoint(*tag.Key, *tag.Value, promTagValue) + if err != nil { + log.Error("Failed to parse Tag, skip metric", err) + continue + } + + metrics = append(metrics, *parsedMetric) + } + continue + + } + if *tag.Key == promTagValue { + log.Debugf("Key %v is v2 tag with Value \n %v", *tag.Key, *tag.Value) + err := json.Unmarshal([]byte(*tag.Value), &metrics) + if err != nil { + return nil, err + } + } + } + + return metrics, nil +} + +func parseMetricEndpoint(key string, value string, prefix string) (*discovery.InstanceMetrics, error) { + r, _ := regexp.Compile(prefix + ":(.*?)(/.*)") + parsedMetric := r.FindStringSubmatch(key) + + if len(parsedMetric) == 3 { + //TO BE FAIR, NONE OF THIS IS A GOOD IDEA. THIS IS AT BEST A DIRTY AND FUGLY QUICK FIX. THIS SHOULD BE REFACTORED TO DO SOMETHING REASONABLE + var scheme = "http" + parsedString := strings.Split(parsedMetric[1], ":") + if len(parsedString) == 2 { + scheme = parsedString[1] + } + + parsedPort, err := strconv.ParseInt(parsedString[0], 10, 64) + if err != nil { + return nil, err + } + + return &discovery.InstanceMetrics{ + Name: value, + Path: parsedMetric[2], + Port: parsedPort, + Scheme: scheme, + }, nil + } + + return nil, errors.New("Failed to match regex pattern") +} + +func matchKeyPattern(key string, prefix string) bool { + r, _ := regexp.Compile(prefix + ":(.*?)(/.*)") + + return r.MatchString(key) +} diff --git a/pkg/awsdiscovery/discovery_aws_test.go b/pkg/awsdiscovery/discovery_aws_test.go new file mode 100644 index 0000000..2e46bbe --- /dev/null +++ b/pkg/awsdiscovery/discovery_aws_test.go @@ -0,0 +1,187 @@ +package awsdiscovery + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" + "github.com/daspawnw/prometheus-aws-discovery/pkg/test" + "github.com/mitchellh/mapstructure" + "github.com/nsf/jsondiff" +) + +func TestDiscoveryHasCorrectEndpointsV1(t *testing.T) { + opts := jsondiff.DefaultConsoleOptions() + + scrapeTargets := make([]discovery.Instance, 0) + + for _, instanceData := range test.Instances { + var instance discovery.Instance + mapstructure.Decode(instanceData, &instance) + scrapeTargets = append(scrapeTargets, instance) + } + testJSONContentBytes, _ := discovery.TargetConfigBytes(scrapeTargets) + + ec2Client := &MockEC2Client{ + instances: EC2InstanceList(), + err: nil, + } + + d := &DiscoveryClientAWS{ + TagPrefix: true, + Tag: "prom/scrape", + } + + d.SetEC2Client(ec2Client) + scrapeTargets, err := d.GetInstances() + if err != nil { + t.Errorf("Failed to discover instances %v", err) + } + + scrapeTargetBytes, err := discovery.TargetConfigBytes(scrapeTargets) + if err != nil { + t.Errorf("TargetConfigBytes %v", err) + } + res, diff := jsondiff.Compare(scrapeTargetBytes, testJSONContentBytes, &opts) + if res != 0 { + t.Errorf("JSONDiff V2 Tag \n%v \n%v", res, diff) + } + +} +func TestDiscoveryHasCorrectEndpointsV2(t *testing.T) { + opts := jsondiff.DefaultConsoleOptions() + + ec2Client := &MockEC2Client{ + instances: EC2InstanceList(), + err: nil, + } + + d := &DiscoveryClientAWS{ + TagPrefix: false, + Tag: "prom", + } + d.SetEC2Client(ec2Client) + + scrapeTargets, err := d.parseScrapeConfigs(EC2InstanceList()) + if err != nil { + t.Errorf("parseScrapeConfigsError %v", err) + } + if len(scrapeTargets) != 3 { + t.Errorf("Expected three Instances Targets") + } + if len(scrapeTargets[0].Metrics)+len(scrapeTargets[1].Metrics) != 0 { + t.Errorf("Expected first and Second Instance to not have Targets") + } + if len(scrapeTargets[2].Metrics) != 2 { + t.Errorf("Expected third to have two Targets") + } + + scrapeTargetBytes, err := discovery.TargetConfigBytes(scrapeTargets) + if err != nil { + + t.Errorf("TargetConfigBytes %v", err) + } + testJSONContentBytes, _ := discovery.TargetConfigBytes(scrapeTargets) + + res, diff := jsondiff.Compare(scrapeTargetBytes, testJSONContentBytes, &opts) + if res != 0 { + t.Log("Expected JSON Strings to match") + t.Errorf("JSONDiff V2 Tag \n%v \n%v", res, diff) + } + +} + +type MockEC2Client struct { + ec2iface.EC2API + instances []*ec2.Instance + err error +} + +func (c *MockEC2Client) DescribeInstances(in *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + var reservations []*ec2.Reservation + var reservation ec2.Reservation + + reservation.Instances = c.instances + reservations = append(reservations, &reservation) + + return &ec2.DescribeInstancesOutput{ + Reservations: reservations, + }, nil +} +func EC2InstanceList() []*ec2.Instance { + var instances []*ec2.Instance + + // instance 1 + instances = append(instances, &ec2.Instance{ + InstanceType: aws.String("t2.medium"), + PrivateIpAddress: aws.String("127.0.0.1"), + Tags: []*ec2.Tag{ + { + Key: aws.String("prom/scrape:9100/metrics"), + Value: aws.String("node_exporter"), + }, + + { + Key: aws.String("prom/scrape:8080:https/metrics"), + Value: aws.String("blackbox_exporter"), + }, + { + Key: aws.String("Name"), + Value: aws.String("Testinstance1"), + }, + { + Key: aws.String("billingnumber"), + Value: aws.String("1111"), + }, + }, + }) + + // instance 2 + instances = append(instances, &ec2.Instance{ + InstanceType: aws.String("t2.small"), + PrivateIpAddress: aws.String("127.0.0.2"), + Tags: []*ec2.Tag{ + { + Key: aws.String("prom/scrape:9100/metrics"), + Value: aws.String("node_exporter"), + }, + { + Key: aws.String("Name"), + Value: aws.String("Testinstance2"), + }, + { + Key: aws.String("billingnumber"), + Value: aws.String("2222"), + }, + { + Key: aws.String("prom/scrape:8888"), + Value: aws.String("test_exporter"), + }, + }, + }) + //instance v2 tags + // + // instance 3 + instances = append(instances, &ec2.Instance{ + InstanceType: aws.String("t2.small"), + PrivateIpAddress: aws.String("127.0.0.2"), + Tags: []*ec2.Tag{ + { + Key: aws.String("prom"), + Value: aws.String(test.V2TagValue), + }, + { + Key: aws.String("Name"), + Value: aws.String("Testinstance3"), + }, + { + Key: aws.String("billingnumber"), + Value: aws.String("2222"), + }, + }, + }) + + return instances +} diff --git a/pkg/azurediscovery/discovery_azure.go b/pkg/azurediscovery/discovery_azure.go new file mode 100644 index 0000000..01eb5db --- /dev/null +++ b/pkg/azurediscovery/discovery_azure.go @@ -0,0 +1,97 @@ +package azurediscovery + +import ( + "context" + "encoding/json" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-03-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-11-01/network" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" + log "github.com/sirupsen/logrus" +) + +type DiscoveryClientAZURE struct { + TagPrefix bool + Tag string + Subscription string +} + +func (d DiscoveryClientAZURE) GetInstances() ([]discovery.Instance, error) { + authorizer, err := auth.NewAuthorizerFromEnvironment() + if err != nil { + log.Error(err) + return nil, err + } + + vmssinstanceClient := compute.NewVirtualMachineScaleSetVMsClient(d.Subscription) + vmssinstanceClient.Authorizer = authorizer + vmssClient := compute.NewVirtualMachineScaleSetsClient(d.Subscription) + vmssClient.Authorizer = authorizer + interfacesClient := network.NewInterfacesClient(d.Subscription) + interfacesClient.Authorizer = authorizer + interfaceIPConfigurationClient := network.NewInterfaceIPConfigurationsClient(d.Subscription) + interfaceIPConfigurationClient.Authorizer = authorizer + targetInstances := []discovery.Instance{} + + for vmssList, err := vmssClient.ListAll(context.Background()); vmssList.NotDone(); err = vmssList.Next() { + if err != nil { + log.Error(err) + return nil, err + } + + for _, vmss := range vmssList.Values() { + var metrics = []discovery.InstanceMetrics{} + if val, ok := vmss.Tags[d.Tag]; ok { + if !ok { + continue + } + json.Unmarshal([]byte(*val), &metrics) + } + + log.Debugf("Found VMSS: %v\n", *vmss.Name) + rg := strings.Split(*vmss.ID, "/")[4] + log.Debugf("VMSS RG: %v\n", rg) + //InterfaceIPConfiguration InterfaceIPConfigurationPropertiesFormat + for interfaceList, err := interfacesClient.ListVirtualMachineScaleSetNetworkInterfaces(context.Background(), rg, *vmss.Name); interfaceList.NotDone(); err = interfaceList.Next() { + if err != nil { + log.Error(err) + return nil, err + } + + for _, interfaceValue := range interfaceList.Values() { + //IPConfigurationPropertiesFormat + log.Debugf("Interface IPConfigs %v", &interfaceValue) + tagsBytes, err := json.Marshal(&vmss.Tags) + if err != nil { + log.Error(err) + return nil, err + } + var tags map[string]string + err = json.Unmarshal(tagsBytes, &tags) + if err != nil { + log.Error(err) + return nil, err + } + log.Debugf("Tags: %v", tags) + + for _, ipConfigValue := range *interfaceValue.IPConfigurations { + log.Debugf("ipConfiguration: %v", *ipConfigValue.PrivateIPAddress) + targetInstance := discovery.Instance{ + PrivateIP: *ipConfigValue.PrivateIPAddress, + InstanceType: *vmss.Name, + Tags: tags, + Metrics: metrics, + } + targetInstances = append(targetInstances, targetInstance) + } + + } + + } + } + } + + return targetInstances, nil +} diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 481a80f..b48f75a 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -1,142 +1,5 @@ package discovery -import ( - "errors" - "regexp" - "strconv" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" - log "github.com/sirupsen/logrus" -) - -type Discovery struct { - ec2Client ec2iface.EC2API - tagPrefix string -} - -func NewDiscovery(ec2Client ec2iface.EC2API, tagPrefix string) Discovery { - c := Discovery{ - ec2Client: ec2Client, - tagPrefix: tagPrefix, - } - - return c -} - -func (d Discovery) Discover() (*endpoints.DiscoveredInstances, error) { - ec2Input := ec2.DescribeInstancesInput{ - Filters: d.filter(), - } - output, err := d.ec2Client.DescribeInstances(&ec2Input) - if err != nil { - log.Error("Failed to load ec2 instances") - return nil, err - } - - instances := []endpoints.Instance{} - endpointCount := 0 - for _, res := range output.Reservations { - for _, instance := range res.Instances { - - log.Debugf("Extract metric endpoint(s) from instance with ip %s", *instance.PrivateIpAddress) - metricEndpoints := extractMetricEndpoints(instance.Tags, d.tagPrefix) - endpointCount += len(metricEndpoints) - log.Debugf("Instance with ip %s has %d metric endpoint(s)", *instance.PrivateIpAddress, len(metricEndpoints)) - - d := endpoints.Instance{ - InstanceType: *instance.InstanceType, - PrivateIP: *instance.PrivateIpAddress, - Tags: cleanupTagList(instance.Tags, d.tagPrefix), - Metrics: metricEndpoints, - } - - instances = append(instances, d) - } - } - log.Infof("Discovered %d instance(s) with %d endpoint(s)", len(instances), endpointCount) - return &endpoints.DiscoveredInstances{instances}, nil -} - -func (d Discovery) filter() []*ec2.Filter { - filters := []*ec2.Filter{ - { - Name: aws.String("tag-key"), - Values: []*string{aws.String(d.tagPrefix + "*")}, - }, - { - Name: aws.String("instance-state-name"), - Values: []*string{aws.String("running")}, - }, - } - - return filters -} - -func cleanupTagList(tags []*ec2.Tag, prefix string) map[string]string { - tagMap := make(map[string]string) - - for _, tag := range tags { - // remove prom/scrape annotation and aws based tags from tag list - if !matchKeyPattern(*tag.Key, prefix) && !strings.Contains(*tag.Key, ":") { - tagMap[*tag.Key] = *tag.Value - } - } - - return tagMap -} - -func extractMetricEndpoints(tags []*ec2.Tag, prefix string) []endpoints.InstanceMetrics { - metrics := []endpoints.InstanceMetrics{} - - for _, tag := range tags { - if matchKeyPattern(*tag.Key, prefix) { - parsedMetric, err := parseMetricEndpoint(*tag.Key, *tag.Value, prefix) - if err != nil { - log.Error("Failed to parse Tag, skip metric", err) - continue - } - - metrics = append(metrics, *parsedMetric) - } - } - - return metrics -} - -func parseMetricEndpoint(key string, value string, prefix string) (*endpoints.InstanceMetrics, error) { - r, _ := regexp.Compile(prefix + ":(.*?)(/.*)") - parsedMetric := r.FindStringSubmatch(key) - - if len(parsedMetric) == 3 { - //TO BE FAIR, NONE OF THIS IS A GOOD IDEA. THIS IS AT BEST A DIRTY AND FUGLY QUICK FIX. THIS SHOULD BE REFACTORED TO DO SOMETHING REASONABLE - var scheme = "http" - parsedString := strings.Split(parsedMetric[1], ":") - if len(parsedString) == 2 { - scheme = parsedString[1] - } - - parsedPort, err := strconv.ParseInt(parsedString[0], 10, 64) - if err != nil { - return nil, err - } - - return &endpoints.InstanceMetrics{ - Name: value, - Path: parsedMetric[2], - Port: parsedPort, - Scheme: scheme, - }, nil - } - - return nil, errors.New("Failed to match regex pattern") -} - -func matchKeyPattern(key string, prefix string) bool { - r, _ := regexp.Compile(prefix + ":(.*?)(/.*)") - - return r.MatchString(key) +type DiscoveryClient interface { + GetInstances() ([]Instance, error) } diff --git a/pkg/discovery/discovery_test.go b/pkg/discovery/discovery_test.go index 59eabda..5844159 100644 --- a/pkg/discovery/discovery_test.go +++ b/pkg/discovery/discovery_test.go @@ -1,44 +1 @@ package discovery - -import ( - "reflect" - "testing" - - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/daspawnw/prometheus-aws-discovery/pkg/libtesting" -) - -func TestDiscoveryHasCorrectEndpoints(t *testing.T) { - ec2Client := &MockEC2Client{ - instances: libtesting.EC2InstanceList(), - err: nil, - } - d := NewDiscovery(ec2Client, "prom/scrape") - returnedInstanceList, err := d.Discover() - if err != nil { - t.Error("Failed to discover instances", err) - } - expectedInstanceList := libtesting.InstanceList() - if !reflect.DeepEqual(expectedInstanceList, returnedInstanceList.Instances) { - t.Errorf("Expected instance list %v to equal returned instance list %v", expectedInstanceList, returnedInstanceList) - } -} - -type MockEC2Client struct { - ec2iface.EC2API - instances []*ec2.Instance - err error -} - -func (c *MockEC2Client) DescribeInstances(in *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { - var reservations []*ec2.Reservation - var reservation ec2.Reservation - - reservation.Instances = c.instances - reservations = append(reservations, &reservation) - - return &ec2.DescribeInstancesOutput{ - Reservations: reservations, - }, nil -} diff --git a/pkg/endpoints/endpoints.go b/pkg/discovery/endpoints.go similarity index 85% rename from pkg/endpoints/endpoints.go rename to pkg/discovery/endpoints.go index dc8b50a..a0de4d6 100644 --- a/pkg/endpoints/endpoints.go +++ b/pkg/discovery/endpoints.go @@ -1,4 +1,4 @@ -package endpoints +package discovery import ( "encoding/json" @@ -14,14 +14,10 @@ type Instance struct { } type InstanceMetrics struct { - Name string - Port int64 - Path string - Scheme string -} - -type DiscoveredInstances struct { - Instances []Instance + Name string `json:"name"` + Port int64 `json:"port"` + Path string `json:"path"` + Scheme string `json:"scheme"` } type outputFormat struct { @@ -29,10 +25,10 @@ type outputFormat struct { Labels map[string]string `json:"labels"` } -func ToJSONString(d DiscoveredInstances) ([]byte, error) { +func TargetConfigBytes(d []Instance) ([]byte, error) { outputList := []outputFormat{} - for _, instance := range d.Instances { + for _, instance := range d { for _, metric := range instance.Metrics { targetHost := fmt.Sprintf("%s:%d", instance.PrivateIP, metric.Port) l := labels(instance.Tags, targetHost, metric.Path, metric.Name, metric.Scheme) diff --git a/pkg/discovery/endpoints_test.go b/pkg/discovery/endpoints_test.go new file mode 100644 index 0000000..f38cd47 --- /dev/null +++ b/pkg/discovery/endpoints_test.go @@ -0,0 +1,50 @@ +package discovery + +import ( + "encoding/json" + "testing" + + "github.com/daspawnw/prometheus-aws-discovery/pkg/test" + "github.com/mitchellh/mapstructure" + "github.com/nsf/jsondiff" +) + +func TestJSONStringHasCorrectContent(t *testing.T) { + instances := make([]Instance, 0) + for _, instanceData := range test.Instances { + var instance Instance + mapstructure.Decode(instanceData, &instance) + instances = append(instances, instance) + } + returnedJSONContentBytes, _ := TargetConfigBytes(instances) + expectedJSON := test.Targets + expectedJSONBytes, _ := json.Marshal(expectedJSON) + opts := jsondiff.DefaultConsoleOptions() + res, text := jsondiff.Compare(returnedJSONContentBytes, expectedJSONBytes, &opts) + if res != 0 { + t.Errorf("JSONDiff \n%v \n%v", res, text) + } + + badJSON := []map[string]interface{}{ + { + "targets": []string{ + "127.0.0.1:9100", + }, + "labels": map[string]string{ + "__address__": "127.0.0.1:9100", + "__metrics_path__": "/metrics", + "__scheme__": "http", + "billingnumber": "1111", + "instancename": "Testinstance1", + "name": "node_exporter", + "spotprice": "123", + }, + }, + } + badJSONBytes, _ := json.Marshal(badJSON) + res, text = jsondiff.Compare(badJSONBytes, returnedJSONContentBytes, &opts) + if res == 0 { + t.Errorf("JSONDiff \n%v \n%v", res, text) + } + +} diff --git a/pkg/endpoints/endpoints_test.go b/pkg/endpoints/endpoints_test.go deleted file mode 100644 index d471fb3..0000000 --- a/pkg/endpoints/endpoints_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package endpoints - -import ( - "testing" -) - -func TestToJsonStringHasCorrectContent(t *testing.T) { - - instanceList := InstanceList() - - returnedJSONContentBytes, _ := ToJSONString(DiscoveredInstances{Instances: instanceList}) - expectedJSONContent := "[{\"targets\":[\"127.0.0.1:9100\"],\"labels\":{\"__address__\":\"127.0.0.1:9100\",\"__metrics_path__\":\"/metrics\",\"__scheme__\":\"http\",\"billingnumber\":\"1111\",\"instancename\":\"Testinstance1\",\"name\":\"node_exporter\",\"spotprice\":\"123\"}},{\"targets\":[\"127.0.0.1:8080\"],\"labels\":{\"__address__\":\"127.0.0.1:8080\",\"__metrics_path__\":\"/metrics\",\"__scheme__\":\"https\",\"billingnumber\":\"1111\",\"instancename\":\"Testinstance1\",\"name\":\"blackbox_exporter\",\"spotprice\":\"123\"}},{\"targets\":[\"127.0.0.2:9100\"],\"labels\":{\"__address__\":\"127.0.0.2:9100\",\"__metrics_path__\":\"/metrics\",\"__scheme__\":\"http\",\"billingnumber\":\"2222\",\"instancename\":\"Testinstance2\",\"name\":\"node_exporter\"}}]" - - if expectedJSONContent != string(returnedJSONContentBytes) { - t.Errorf("Expected json string with content %s, but got %s", expectedJSONContent, string(returnedJSONContentBytes)) - } - -} - -func InstanceList() []Instance { - expectedInstanceList := []Instance{} - - // Instance 1 - tagsI1 := make(map[string]string) - tagsI1["Name"] = "Testinstance1" - tagsI1["billingnumber"] = "1111" - tagsI1["Spot price"] = "123" - metricsI1 := []InstanceMetrics{} - m1I1 := InstanceMetrics{ - Name: "node_exporter", - Path: "/metrics", - Port: 9100, - Scheme: "http", - } - m2I1 := InstanceMetrics{ - Name: "blackbox_exporter", - Path: "/metrics", - Port: 8080, - Scheme: "https", - } - metricsI1 = append(metricsI1, m1I1, m2I1) - i1 := Instance{ - InstanceType: "t2.medium", - PrivateIP: "127.0.0.1", - Tags: tagsI1, - Metrics: metricsI1, - } - expectedInstanceList = append(expectedInstanceList, i1) - - // Instance 2 - tagsI2 := make(map[string]string) - tagsI2["Name"] = "Testinstance2" - tagsI2["billingnumber"] = "2222" - tagsI2[" "] = "empty" - metricsI2 := []InstanceMetrics{} - m1I2 := InstanceMetrics{ - Name: "node_exporter", - Path: "/metrics", - Port: 9100, - Scheme: "http", - } - metricsI2 = append(metricsI2, m1I2) - i2 := Instance{ - InstanceType: "t2.small", - PrivateIP: "127.0.0.2", - Tags: tagsI2, - Metrics: metricsI2, - } - expectedInstanceList = append(expectedInstanceList, i2) - return expectedInstanceList -} diff --git a/pkg/libtesting/libtesting.go b/pkg/libtesting/libtesting.go deleted file mode 100644 index 60be5e9..0000000 --- a/pkg/libtesting/libtesting.go +++ /dev/null @@ -1,116 +0,0 @@ -package libtesting - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" -) - -func InstanceList() []endpoints.Instance { - expectedInstanceList := []endpoints.Instance{} - - // Instance 1 - tagsI1 := make(map[string]string) - tagsI1["Name"] = "Testinstance1" - tagsI1["billingnumber"] = "1111" - metricsI1 := []endpoints.InstanceMetrics{} - m1I1 := endpoints.InstanceMetrics{ - Name: "node_exporter", - Path: "/metrics", - Port: 9100, - Scheme: "http", - } - m2I1 := endpoints.InstanceMetrics{ - Name: "blackbox_exporter", - Path: "/metrics", - Port: 8080, - Scheme: "https", - } - metricsI1 = append(metricsI1, m1I1, m2I1) - i1 := endpoints.Instance{ - InstanceType: "t2.medium", - PrivateIP: "127.0.0.1", - Tags: tagsI1, - Metrics: metricsI1, - } - expectedInstanceList = append(expectedInstanceList, i1) - - // Instance 2 - tagsI2 := make(map[string]string) - tagsI2["Name"] = "Testinstance2" - tagsI2["billingnumber"] = "2222" - metricsI2 := []endpoints.InstanceMetrics{} - m1I2 := endpoints.InstanceMetrics{ - Name: "node_exporter", - Path: "/metrics", - Port: 9100, - Scheme: "http", - } - metricsI2 = append(metricsI2, m1I2) - i2 := endpoints.Instance{ - InstanceType: "t2.small", - PrivateIP: "127.0.0.2", - Tags: tagsI2, - Metrics: metricsI2, - } - expectedInstanceList = append(expectedInstanceList, i2) - return expectedInstanceList -} - -func EC2InstanceList() []*ec2.Instance { - var instances []*ec2.Instance - - // instance 1 - var tagsInstance1 []*ec2.Tag - t11 := ec2.Tag{ - Key: aws.String("prom/scrape:9100/metrics"), - Value: aws.String("node_exporter"), - } - t12 := ec2.Tag{ - Key: aws.String("prom/scrape:8080:https/metrics"), - Value: aws.String("blackbox_exporter"), - } - nameTag1 := ec2.Tag{ - Key: aws.String("Name"), - Value: aws.String("Testinstance1"), - } - additionalTag1 := ec2.Tag{ - Key: aws.String("billingnumber"), - Value: aws.String("1111"), - } - tagsInstance1 = append(tagsInstance1, &t11, &t12, &nameTag1, &additionalTag1) - i1 := ec2.Instance{ - InstanceType: aws.String("t2.medium"), - PrivateIpAddress: aws.String("127.0.0.1"), - Tags: tagsInstance1, - } - instances = append(instances, &i1) - - // instance 2 - var tagsInstance2 []*ec2.Tag - t21 := ec2.Tag{ - Key: aws.String("prom/scrape:9100/metrics"), - Value: aws.String("node_exporter"), - } - nameTag2 := ec2.Tag{ - Key: aws.String("Name"), - Value: aws.String("Testinstance2"), - } - additionalTag2 := ec2.Tag{ - Key: aws.String("billingnumber"), - Value: aws.String("2222"), - } - exceptionTag := ec2.Tag{ - Key: aws.String("prom/scrape:8888"), - Value: aws.String("test_exporter"), - } - tagsInstance2 = append(tagsInstance2, &t21, &nameTag2, &additionalTag2, &exceptionTag) - i2 := ec2.Instance{ - InstanceType: aws.String("t2.small"), - PrivateIpAddress: aws.String("127.0.0.2"), - Tags: tagsInstance2, - } - instances = append(instances, &i2) - - return instances -} diff --git a/pkg/output/output.go b/pkg/output/output.go index 1b79fef..cf700cb 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,7 +1,7 @@ package output -import "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" +import "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" type Output interface { - Write(endpoints.DiscoveredInstances) error + Write([]discovery.Instance) error } diff --git a/pkg/outputfile/outputfile.go b/pkg/output/output_file.go similarity index 60% rename from pkg/outputfile/outputfile.go rename to pkg/output/output_file.go index 19bbc4b..94f562e 100644 --- a/pkg/outputfile/outputfile.go +++ b/pkg/output/output_file.go @@ -1,9 +1,9 @@ -package outputfile +package output import ( "io/ioutil" - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" log "github.com/sirupsen/logrus" ) @@ -11,16 +11,18 @@ type OutputFile struct { FilePath string } -func (o OutputFile) Write(instances endpoints.DiscoveredInstances) error { - content, err := endpoints.ToJSONString(instances) +func (o OutputFile) Write(instances []discovery.Instance) error { + content, err := discovery.TargetConfigBytes(instances) if err != nil { log.Error("Failed to convert instances to json string with error", err) return err } e := ioutil.WriteFile(o.FilePath, content, 0666) + if e != nil { log.Errorf("Failed to write to filepath %s", o.FilePath) + return err } - return e + return nil } diff --git a/pkg/output/output_file_test.go b/pkg/output/output_file_test.go new file mode 100644 index 0000000..ee62516 --- /dev/null +++ b/pkg/output/output_file_test.go @@ -0,0 +1,49 @@ +package output + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" +) + +func TestWriteToFile(t *testing.T) { + + instances := []discovery.Instance{ + { + InstanceType: "t2.small", + PrivateIP: "127.0.0.2", + Tags: map[string]string{ + "test": "tags", + }, + Metrics: []discovery.InstanceMetrics{ + { + Name: "someName", + Port: 8888, + Path: "/metrics", + Scheme: "https", + }, + }, + }, + } + + path := "_test.json" + o := OutputFile{ + FilePath: path, + } + + // initial cleanup + os.Remove(path) + + o.Write(instances) + + bs, err := ioutil.ReadFile(path) + + if err != nil || len(bs) == 0 { + t.Error("Failed to write instances list to file", err) + } + + os.Remove(path) + // final cleanup +} diff --git a/pkg/outputkubernetes/outputkubernetes.go b/pkg/output/output_kubernetes.go similarity index 88% rename from pkg/outputkubernetes/outputkubernetes.go rename to pkg/output/output_kubernetes.go index 446d48c..0f0339c 100644 --- a/pkg/outputkubernetes/outputkubernetes.go +++ b/pkg/output/output_kubernetes.go @@ -1,11 +1,11 @@ -package outputkubernetes +package output import ( "context" "io/ioutil" "strings" - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" log "github.com/sirupsen/logrus" apiv1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -17,7 +17,7 @@ import ( ) type OutputKubernetes struct { - clientset kubernetes.Interface + Clientset kubernetes.Interface Namespace string ConfigMapName string ConfigMapField string @@ -37,22 +37,22 @@ func NewOutputKubernetes(kubeconfig string, namespace string, cmName string, cmF } return &OutputKubernetes{ - clientset: clientset, + Clientset: clientset, ConfigMapField: cmField, ConfigMapName: cmName, Namespace: namespace, }, nil } -func (o OutputKubernetes) Write(instances endpoints.DiscoveredInstances) error { - output, err := endpoints.ToJSONString(instances) +func (o OutputKubernetes) Write(instances []discovery.Instance) error { + output, err := discovery.TargetConfigBytes(instances) if err != nil { log.Error("Failed to convert instances to json string") return err } ns := o.getNamespace() - cm, err := loadConfigmap(o.clientset, ns, o.ConfigMapName) + cm, err := loadConfigmap(o.Clientset, ns, o.ConfigMapName) if err != nil { if !errors.IsNotFound(err) { log.Error("Failed to load configmap from k8s", err) @@ -62,13 +62,13 @@ func (o OutputKubernetes) Write(instances endpoints.DiscoveredInstances) error { } if cm != nil { - err := updateConfigmap(o.clientset, cm, o.ConfigMapField, string(output)) + err := updateConfigmap(o.Clientset, cm, o.ConfigMapField, string(output)) if err != nil { log.Error("Failed to update configmap") } return err } else { - err := createConfigmap(o.clientset, o.ConfigMapName, ns, o.ConfigMapField, string(output)) + err := createConfigmap(o.Clientset, o.ConfigMapName, ns, o.ConfigMapField, string(output)) if err != nil { log.Error("Failed to create configmap") } diff --git a/pkg/outputkubernetes/outputkubernetes_test.go b/pkg/output/output_kubernetes_test.go similarity index 79% rename from pkg/outputkubernetes/outputkubernetes_test.go rename to pkg/output/output_kubernetes_test.go index a1fb445..94bcb2a 100644 --- a/pkg/outputkubernetes/outputkubernetes_test.go +++ b/pkg/output/output_kubernetes_test.go @@ -1,31 +1,45 @@ -package outputkubernetes +package output import ( "context" "testing" - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" - "github.com/daspawnw/prometheus-aws-discovery/pkg/libtesting" + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" fake "k8s.io/client-go/kubernetes/fake" ) +var iList = []discovery.Instance{ + { + InstanceType: "t2.small", + PrivateIP: "127.0.0.2", + Tags: map[string]string{ + "test": "tags", + }, + Metrics: []discovery.InstanceMetrics{ + { + Name: "someName", + Port: 8888, + Path: "/metrics", + Scheme: "https", + }, + }, + }, +} + func TestWriteShouldCreateConfigmap(t *testing.T) { var clientset kubernetes.Interface clientset = fake.NewSimpleClientset() o := OutputKubernetes{ - clientset: clientset, + Clientset: clientset, ConfigMapField: "discovery", ConfigMapName: "testcm", Namespace: "default", } - iList := endpoints.DiscoveredInstances{ - Instances: libtesting.InstanceList(), - } o.Write(iList) resp, _ := clientset.CoreV1().ConfigMaps("default").Get(context.TODO(), "testcm", metav1.GetOptions{}) @@ -52,14 +66,12 @@ func TestWriteShouldUpdateConfigmap(t *testing.T) { }) o := OutputKubernetes{ - clientset: clientset, + Clientset: clientset, ConfigMapField: "discovery", ConfigMapName: "testcm", Namespace: "default", } - iList := endpoints.DiscoveredInstances{ - Instances: libtesting.InstanceList(), - } + o.Write(iList) resp, _ := clientset.CoreV1().ConfigMaps("default").Get(context.TODO(), "testcm", metav1.GetOptions{}) diff --git a/pkg/output/output_std.go b/pkg/output/output_std.go new file mode 100644 index 0000000..4e87ee1 --- /dev/null +++ b/pkg/output/output_std.go @@ -0,0 +1,19 @@ +package output + +import ( + "fmt" + + "github.com/daspawnw/prometheus-aws-discovery/pkg/discovery" +) + +type OutputStdOut struct { +} + +func (o OutputStdOut) Write(instances []discovery.Instance) error { + jsonBytes, err := discovery.TargetConfigBytes(instances) + if err != nil { + return err + } + fmt.Printf("Discovered Scrape Configs: \n%v\n", string(jsonBytes)) + return nil +} diff --git a/pkg/outputfile/outputfile_test.go b/pkg/outputfile/outputfile_test.go deleted file mode 100644 index 98df693..0000000 --- a/pkg/outputfile/outputfile_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package outputfile - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/daspawnw/prometheus-aws-discovery/pkg/endpoints" - "github.com/daspawnw/prometheus-aws-discovery/pkg/libtesting" -) - -func TestWriteToFile(t *testing.T) { - instances := endpoints.DiscoveredInstances{ - Instances: libtesting.InstanceList(), - } - path := "_test.json" - o := OutputFile{ - FilePath: path, - } - - // initial cleanup - os.Remove(path) - - o.Write(instances) - - bs, err := ioutil.ReadFile(path) - - if err != nil || len(bs) == 0 { - t.Error("Failed to write instances list to file", err) - } - - os.Remove(path) - // final cleanup -} diff --git a/pkg/test/constants.go b/pkg/test/constants.go new file mode 100644 index 0000000..dfe5a6f --- /dev/null +++ b/pkg/test/constants.go @@ -0,0 +1,90 @@ +package test + +var V2TagValue = `[{"name": "ref-test","port": 9100,"path": "metrics","scheme": "https"},{"name": "ref-node","port": 8080,"path": "node","scheme": "http"}]` +var Targets = []map[string]interface{}{ + { + "targets": []string{ + "127.0.0.1:9100", + }, + "labels": map[string]string{ + "__address__": "127.0.0.1:9100", + "__metrics_path__": "/metrics", + "__scheme__": "http", + "billingnumber": "1111", + "instancename": "Testinstance1", + "name": "node_exporter", + }, + }, + { + "targets": []string{ + "127.0.0.1:8080", + }, + "labels": map[string]string{ + "__address__": "127.0.0.1:8080", + "__metrics_path__": "/metrics", + "__scheme__": "https", + "billingnumber": "1111", + "instancename": "Testinstance1", + "name": "blackbox_exporter", + }, + }, + { + "targets": []string{ + "127.0.0.2:9100", + }, + "labels": map[string]string{ + "__address__": "127.0.0.2:9100", + "__metrics_path__": "/metrics", + "__scheme__": "http", + "billingnumber": "2222", + "instancename": "Testinstance2", + "name": "node_exporter", + }, + }, +} +var Instances = []map[string]interface{}{ + { + "InstanceType": "t2.medium", + "PrivateIP": "127.0.0.1", + "Tags": map[string]string{ + "Name": "Testinstance1", + "billingnumber": "1111", + }, + "Metrics": []map[string]interface{}{ + { + "Name": "node_exporter", + "Port": 9100, + "Path": "/metrics", + "Scheme": "http", + }, { + "Name": "blackbox_exporter", + "Path": "/metrics", + "Port": 8080, + "Scheme": "https", + }, + }, + }, { + "InstanceType": "t2.small", + "PrivateIP": "127.0.0.2", + "Tags": map[string]string{ + "Name": "Testinstance2", + "billingnumber": "2222", + }, + "Metrics": []map[string]interface{}{ + { + "Name": "node_exporter", + "Path": "/metrics", + "Port": 9100, + "Scheme": "http", + }, + }, + }, { + "InstanceType": "t2.small", + "PrivateIP": "127.0.0.2", + "Tags": map[string]string{ + "Name": "Testinstance3", + "billingnumber": "2222", + }, + "Metrics": make(map[string]interface{}, 0), + }, +} diff --git a/readme.md b/readme.md index ed84a3e..575442c 100644 --- a/readme.md +++ b/readme.md @@ -1,11 +1,28 @@ -# Prometheus AWS Discovery +# Prometheus AWS/Azure Tag based Discovery -Prometheus AWS Discovery provides a more flexible way to discover exporters running on ec2 instances. +Prometheus AWS/Azure Discovery provides a more flexible way to discover exporters running on ec2 instances or withing VMSS Instances (Azure VMs currently not covered by binary). + +### AUTH + +AWS Auth is done via the AWS go sdk. Meaning it shoud support ENV Vars & Instance Profiles by default [AWS Docs](https://docs.aws.amazon.com/sdk-for-go/api/aws/session/) + +Azure Auth requires providing ENV Vars. For supported combinations see [Azure Docs](https://github.com/Azure/azure-sdk-for-go#more-authentication-details) + +### Discovery Tag The normal Prometheus ec2_sd_configs object is very limited and doesn't support extended discovery. This limits it for example to the specified list of ports inside the prometheus configuration file. -To overcome this issue this project allows discovery based on EC2 instance tags in a format that the file_sd_config discovery function can read. +To overcome this issue this project allows discovery based on EC2/VMSS tags in a format that the file_sd_config discovery function can read. + +IMPORTANT: V2 Tags for discovery +if the flag --tag is provided, the binary will look for the V2 Tag Structure. +``` +key: example +value: [{"name": "ref-test","port": 9100,"path": "metrics","scheme": "https"},{...},{...}] +``` +IMPORTANT: The below applies to v1 tags. These will be deprecated soon. +if the flag `--tag-prefix` is provided, the binary will currently look for v1 tags with below structure. A tag must have the following format: ``` @@ -44,4 +61,4 @@ This will produce the following file_sd_config: ] ``` -where "..." is the list of tags associated with the EC2 Instance, all AWS specific tags starting with "aws:" are removed from the list of tags. \ No newline at end of file +where "..." is the list of tags associated with the EC2 Instance, all AWS specific tags starting with "aws:" are removed from the list of tags.