From a8f07c8e97bacc75f65e2a1c6cd357da1ba7df98 Mon Sep 17 00:00:00 2001 From: Bill Rich Date: Sun, 3 Apr 2022 10:51:56 -0800 Subject: [PATCH] Automatically update trufflehog (#121) Co-authored-by: Dustin Decker --- .goreleaser.yml | 4 +- go.mod | 7 +++ go.sum | 14 ++++++ main.go | 94 +++++++++++++++++++++++++++++++++- pkg/updater/updater.go | 112 +++++++++++++++++++++++++++++++++++++++++ pkg/version/version.go | 3 ++ 6 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 pkg/updater/updater.go create mode 100644 pkg/version/version.go diff --git a/.goreleaser.yml b/.goreleaser.yml index 2ba53f5a6a7e..6ca48616e22c 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,4 +1,6 @@ builds: + ldflags: + - -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}' - env: [CGO_ENABLED=0] binary: trufflehog goos: @@ -42,4 +44,4 @@ docker_manifests: - name_template: ghcr.io/trufflesecurity/{{ .ProjectName }}:latest image_templates: - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64 - - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8 \ No newline at end of file + - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8 diff --git a/go.mod b/go.mod index e3b36dccace5..a19fc49f8a3c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.17 replace github.com/gitleaks/go-gitdiff => github.com/bill-rich/go-gitdiff v0.7.6-custom1 +replace github.com/jpillora/overseer => github.com/dustin-decker/overseer v1.1.7-custom2 + require ( cloud.google.com/go/secretmanager v1.3.0 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 @@ -13,6 +15,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.16.2 github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 + github.com/denisbrodbeck/machineid v1.0.1 github.com/envoyproxy/protoc-gen-validate v0.6.7 github.com/fatih/color v1.13.0 github.com/felixge/fgprof v0.9.2 @@ -25,6 +28,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-retryablehttp v0.7.0 github.com/joho/godotenv v1.4.0 + github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.12 github.com/paulbellamy/ratecounter v0.2.0 @@ -70,6 +74,7 @@ require ( github.com/emirpasic/gods v1.12.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -82,6 +87,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jpillora/s3 v1.1.4 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -90,6 +96,7 @@ require ( github.com/modern-go/reflect2 v1.0.1 // indirect github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/mod v0.5.0 // indirect golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect diff --git a/go.sum b/go.sum index 7eb8df887d49..4837610de61e 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dustin-decker/overseer v1.1.7-custom2 h1:k6lSAUkdxnICmN5/sEsCKFVYchv2wBfogAdwONk39y0= +github.com/dustin-decker/overseer v1.1.7-custom2/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -186,6 +190,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -331,6 +337,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc= +github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -416,7 +424,11 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU= +github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -447,6 +459,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zricethezav/gitleaks/v8 v8.5.2 h1:+IiLvmBiMGetpgNtzC71EHktTXc1Zt7m9AwnqhhJ04g= github.com/zricethezav/gitleaks/v8 v8.5.2/go.mod h1:TxBuxH8eB/1la8Mc7I6j/ZZYNG/mHGpoi09N2oHx5nQ= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/main.go b/main.go index 567422efbe72..0fa991645134 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "gopkg.in/alecthomas/kingpin.v2" "log" "net/http" _ "net/http/pprof" @@ -12,8 +11,14 @@ import ( "runtime" "strconv" "strings" + "syscall" "time" + "github.com/jpillora/overseer" + "github.com/trufflesecurity/trufflehog/v3/pkg/updater" + "github.com/trufflesecurity/trufflehog/v3/pkg/version" + "gopkg.in/alecthomas/kingpin.v2" + "github.com/felixge/fgprof" "github.com/gorilla/mux" @@ -25,9 +30,91 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) +var ( + cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.") + cmd string + debug = cli.Flag("debug", "Run in debug mode").Bool() + versionFlag = cli.Flag("version", "Prints trufflehog version.").Bool() + jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool() + jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool() + concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int() + noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool() + onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool() + // rules = cli.Flag("rules", "Path to file with custom rules.").String() + printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool() + noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool() + + gitScan = cli.Command("git", "Find credentials in git repositories.") + gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String() + gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() + gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String() + gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String() + gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int() + _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool() + _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool() + _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool() + + githubScan = cli.Command("github", "Find credentials in GitHub repositories.") + githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() + githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() + githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() + githubScanToken = githubScan.Flag("token", "GitHub token.").String() + githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() + githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() + + gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.") + // TODO: Add more GitLab options + gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String() + gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings() + gitlabScanToken = gitlabScan.Flag("token", "GitLab token.").Required().String() + + filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.") + filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings() + // TODO: Add more filesystem scan options. Currently only supports scanning a list of directories. + // filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool() + // filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() + // filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + + s3Scan = cli.Command("s3", "Find credentials in S3 buckets.") + s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate.").String() + s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate.").String() + s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool() + s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings() +) + +func init() { + for i, arg := range os.Args { + if strings.HasPrefix(arg, "--") { + os.Args[i] = strings.ReplaceAll(arg, "_", "-") + } + } + + cmd = kingpin.MustParse(cli.Parse(os.Args[1:])) +} + func main() { + updateCfg := overseer.Config{ + Program: run, + Debug: *debug, + RestartSignal: syscall.SIGTERM, + // TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15 + // PreUpgrade: checkUpdateSignature(binaryPath string), + } + + if !*noUpdate { + updateCfg.Fetcher = updater.Fetcher(version.BuildVersion) + } + err := overseer.RunErr(updateCfg) + if err != nil { + logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷") + } +} + +func run(state overseer.State) { cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.") debug := cli.Flag("debug", "Run in debug mode").Bool() + versionFlag := cli.Flag("version", "Prints trufflehog version.").Bool() jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool() jsonLegacy := cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool() concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int() @@ -82,6 +169,11 @@ func main() { cmd := kingpin.MustParse(cli.Parse(os.Args[1:])) + if *versionFlag { + fmt.Println("trufflehog " + version.BuildVersion) + return + } + // When setting a base commit, chunks must be scanned in order. if *gitScanSinceCommit != "" { *concurrency = 1 diff --git a/pkg/updater/updater.go b/pkg/updater/updater.go new file mode 100644 index 000000000000..c41face4ff32 --- /dev/null +++ b/pkg/updater/updater.go @@ -0,0 +1,112 @@ +package updater + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "runtime" + "strings" + "time" + + "github.com/go-errors/errors" + "github.com/jpillora/overseer/fetcher" + log "github.com/sirupsen/logrus" + + "github.com/trufflesecurity/trufflehog/v3/pkg/version" +) + +func Fetcher(version string) fetcher.Interface { + return &OSS{ + CurrentVersion: version, + } +} + +type OSS struct { + Interval time.Duration + CurrentVersion string +} + +// Init validates the provided config +func (g *OSS) Init() error { + //initiate OSS connection + return nil +} + +const url = "https://oss.trufflehog.org/updates" + +type FormData struct { + OS string + Arch string + CurrentVersion string + Timezone string + Binary string +} + +// Fetch binary from URL via OSS client +func (g *OSS) Fetch() (io.Reader, error) { + log.Debug("fetching trufflehog update") + + zone, _ := time.Now().Zone() + data := &FormData{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + CurrentVersion: version.BuildVersion, + Timezone: zone, + Binary: "trufflehog", + } + + dataByte, err := json.Marshal(data) + if err != nil { + return nil, err + } + reader := bytes.NewReader(dataByte) + resp, err := http.Post(url, "application/json", reader) + if err != nil || resp == nil { + return nil, err + } + defer resp.Body.Close() + + newBinBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + buffer := bytes.NewReader(newBinBytes) + switch runtime.GOOS { + case "windows": + zipReader, err := zip.NewReader(buffer, int64(len(newBinBytes))) + if err != nil { + return nil, errors.Errorf("Failed to read zip archive: %s", err) + } + for _, f := range zipReader.File { + if strings.HasPrefix(f.Name, "trufflehog") { + return f.Open() + } + } + default: + gzipReader, err := gzip.NewReader(buffer) + if err != nil { + return nil, errors.Errorf("Failed to read gzip archive: %s", err) + } + defer gzipReader.Close() + tarReader := tar.NewReader(gzipReader) + for { + header, err := tarReader.Next() + if err == io.EOF { + return nil, errors.New("unable to get update") + } + + if header.Typeflag == tar.TypeReg { + if strings.HasPrefix(header.Name, "trufflehog") { + return tarReader, nil + } + } + } + } + return nil, errors.New("unable to get update") +} diff --git a/pkg/version/version.go b/pkg/version/version.go new file mode 100644 index 000000000000..3c0765e71209 --- /dev/null +++ b/pkg/version/version.go @@ -0,0 +1,3 @@ +package version + +var BuildVersion = "dev"