diff --git a/Makefile b/Makefile index 22be86e..e5327ad 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,9 @@ BINARIES=$(addprefix bin/mtr-exporter-$(VERSION)., $(BUILDS)) LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildDate=$(BUILD_DATE) -X main.GitHash=$(GIT_HASH)" -mtr-exporter: cmd/mtr-exporter - go build -v -o $@ ./$^ +mtr-exporter: bin/mtr-exporter +bin/mtr-exporter: cmd/mtr-exporter bin + go build -v -o $@ ./$< ###################################################### ## release related diff --git a/README.md b/README.md index 45d7c38..06c6393 100644 --- a/README.md +++ b/README.md @@ -43,34 +43,40 @@ When [prometheus] scrapes the data, you can visualise the observed values: ## Usage - $> mtr-exporter [OPTS] -- [MTR-OPTS] - - mtr-exporter [FLAGS] -- [MTR-FLAGS] + $> mtr-exporter [FLAGS] -- [MTR-FLAGS] FLAGS: - -bind - bind address (default ":8080") - -h show help - -mtr - path to mtr binary (default "mtr") - -schedule - schedule at which often mtr is launched (default "@every 60s") - examples: - @every - example "@every 60s" - @hourly - run once per hour - 10 * * * * - execute 10 minutes after the full hour - see https://en.wikipedia.org/wiki/Cron - -tslogs - use timestamps in logs + -bind + bind address (default ":8080") + -h + show help + -jobs + file describing multiple mtr-jobs. syntax is given below. + -label + use in prometheus-metrics (default: "mtr-exporter-cli") + -mtr + path to mtr binary (default: "mtr") + -schedule + schedule at which often mtr is launched (default: "@every 60s") + examples: + @every - example "@every 60s" + @hourly - run once per hour + 10 * * * * - execute 10 minutes after the full hour + see https://en.wikipedia.org/wiki/Cron + -tslogs + use timestamps in logs + -watch-jobs + periodically watch the file defined via -jobs (default: "") + if it has changed stop previously running mtr-jobs and apply + all jobs defined in -jobs. -version - show version - + show version MTR-FLAGS: see "man mtr" for valid flags to mtr. At `/metrics` the measured values of the last run are exposed. -Examples: +### Examples $> mtr-exporter -- example.com # probe every minute "example.com" @@ -82,6 +88,18 @@ Examples: # probe every 30s "example.com", wait 1s for response, try a max of 3 hops, # use interface "ven3", do not resolve DNS. +### Jobs-File Syntax + + # comment lines start with '#' are ignored + # empty lines are ignored as well + label -- -- mtr-flags + +Example: + + quad9 -- @every 120s -- -I ven1 -n 9.9.9.9 + example.com -- @every 45s -- -I ven2 -n example.com + + ## Requirements Runtime: @@ -100,7 +118,7 @@ Build: One-off building and "installation": - $> go get github.com/mgumz/mtr-exporter/cmd/mtr-exporter + $> go install github.com/mgumz/mtr-exporter/cmd/mtr-exporter@latest ## License diff --git a/cmd/mtr-exporter/main.go b/cmd/mtr-exporter/main.go index 979628a..811cdcb 100644 --- a/cmd/mtr-exporter/main.go +++ b/cmd/mtr-exporter/main.go @@ -10,6 +10,8 @@ import ( "net/http" "os" + "github.com/mgumz/mtr-exporter/pkg/job" + "github.com/robfig/cron/v3" ) @@ -17,8 +19,11 @@ func main() { log.SetFlags(0) mtrBin := flag.String("mtr", "mtr", "path to `mtr` binary") + jobLabel := flag.String("label", "mtr-exporter-cli", "job label") bind := flag.String("bind", ":8080", "bind address") - schedule := flag.String("schedule", "@every 60s", "Schedule at which often `mtr` is launched") + jobFile := flag.String("jobs", "", "file containing job definitions") + schedule := flag.String("schedule", "@every 60s", "schedule at which often `mtr` is launched") + doWatchJobsFile := flag.String("watch-jobs", "", "re-parse -jobs file to schedule") doPrintVersion := flag.Bool("version", false, "show version") doPrintUsage := flag.Bool("h", false, "show help") doTimeStampLogs := flag.Bool("tslogs", false, "use timestamps in logs") @@ -30,44 +35,61 @@ func main() { printVersion() return } - if *doPrintUsage { flag.Usage() return } - if *doTimeStampLogs { log.SetFlags(log.LstdFlags | log.LUTC) } - if len(flag.Args()) == 0 { - log.Println("error: no mtr arguments given - at least the target host must be defined.") - os.Exit(1) + collector := job.NewCollector() + scheduler := cron.New() - return + if len(flag.Args()) > 0 { + j := job.NewJob(*mtrBin, flag.Args(), *schedule) + j.Label = *jobLabel + if _, err := scheduler.AddJob(j.Schedule, j); err != nil { + log.Printf("error: unable to add %q to scheduler: %v", j.Label, err) + os.Exit(1) + } + if !collector.AddJob(j.JobMeta) { + log.Printf("error: unable to add %q to collector", j.Label) + os.Exit(1) + } + j.UpdateFn = func(meta job.JobMeta) bool { return collector.UpdateJob(meta) } } - job := newMtrJob(*mtrBin, flag.Args()) - - c := cron.New() - - _, err := c.AddFunc(*schedule, func() { - log.Println("launching", job.cmdLine) - if err := job.Launch(); err != nil { - log.Println("failed:", err) - return + if *jobFile != "" { + if *doWatchJobsFile != "" { + log.Printf("info: watching %q at %q", *jobFile, *doWatchJobsFile) + job.WatchJobsFile(*jobFile, *mtrBin, *doWatchJobsFile, collector) + } else { + jobs, _, err := job.ParseJobFile(*jobFile, *mtrBin) + if err != nil { + log.Printf("error: parsing jobs file %q: %s", *jobFile, err) + os.Exit(1) + } + if jobs.Empty() { + log.Println("error: no mtr jobs defined - provide at least one via -file or via arguments") + os.Exit(1) + } + for _, j := range jobs { + if collector.AddJob(j.JobMeta) { + if _, err := scheduler.AddJob(j.Schedule, j); err != nil { + log.Printf("error: unable to add %q to collector: %v", j.Label, err) + os.Exit(1) + } + j.UpdateFn = func(meta job.JobMeta) bool { return collector.UpdateJob(meta) } + } // FIXME: log failed addition to collector, most likely + // due to duplicate label + } } - log.Println("done: ", - len(job.Report.Hubs), "hops in", job.Duration, ".") - }) - if err != nil { - log.Fatalf(err.Error()) - os.Exit(1) } - c.Start() + scheduler.Start() - http.Handle("/metrics", job) + http.Handle("/metrics", collector) http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "OK") }) diff --git a/cmd/mtr-exporter/prometheus.go b/cmd/mtr-exporter/prometheus.go deleted file mode 100644 index 6ecc7ca..0000000 --- a/cmd/mtr-exporter/prometheus.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "fmt" - "io" - "net/http" - "sort" - "strconv" - "strings" - "time" -) - -// ServeHTTP writes promtheues styled metrics about the last executed `mtr` -// run, see https://prometheus.io/docs/instrumenting/exposition_formats/#line-format -// -// NOTE: at the moment, no use of github.com/prometheus/client_golang/prometheus -// because overhead in size and complexity. once mtr-exporter requires features -// like push-gateway-export or graphite export or the like, we switch. -func (job *mtrJob) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if job.Report == nil { - fmt.Fprintln(w, "# no current mtr runs performed (yet).") - return - } - - // the original job.Report might be changed in the - // background by a successful run of mtr. copy (pointer to) the report - // to have something safe to work on - job.Lock() - report := job.Report - ts := job.Launched.UTC() - d := job.Duration - job.Unlock() - - labels := report.Mtr.Labels() - tsMs := ts.UnixNano() / int64(time.Millisecond) - - fmt.Fprintf(w, "# mtr run: %s\n", ts.Format(time.RFC3339Nano)) - fmt.Fprintf(w, "# cmdline: %s\n", job.cmdLine) - fmt.Fprintf(w, "mtr_report_duration_ms_gauge{%s} %d %d\n", - labels2Prom(labels), d/time.Millisecond, tsMs) - fmt.Fprintf(w, "mtr_report_count_hubs_gauge{%s} %d %d\n", - labels2Prom(labels), len(report.Hubs), tsMs) - - for i, hub := range report.Hubs { - labels["host"] = hub.Host - labels["count"] = strconv.FormatInt(hub.Count, integerBase) - // mark last hub to have it easily identified - if i < (len(report.Hubs) - 1) { - hub.writeMetrics(w, labels2Prom(labels), tsMs) - } else { - labels["last"] = "true" - hub.writeMetrics(w, labels2Prom(labels), tsMs) - delete(labels, "last") - } - } -} - -func (hub *mtrHub) writeMetrics(w io.Writer, labels string, ts int64) { - fmt.Fprintf(w, "mtr_report_loss_gauge{%s} %f %d\n", labels, hub.Loss, ts) - fmt.Fprintf(w, "mtr_report_snt_gauge{%s} %d %d\n", labels, hub.Snt, ts) - fmt.Fprintf(w, "mtr_report_last_gauge{%s} %f %d\n", labels, hub.Last, ts) - fmt.Fprintf(w, "mtr_report_avg_gauge{%s} %f %d\n", labels, hub.Avg, ts) - fmt.Fprintf(w, "mtr_report_best_gauge{%s} %f %d\n", labels, hub.Best, ts) - fmt.Fprintf(w, "mtr_report_wrst_gauge{%s} %f %d\n", labels, hub.Wrst, ts) - fmt.Fprintf(w, "mtr_report_stdev_gauge{%s} %f %d\n", labels, hub.StDev, ts) -} - -func labels2Prom(labels map[string]string) string { - sl := make(sort.StringSlice, 0, len(labels)) - for k, v := range labels { - sl = append(sl, fmt.Sprintf("%s=%q", k, v)) - } - - sl.Sort() - - return strings.Join(sl, ",") -} diff --git a/cmd/mtr-exporter/usage.go b/cmd/mtr-exporter/usage.go index e8a0607..287232b 100644 --- a/cmd/mtr-exporter/usage.go +++ b/cmd/mtr-exporter/usage.go @@ -3,25 +3,35 @@ package main import "fmt" func usage() { - usage := `Usage: mtr-exporter [FLAGS] -- [MTR-FLAGS] + + const usage string = `Usage: mtr-exporter [FLAGS] -- [MTR-FLAGS] FLAGS: --bind - bind address (default ":8080") --h show help --mtr - path to mtr binary (default "mtr") --schedule - schedule at which often mtr is launched (default "@every 60s") - examples: - @every - example "@every 60s" - @hourly - run once per hour - 10 * * * * - execute 10 minutes after the full hour - see https://en.wikipedia.org/wiki/Cron --tslogs - use timestamps in logs +-bind + bind address (default ":8080") +-h + show help +-jobs + file describing multiple mtr-jobs. syntax is given below. +-label + use in prometheus-metrics (default: "mtr-exporter-cli") +-mtr + path to mtr binary (default: "mtr") +-schedule + schedule at which often mtr is launched (default: "@every 60s") + examples: + @every - example "@every 60s" + @hourly - run once per hour + 10 * * * * - execute 10 minutes after the full hour + see https://en.wikipedia.org/wiki/Cron +-tslogs + use timestamps in logs +-watch-jobs + periodically watch the file defined via -jobs (default: "") + if it has changed stop previously running mtr-jobs and apply + all jobs defined in -jobs. -version - show version + show version MTR-FLAGS: see "man mtr" for valid flags to mtr. @@ -36,7 +46,13 @@ $> mtr-exporter -- -n example.com $> mtr-exporter -schedule "@every 30s" -- -G 1 -m 3 -I ven3 -n example.com # probe every 30s "example.com", wait 1s for response, try a max of 3 hops, -# use interface "ven3", do not resolve DNS.` +# use interface "ven3", do not resolve DNS. + +Example Job File: + + # comments are ignored + job1 -- @every 30s -- -I ven1 -n example.com + job2 -- @every 30s -- -I ven2 -n example.com` fmt.Println(usage) } diff --git a/go.mod b/go.mod index cbba384..07f4ed7 100644 --- a/go.mod +++ b/go.mod @@ -3,5 +3,6 @@ module github.com/mgumz/mtr-exporter go 1.14 require ( + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/robfig/cron/v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 0667807..7780d65 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= diff --git a/pkg/job/collector.go b/pkg/job/collector.go new file mode 100644 index 0000000..6f9dc33 --- /dev/null +++ b/pkg/job/collector.go @@ -0,0 +1,65 @@ +package job + +import ( + "sync" +) + +type Collector struct { + jobs []JobMeta + + mu sync.Mutex +} + +func NewCollector() *Collector { + return new(Collector) +} + +func (c *Collector) RemoveJob(label string) bool { + + c.mu.Lock() + defer c.mu.Unlock() + + jobs := []JobMeta{} + + for i := range c.jobs { + if c.jobs[i].Label != label { + jobs = append(jobs, c.jobs[i]) + } + } + + if len(jobs) < len(c.jobs) { + c.jobs = jobs + return true + } + return false +} + +func (c *Collector) AddJob(job JobMeta) bool { + + c.mu.Lock() + defer c.mu.Unlock() + + for i := range c.jobs { + if job.Label == c.jobs[i].Label { + return false + } + } + c.jobs = append(c.jobs, job) + + return true +} + +func (c *Collector) UpdateJob(job JobMeta) bool { + + c.mu.Lock() + defer c.mu.Unlock() + + for i := range c.jobs { + if c.jobs[i].Label == job.Label { + c.jobs[i] = job + return true + } + } + + return false +} diff --git a/pkg/job/cron.go b/pkg/job/cron.go new file mode 100644 index 0000000..5cbec1e --- /dev/null +++ b/pkg/job/cron.go @@ -0,0 +1,17 @@ +package job + +import ( + "log" +) + +// cron.v3 interface +func (job *Job) Run() { + + log.Printf("info: %q launching via %q", job.Label, job.cmdLine) + if err := job.Launch(); err != nil { + log.Printf("info: %q failed: %s", job.Label, err) + return + } + log.Printf("info: %q done: %d hops in %s.", job.Label, + len(job.Report.Hubs), job.Duration) +} diff --git a/pkg/job/file.go b/pkg/job/file.go new file mode 100644 index 0000000..2fd47ba --- /dev/null +++ b/pkg/job/file.go @@ -0,0 +1,28 @@ +package job + +import ( + "crypto/sha256" + "io" + "os" +) + +// mtrJobFile definition +// +// # comments, ignore everything after # +// ^space*$ - empty lines +//