feat: jitter on cron jobs

job/job.go

@@ -3,7 +3,9 @@ package job
 import (
 	"container/ring"
 	gocontext "context"
+	"crypto/rand"
 	"fmt"
+	"math/big"
 	"strconv"
 	"sync"
 	"time"
@@ -26,6 +28,11 @@ const (
 	ResourceTypeUpstream      = "upstream"
 )
 
+const (
+	// iterationJitterPercent sets the maximum percent by which to jitter each subsequent invocation of a periodic job
+	iterationJitterPercent = 10
+)
+
 var evictedJobs chan uuid.UUID
 
 // deleteEvictedJobs deletes job_history rows from the DB every job.eviction.period(1m),
@@ -488,7 +495,6 @@ func (j *Job) GetResourcedName() string {
 }
 
 func (j *Job) AddToScheduler(cronRunner *cron.Cron) error {
-
 	echo.RegisterCron(cronRunner)
 	cronRunner.Start()
 
@@ -511,20 +517,39 @@ func (j *Job) AddToScheduler(cronRunner *cron.Cron) error {
 		j.Context.Infof("skipping scheduling")
 		return nil
 	}
-
 	j.Context.Logger.Named(j.GetResourcedName()).V(1).Infof("scheduled %s", schedule)
-	entryID, err := cronRunner.AddJob(schedule, j)
+
+	// Attempt to get a fixed interval from the schedule
+	// to measure the appropriate jitter.
+	// NOTE: Only works for fixed interval schedules.
+	parsedSchedule, err := cron.ParseStandard(j.Schedule)
+	if err != nil {
+		return fmt.Errorf("failed to parse schedule: %s", err)
+	}
+	interval := time.Until(parsedSchedule.Next(time.Now()))
+
+	delayedJob := cron.FuncJob(func() {
+		delayPercent, _ := rand.Int(rand.Reader, big.NewInt(int64(iterationJitterPercent)))
+		jitterDuration := time.Duration((int64(interval) * delayPercent.Int64()) / 100)
+		j.Context.Logger.Named(j.GetResourcedName()).V(2).Infof("jitter (%d%%): %s", delayPercent, jitterDuration)
+		time.Sleep(jitterDuration)
+		j.Run()
+	})
+	entryID, err := cronRunner.AddJob(schedule, delayedJob)
 	if err != nil {
 		return fmt.Errorf("[%s] failed to schedule job: %s", j.Label(), err)
 	}
 	j.entryID = &entryID
+
 	if j.RunNow {
 		// Run in a goroutine since AddToScheduler should be non-blocking
 		defer func() { go j.Run() }()
 	}
+
 	j.unschedule = func() {
 		cronRunner.Remove(*j.entryID)
 	}
+
 	return nil
 }