Merge pull request #423 from jzelinskie/sleep-updater

updater: sleep before continuing the lock loop

updater.go

@@ -31,10 +31,11 @@ import (
 )
 
 const (
-	updaterLastFlagName        = "updater/last"
+	updaterLastFlagName              = "updater/last"
-	updaterLockName            = "updater"
+	updaterLockName                  = "updater"
-	updaterLockDuration        = updaterLockRefreshDuration + time.Minute*2
+	updaterLockDuration              = updaterLockRefreshDuration + time.Minute*2
-	updaterLockRefreshDuration = time.Minute * 8
+	updaterLockRefreshDuration       = time.Minute * 8
+	updaterSleepBetweenLoopsDuration = time.Minute
 )
 
 var (
@@ -124,7 +125,14 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 				if stop {
 					break
 				}
+
+				// Sleep for a short duration to prevent pinning the CPU on a
+				// consistent failure.
+				if stopped := sleepUpdater(time.Now().Add(updaterSleepBetweenLoopsDuration), st); stopped {
+					break
+				}
 				continue
+
 			} else {
 				lockOwner, lockExpiration, err := datastore.FindLock(updaterLockName)
 				if err != nil {
@@ -137,14 +145,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 			}
 		}
 
-		// Sleep, but remain stoppable until approximately the next update time.
+		if stopped := sleepUpdater(nextUpdate, st); stopped {
-		now := time.Now().UTC()
+			break
-		waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
-		log.WithField("scheduled time", waitUntil).Debug("next update attempt scheduled")
-		if !waitUntil.Before(now) {
-			if !st.Sleep(waitUntil.Sub(time.Now())) {
-				break
-			}
 		}
 	}
 
@@ -159,6 +161,19 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 	log.Info("updater service stopped")
 }
 
+// sleepUpdater sleeps the updater for an approximate duration, but remains
+// able to be cancelled by a stopper.
+func sleepUpdater(approxWakeup time.Time, st *stopper.Stopper) (stopped bool) {
+	waitUntil := approxWakeup.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
+	log.WithField("scheduled time", waitUntil).Debug("updater sleeping")
+	if !waitUntil.Before(time.Now().UTC()) {
+		if !st.Sleep(waitUntil.Sub(time.Now())) {
+			return true
+		}
+	}
+	return false
+}
+
 // update fetches all the vulnerabilities from the registered fetchers, upserts
 // them into the database and then sends notifications.
 func update(datastore database.Datastore, firstUpdate bool) {