Merge branch 'nvdfix'

ext/vulnmdsrc/nvd/nvd.go

@@ -57,8 +57,9 @@ type NVDMetadata struct {
 }
 
 type NVDmetadataCVSSv2 struct {
-	Vectors string
-	Score   float64
+	PublishedDateTime string
+	Vectors           string
+	Score             float64
 }
 
 func init() {
@@ -82,6 +83,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error {
 	// Get data feeds.
 	dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath)
 	if err != nil {
+		log.Info(err)
 		return err
 	}
 	a.dataFeedHashes = dataFeedHashes
@@ -160,8 +162,17 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin
 				}
 			}
 
+			//A custom HTTP client limiting the amound of Idle connections is helpful when retrieving the feeds from nvd.nist.gov due to the number of timeouts.
+			var netClient = &http.Client{}
+			tr := &http.Transport{
+				MaxIdleConns:        20,
+				MaxIdleConnsPerHost: 20,
+			}
+
+			netClient = &http.Client{Transport: tr}
+
 			// Download data feed.
-			r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName))
+			r, err := netClient.Get(fmt.Sprintf(dataFeedURL, dataFeedName))
 			if err != nil {
 				log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed")
 				return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload

ext/vulnmdsrc/nvd/xml.go

@@ -26,8 +26,9 @@ type nvd struct {
 }
 
 type nvdEntry struct {
-	Name string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"`
-	CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"`
+	Name              string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"`
+	CVSS              nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"`
+	PublishedDateTime string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 published-datetime"`
 }
 
 type nvdCVSS struct {
@@ -64,8 +65,9 @@ func init() {
 func (n nvdEntry) Metadata() *NVDMetadata {
 	metadata := &NVDMetadata{
 		CVSSv2: NVDmetadataCVSSv2{
-			Vectors: n.CVSS.BaseMetrics.String(),
-			Score:   n.CVSS.BaseMetrics.Score,
+			PublishedDateTime: n.PublishedDateTime,
+			Vectors:           n.CVSS.BaseMetrics.String(),
+			Score:             n.CVSS.BaseMetrics.Score,
 		},
 	}
 

updater.go

@@ -31,10 +31,11 @@ import (
 )
 
 const (
-	updaterLastFlagName        = "updater/last"
-	updaterLockName            = "updater"
-	updaterLockDuration        = updaterLockRefreshDuration + time.Minute*2
-	updaterLockRefreshDuration = time.Minute * 8
+	updaterLastFlagName              = "updater/last"
+	updaterLockName                  = "updater"
+	updaterLockDuration              = updaterLockRefreshDuration + time.Minute*2
+	updaterLockRefreshDuration       = time.Minute * 8
+	updaterSleepBetweenLoopsDuration = time.Minute
 )
 
 var (
@@ -124,7 +125,14 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 				if stop {
 					break
 				}
+
+				// Sleep for a short duration to prevent pinning the CPU on a
+				// consistent failure.
+				if stopped := sleepUpdater(time.Now().Add(updaterSleepBetweenLoopsDuration), st); stopped {
+					break
+				}
 				continue
+
 			} else {
 				lockOwner, lockExpiration, err := datastore.FindLock(updaterLockName)
 				if err != nil {
@@ -137,14 +145,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 			}
 		}
 
-		// Sleep, but remain stoppable until approximately the next update time.
-		now := time.Now().UTC()
-		waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
-		log.WithField("scheduled time", waitUntil).Debug("next update attempt scheduled")
-		if !waitUntil.Before(now) {
-			if !st.Sleep(waitUntil.Sub(time.Now())) {
-				break
-			}
+		if stopped := sleepUpdater(nextUpdate, st); stopped {
+			break
 		}
 	}
 
@@ -159,6 +161,19 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 	log.Info("updater service stopped")
 }
 
+// sleepUpdater sleeps the updater for an approximate duration, but remains
+// able to be cancelled by a stopper.
+func sleepUpdater(approxWakeup time.Time, st *stopper.Stopper) (stopped bool) {
+	waitUntil := approxWakeup.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second)
+	log.WithField("scheduled time", waitUntil).Debug("updater sleeping")
+	if !waitUntil.Before(time.Now().UTC()) {
+		if !st.Sleep(waitUntil.Sub(time.Now())) {
+			return true
+		}
+	}
+	return false
+}
+
 // update fetches all the vulnerabilities from the registered fetchers, upserts
 // them into the database and then sends notifications.
 func update(datastore database.Datastore, firstUpdate bool) {