From 4f8d6bee1ea59535f1bd4040304c24f2095f7e12 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 19 Jun 2017 11:43:18 -0400 Subject: [PATCH 1/6] Merge pull request #416 from tianon/debian-buster Add Debian Buster (10) and update "*stable" aliases --- database/namespace_mapping.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/database/namespace_mapping.go b/database/namespace_mapping.go index 47439d45..d7ff67c3 100644 --- a/database/namespace_mapping.go +++ b/database/namespace_mapping.go @@ -21,13 +21,15 @@ var DebianReleasesMapping = map[string]string{ "wheezy": "7", "jessie": "8", "stretch": "9", + "buster": "10", "sid": "unstable", // Class names - "oldstable": "7", - "stable": "8", - "testing": "9", - "unstable": "unstable", + "oldoldstable": "7", + "oldstable": "8", + "stable": "9", + "testing": "10", + "unstable": "unstable", } // UbuntuReleasesMapping translates Ubuntu code names to version numbers From 2453d67c368a76dce283275a7e14fda59114894b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 12 Jun 2017 11:13:59 -0400 Subject: [PATCH 2/6] Merge pull request #413 from transcedentalia/master Fixing the always database revision 0 for Ubuntu --- ext/featurens/aptsources/aptsources.go | 2 ++ ext/vulnsrc/ubuntu/ubuntu.go | 1 + 2 files changed, 3 insertions(+) diff --git a/ext/featurens/aptsources/aptsources.go b/ext/featurens/aptsources/aptsources.go index 3330ec69..c43818e7 100644 --- a/ext/featurens/aptsources/aptsources.go +++ b/ext/featurens/aptsources/aptsources.go @@ -68,6 +68,8 @@ func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { OS = "debian" break } + + line[2] = strings.Split(line[2], "/")[0] version, found = database.UbuntuReleasesMapping[line[2]] if found { OS = "ubuntu" diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index c328ae31..28803c76 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -344,6 +344,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability // Only consider the package if its status is needed, active, deferred, not-affected or // released. Ignore DNE (package does not exist), needs-triage, ignored, pending. if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" { + md["release"] = strings.Split(md["release"], "/")[0] if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored { continue } From b2519a044a7e5ed78e3edafe22a955fb1a189a4e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 2 Jun 2017 17:40:15 -0700 Subject: [PATCH 3/6] Merge pull request #407 from swestcott/kubernetes-config-fix Updated Kubernetes config with new db connection string config --- contrib/k8s/config.yaml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/contrib/k8s/config.yaml b/contrib/k8s/config.yaml index fd951e57..13c48e90 100644 --- a/contrib/k8s/config.yaml +++ b/contrib/k8s/config.yaml @@ -15,13 +15,16 @@ # The values specified here are the default values that Clair uses if no configuration file is specified or if the keys are not defined. clair: database: - # PostgreSQL Connection string - # http://www.postgresql.org/docs/9.4/static/libpq-connect.html - source: postgres://postgres:password@postgres:5432/postgres?sslmode=disable + # Database driver + type: pgsql + options: + # PostgreSQL Connection string + # https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING + source: postgres://postgres:password@postgres:5432/postgres?sslmode=disable - # Number of elements kept in the cache - # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database. - cacheSize: 16384 + # Number of elements kept in the cache + # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database. + cachesize: 16384 api: # API server port @@ -37,7 +40,7 @@ clair: # 32-bit URL-safe base64 key used to encrypt pagination tokens # If one is not provided, it will be generated. # Multiple clair instances in the same cluster need the same value. - paginationKey: + paginationkey: # Optional PKI configuration # If you want to easily generate client certificates and CAs, try the following projects: @@ -58,7 +61,7 @@ clair: attempts: 3 # Duration before a failed notification is retried - renotifyInterval: 2h + renotifyinterval: 2h http: # Optional endpoint that will receive notifications via POST requests @@ -72,3 +75,6 @@ clair: cafile: keyfile: certfile: + + # Optional HTTP Proxy: must be a valid URL (including the scheme). + proxy: From f8a1359a60353ae70f7cc78d9c6ab370d1e957d9 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 29 Jun 2017 15:59:10 -0400 Subject: [PATCH 4/6] Merge pull request #423 from jzelinskie/sleep-updater updater: sleep before continuing the lock loop --- updater.go | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/updater.go b/updater.go index 71ac147d..2e3aa216 100644 --- a/updater.go +++ b/updater.go @@ -31,10 +31,11 @@ import ( ) const ( - updaterLastFlagName = "updater/last" - updaterLockName = "updater" - updaterLockDuration = updaterLockRefreshDuration + time.Minute*2 - updaterLockRefreshDuration = time.Minute * 8 + updaterLastFlagName = "updater/last" + updaterLockName = "updater" + updaterLockDuration = updaterLockRefreshDuration + time.Minute*2 + updaterLockRefreshDuration = time.Minute * 8 + updaterSleepBetweenLoopsDuration = time.Minute ) var ( @@ -124,7 +125,14 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper if stop { break } + + // Sleep for a short duration to prevent pinning the CPU on a + // consistent failure. + if stopped := sleepUpdater(time.Now().Add(updaterSleepBetweenLoopsDuration), st); stopped { + break + } continue + } else { lockOwner, lockExpiration, err := datastore.FindLock(updaterLockName) if err != nil { @@ -137,14 +145,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper } } - // Sleep, but remain stoppable until approximately the next update time. - now := time.Now().UTC() - waitUntil := nextUpdate.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second) - log.WithField("scheduled time", waitUntil).Debug("next update attempt scheduled") - if !waitUntil.Before(now) { - if !st.Sleep(waitUntil.Sub(time.Now())) { - break - } + if stopped := sleepUpdater(nextUpdate, st); stopped { + break } } @@ -159,6 +161,19 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper log.Info("updater service stopped") } +// sleepUpdater sleeps the updater for an approximate duration, but remains +// able to be cancelled by a stopper. +func sleepUpdater(approxWakeup time.Time, st *stopper.Stopper) (stopped bool) { + waitUntil := approxWakeup.Add(time.Duration(rand.ExpFloat64()/0.5) * time.Second) + log.WithField("scheduled time", waitUntil).Debug("updater sleeping") + if !waitUntil.Before(time.Now().UTC()) { + if !st.Sleep(waitUntil.Sub(time.Now())) { + return true + } + } + return false +} + // update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. func update(datastore database.Datastore, firstUpdate bool) { From 70db22b9967ba4b40648759b64da12f4bbeb1e65 Mon Sep 17 00:00:00 2001 From: ErikThoreson Date: Tue, 5 Dec 2017 12:51:32 -0600 Subject: [PATCH 5/6] custom http client for nvd, add publication date --- ext/vulnmdsrc/nvd/nvd.go | 17 ++++++++++++++--- ext/vulnmdsrc/nvd/xml.go | 10 ++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 5be08b97..1f579654 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -57,8 +57,9 @@ type NVDMetadata struct { } type NVDmetadataCVSSv2 struct { - Vectors string - Score float64 + PublishedDateTime string + Vectors string + Score float64 } func init() { @@ -82,6 +83,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error { // Get data feeds. dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath) if err != nil { + log.Info(err) return err } a.dataFeedHashes = dataFeedHashes @@ -160,8 +162,17 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } } + //A custom HTTP client limiting the amound of Idle connections is helpful when retrieving the feeds from nvd.nist.gov due to the number of timeouts. + var netClient = &http.Client{} + tr := &http.Transport{ + MaxIdleConns: 20, + MaxIdleConnsPerHost: 20, + } + + netClient = &http.Client{Transport: tr} + // Download data feed. - r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName)) + r, err := netClient.Get(fmt.Sprintf(dataFeedURL, dataFeedName)) if err != nil { log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload diff --git a/ext/vulnmdsrc/nvd/xml.go b/ext/vulnmdsrc/nvd/xml.go index 9c6ffd98..b25b67ad 100644 --- a/ext/vulnmdsrc/nvd/xml.go +++ b/ext/vulnmdsrc/nvd/xml.go @@ -26,8 +26,9 @@ type nvd struct { } type nvdEntry struct { - Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` - CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` + Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` + CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` + PublishedDateTime string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 published-datetime"` } type nvdCVSS struct { @@ -64,8 +65,9 @@ func init() { func (n nvdEntry) Metadata() *NVDMetadata { metadata := &NVDMetadata{ CVSSv2: NVDmetadataCVSSv2{ - Vectors: n.CVSS.BaseMetrics.String(), - Score: n.CVSS.BaseMetrics.Score, + PublishedDateTime: n.PublishedDateTime, + Vectors: n.CVSS.BaseMetrics.String(), + Score: n.CVSS.BaseMetrics.Score, }, } From 5101f20d3ca8aa4abff35308b4b39504d2dc71e2 Mon Sep 17 00:00:00 2001 From: ErikThoreson Date: Wed, 6 Dec 2017 10:06:28 -0600 Subject: [PATCH 6/6] removing troubleshooting lines --- ext/vulnmdsrc/nvd/nvd.go | 1 - updater.go | 1 - 2 files changed, 2 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 1f579654..92775763 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -83,7 +83,6 @@ func (a *appender) BuildCache(datastore database.Datastore) error { // Get data feeds. dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath) if err != nil { - log.Info(err) return err } a.dataFeedHashes = dataFeedHashes diff --git a/updater.go b/updater.go index b579dded..792e068b 100644 --- a/updater.go +++ b/updater.go @@ -142,7 +142,6 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper break } continue - } else { lockOwner, lockExpiration, ok, err := findLock(datastore, updaterLockName) if !ok || err != nil {