*: add metadata support along with NVD CVSS

api/v1/models.go

@@ -63,6 +63,7 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil
 					Namespace:   dbVuln.Namespace.Name,
 					Description: dbVuln.Description,
 					Severity:    string(dbVuln.Severity),
+					Metadata:    dbVuln.Metadata,
 				}
 
 				if dbVuln.FixedBy != types.MaxVersion {
@@ -78,13 +79,14 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil
 }
 
 type Vulnerability struct {
-	Name        string    `json:"Name,omitempty"`
+	Name        string                 `json:"Name,omitempty"`
-	Namespace   string    `json:"Namespace,omitempty"`
+	Namespace   string                 `json:"Namespace,omitempty"`
-	Description string    `json:"Description,omitempty"`
+	Description string                 `json:"Description,omitempty"`
-	Link        string    `json:"Link,omitempty"`
+	Link        string                 `json:"Link,omitempty"`
-	Severity    string    `json:"Severity,omitempty"`
+	Severity    string                 `json:"Severity,omitempty"`
-	FixedBy     string    `json:"FixedBy,omitempty"`
+	Metadata    map[string]interface{} `json:"Metadata,omitempty"`
-	FixedIn     []Feature `json:"FixedIn,omitempty"`
+	FixedBy     string                 `json:"FixedBy,omitempty"`
+	FixedIn     []Feature              `json:"FixedIn,omitempty"`
 }
 
 func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
@@ -115,6 +117,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
 		Description: v.Description,
 		Link:        v.Link,
 		Severity:    severity,
+		Metadata:    v.Metadata,
 		FixedIn:     dbFeatures,
 	}, nil
 }
@@ -126,6 +129,7 @@ func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn b
 		Description: dbVuln.Description,
 		Link:        dbVuln.Link,
 		Severity:    string(dbVuln.Severity),
+		Metadata:    dbVuln.Metadata,
 	}
 
 	if withFixedIn {

cmd/clair/main.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/coreos/clair/updater/fetchers/debian"
 	_ "github.com/coreos/clair/updater/fetchers/rhel"
 	_ "github.com/coreos/clair/updater/fetchers/ubuntu"
+	_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"
 
 	_ "github.com/coreos/clair/worker/detectors/data/aci"
 	_ "github.com/coreos/clair/worker/detectors/data/docker"

database/models.go

@@ -15,6 +15,8 @@
 package database
 
 import (
+	"database/sql/driver"
+	"encoding/json"
 	"time"
 
 	"github.com/coreos/clair/utils/types"
@@ -65,6 +67,8 @@ type Vulnerability struct {
 	Link        string
 	Severity    types.Priority
 
+	Metadata MetadataMap
+
 	FixedIn                        []FeatureVersion
 	LayersIntroducingVulnerability []Layer
 
@@ -73,6 +77,21 @@ type Vulnerability struct {
 	FixedBy types.Version `json:",omitempty"`
 }
 
+type MetadataMap map[string]interface{}
+
+func (mm *MetadataMap) Scan(value interface{}) error {
+	val, ok := value.([]byte)
+	if !ok {
+		return nil
+	}
+	return json.Unmarshal(val, mm)
+}
+
+func (mm *MetadataMap) Value() (driver.Value, error) {
+	json, err := json.Marshal(*mm)
+	return string(json), err
+}
+
 type VulnerabilityNotification struct {
 	Model
 

database/pgsql/layer.go

@@ -163,7 +163,7 @@ func (pgSQL *pgSQL) loadAffectedBy(featureVersions []database.FeatureVersion) er
 		var vulnerability database.Vulnerability
 		err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
 			&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
-			&vulnerability.Namespace.Name, &vulnerability.FixedBy)
+			&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy)
 		if err != nil {
 			return handleError("s_featureversions_vulnerabilities.Scan()", err)
 		}

database/pgsql/migrations/20151222113213_Initial.sql

@@ -88,6 +88,7 @@ CREATE TABLE IF NOT EXISTS Vulnerability (
   description TEXT NULL,
   link VARCHAR(128) NULL,
   severity severity NOT NULL,
+  metadata TEXT NULL,
 
   UNIQUE (namespace_id, name));
 

database/pgsql/queries.go

@@ -104,8 +104,8 @@ func init() {
     ORDER BY ltree.ordering`
 
 	queries["s_featureversions_vulnerabilities"] = `
-    SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, vn.name,
+    SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata,
-      vfif.version
+      vn.name, vfif.version
     FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v,
          Namespace vn, Vulnerability_FixedIn_Feature vfif
     WHERE vafv.featureversion_id = ANY($1::integer[])
@@ -144,7 +144,7 @@ func init() {
 
 	// vulnerability.go
 	queries["f_vulnerability"] = `
-    SELECT v.id, n.id, v.description, v.link, v.severity, vfif.version, f.id, f.Name
+    SELECT v.id, n.id, v.description, v.link, v.severity, v.metadata, vfif.version, f.id, f.Name
     FROM Vulnerability v
       JOIN Namespace n ON v.namespace_id = n.id
       LEFT JOIN Vulnerability_FixedIn_Feature vfif ON v.id = vfif.vulnerability_id
@@ -152,12 +152,14 @@ func init() {
     WHERE n.Name = $1 AND v.Name = $2`
 
 	queries["i_vulnerability"] = `
-    INSERT INTO Vulnerability(namespace_id, name, description, link, severity)
+    INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata)
-    VALUES($1, $2, $3, $4, $5)
+    VALUES($1, $2, $3, $4, $5, $6)
     RETURNING id`
 
 	queries["u_vulnerability"] = `
-    UPDATE Vulnerability SET description = $2, link = $3, severity = $4 WHERE id = $1`
+    UPDATE Vulnerability
+    SET description = $2, link = $3, severity = $4, metadata = $5
+    WHERE id = $1`
 
 	queries["i_vulnerability_fixedin_feature"] = `
     INSERT INTO Vulnerability_FixedIn_Feature(vulnerability_id, feature_id, version)

database/pgsql/vulnerability.go

@@ -16,7 +16,9 @@ package pgsql
 
 import (
 	"database/sql"
+	"encoding/json"
 	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/coreos/clair/database"
@@ -50,8 +52,8 @@ func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vuln
 		var featureVersionFeatureName zero.String
 
 		err := rows.Scan(&vulnerability.ID, &vulnerability.Namespace.ID, &vulnerability.Description,
-			&vulnerability.Link, &vulnerability.Severity, &featureVersionVersion, &featureVersionID,
+			&vulnerability.Link, &vulnerability.Severity, &vulnerability.Metadata,
-			&featureVersionFeatureName)
+			&featureVersionVersion, &featureVersionID, &featureVersionFeatureName)
 		if err != nil {
 			return vulnerability, handleError("f_vulnerability.Scan()", err)
 		}
@@ -139,6 +141,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
 		if vulnerability.Description == existingVulnerability.Description &&
 			vulnerability.Link == existingVulnerability.Link &&
 			vulnerability.Severity == existingVulnerability.Severity &&
+			reflect.DeepEqual(castMetadata(vulnerability.Metadata), existingVulnerability.Metadata) &&
 			len(newFixedInFeatureVersions) == 0 &&
 			len(updatedFixedInFeatureVersions) == 0 {
 
@@ -191,7 +194,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
 	if existingVulnerability.ID == 0 {
 		// Insert new vulnerability.
 		err = tx.QueryRow(getQuery("i_vulnerability"), namespaceID, vulnerability.Name,
-			vulnerability.Description, vulnerability.Link, &vulnerability.Severity).Scan(&vulnerability.ID)
+			vulnerability.Description, vulnerability.Link, &vulnerability.Severity,
+			&vulnerability.Metadata).Scan(&vulnerability.ID)
 		if err != nil {
 			tx.Rollback()
 			return handleError("i_vulnerability", err)
@@ -202,7 +206,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
 			vulnerability.Link != existingVulnerability.Link ||
 			vulnerability.Severity != existingVulnerability.Severity {
 			_, err = tx.Exec(getQuery("u_vulnerability"), existingVulnerability.ID,
-				vulnerability.Description, vulnerability.Link, &vulnerability.Severity)
+				vulnerability.Description, vulnerability.Link, &vulnerability.Severity,
+				&vulnerability.Metadata)
 			if err != nil {
 				tx.Rollback()
 				return handleError("u_vulnerability", err)
@@ -244,6 +249,17 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
 	return nil
 }
 
+// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that
+// everything has the interface{} type.
+// It is required when comparing crafted MetadataMap against MetadataMap that we get from the
+// database.
+func castMetadata(m database.MetadataMap) database.MetadataMap {
+	c := make(database.MetadataMap)
+	j, _ := json.Marshal(m)
+	json.Unmarshal(j, &c)
+	return c
+}
+
 func diffFixedIn(vulnerability, existingVulnerability database.Vulnerability) (newFixedIn, updatedFixedIn []database.FeatureVersion) {
 	// Build FeatureVersion.Feature.Namespace.Name:FeatureVersion.Feature.Name (NaN) structures.
 	vulnerabilityFixedInNameMap, vulnerabilityFixedInNameSlice := createFeatureVersionNameMap(vulnerability.FixedIn)

database/pgsql/vulnerability_test.go

@@ -15,6 +15,7 @@
 package pgsql
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/coreos/clair/database"
@@ -195,6 +196,14 @@ func TestInsertVulnerability(t *testing.T) {
 	}
 
 	// Insert a simple vulnerability and find it.
+	v1meta := make(map[string]interface{})
+	v1meta["TestInsertVulnerabilityMetadata1"] = "TestInsertVulnerabilityMetadataValue1"
+	v1meta["TestInsertVulnerabilityMetadata2"] = struct {
+		Test string
+	}{
+		Test: "TestInsertVulnerabilityMetadataValue1",
+	}
+
 	v1 := database.Vulnerability{
 		Name:        "TestInsertVulnerability1",
 		Namespace:   n1,
@@ -202,6 +211,7 @@ func TestInsertVulnerability(t *testing.T) {
 		Severity:    types.Low,
 		Description: "TestInsertVulnerabilityDescription1",
 		Link:        "TestInsertVulnerabilityLink1",
+		Metadata:    v1meta,
 	}
 	err = datastore.InsertVulnerabilities([]database.Vulnerability{v1})
 	if assert.Nil(t, err) {
@@ -245,6 +255,8 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
 	assert.Equal(t, expected.Description, actual.Description)
 	assert.Equal(t, expected.Link, actual.Link)
 	assert.Equal(t, expected.Severity, actual.Severity)
+	assert.True(t, reflect.DeepEqual(castMetadata(expected.Metadata), actual.Metadata), "Got metadata %#v, expected %#v", actual.Metadata, castMetadata(expected.Metadata))
+
 	if assert.Len(t, actual.FixedIn, len(expected.FixedIn)) {
 		for _, actualFeatureVersion := range actual.FixedIn {
 			found := false

updater/fetchers/ubuntu/ubuntu.go

@@ -428,5 +428,7 @@ func ubuntuPriorityToSeverity(priority string) types.Priority {
 
 // Clean deletes any allocated resources.
 func (fetcher *UbuntuFetcher) Clean() {
-	os.RemoveAll(fetcher.repositoryLocalPath)
+	if fetcher.repositoryLocalPath != "" {
+		os.RemoveAll(fetcher.repositoryLocalPath)
+	}
 }

updater/metadata_fetchers.go

@@ -0,0 +1,64 @@
+// Copyright 2015 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package updater
+
+import (
+	"sync"
+
+	"github.com/coreos/clair/database"
+)
+
+var metadataFetchers = make(map[string]MetadataFetcher)
+
+type VulnerabilityWithLock struct {
+	*database.Vulnerability
+	Lock sync.Mutex
+}
+
+// MetadataFetcher
+type MetadataFetcher interface {
+	// Load runs right before the Updater calls AddMetadata for each vulnerabilities.
+	Load(database.Datastore) error
+
+	// AddMetadata adds metadata to the given database.Vulnerability.
+	// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
+	AddMetadata(*VulnerabilityWithLock) error
+
+	// Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities.
+	Unload()
+
+	// Clean deletes any allocated resources.
+	// It is invoked when Clair stops.
+	Clean()
+}
+
+// RegisterFetcher makes a Fetcher available by the provided name.
+// If Register is called twice with the same name or if driver is nil,
+// it panics.
+func RegisterMetadataFetcher(name string, f MetadataFetcher) {
+	if name == "" {
+		panic("updater: could not register a MetadataFetcher with an empty name")
+	}
+
+	if f == nil {
+		panic("updater: could not register a nil MetadataFetcher")
+	}
+
+	if _, dup := fetchers[name]; dup {
+		panic("updater: RegisterMetadataFetcher called twice for " + name)
+	}
+
+	metadataFetchers[name] = f
+}

updater/metadata_fetchers/nvd/nested_read_closer.go

@@ -0,0 +1,19 @@
+package nvd
+
+import "io"
+
+// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
+// io.ReadCloser.
+// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
+// simple manner.
+type NestedReadCloser struct {
+	io.Reader
+	NestedReadClosers []io.ReadCloser
+}
+
+// Close closes the gzip.Reader and the underlying io.ReadCloser.
+func (nrc *NestedReadCloser) Close() {
+	for _, nestedReadCloser := range nrc.NestedReadClosers {
+		nestedReadCloser.Close()
+	}
+}

updater/metadata_fetchers/nvd/nvd.go

@@ -0,0 +1,228 @@
+package nvd
+
+import (
+	"bufio"
+	"compress/gzip"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/updater"
+	cerrors "github.com/coreos/clair/utils/errors"
+	"github.com/coreos/pkg/capnslog"
+)
+
+const (
+	dataFeedURL     string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz"
+	dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta"
+
+	metadataKey string = "NVD"
+)
+
+var (
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers")
+)
+
+type NVDMetadataFetcher struct {
+	localPath      string
+	dataFeedHashes map[string]string
+	lock           sync.Mutex
+
+	metadata map[string]NVDMetadata
+}
+
+type NVDMetadata struct {
+	CVSSv2 NVDmetadataCVSSv2
+}
+
+type NVDmetadataCVSSv2 struct {
+	Vectors string
+	Score   float64
+}
+
+func init() {
+	updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{})
+}
+
+func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
+	fetcher.lock.Lock()
+	defer fetcher.lock.Unlock()
+
+	var err error
+	fetcher.metadata = make(map[string]NVDMetadata)
+
+	// Init if necessary.
+	if fetcher.localPath == "" {
+		// Create a temporary folder to store the NVD data and create hashes struct.
+		if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil {
+			return cerrors.ErrFilesystem
+		}
+
+		fetcher.dataFeedHashes = make(map[string]string)
+	}
+
+	// Get data feeds.
+	dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath)
+	if err != nil {
+		return err
+	}
+	fetcher.dataFeedHashes = dataFeedHashes
+
+	// Parse data feeds.
+	for dataFeedName, dataFeedReader := range dataFeedReaders {
+		var nvd nvd
+		if err = xml.NewDecoder(dataFeedReader).Decode(&nvd); err != nil {
+			log.Errorf("could not decode NVD data feed '%s': %s", dataFeedName, err)
+			return cerrors.ErrCouldNotParse
+		}
+
+		// For each entry of this data feed:
+		for _, nvdEntry := range nvd.Entries {
+			// Create metadata entry.
+			if metadata := nvdEntry.Metadata(); metadata != nil {
+				fetcher.metadata[nvdEntry.Name] = *metadata
+			}
+		}
+
+		dataFeedReader.Close()
+	}
+
+	return nil
+}
+
+func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error {
+	fetcher.lock.Lock()
+	defer fetcher.lock.Unlock()
+
+	if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok {
+		vulnerability.Lock.Lock()
+		defer vulnerability.Lock.Unlock()
+
+		// Create Metadata map if necessary.
+		if vulnerability.Metadata == nil {
+			vulnerability.Metadata = make(map[string]interface{})
+		}
+
+		vulnerability.Metadata[metadataKey] = nvdMetadata
+	}
+
+	return nil
+}
+
+func (fetcher *NVDMetadataFetcher) Unload() {
+	fetcher.lock.Lock()
+	defer fetcher.lock.Unlock()
+
+	fetcher.metadata = nil
+}
+
+func (fetcher *NVDMetadataFetcher) Clean() {
+	fetcher.lock.Lock()
+	defer fetcher.lock.Unlock()
+
+	if fetcher.localPath != "" {
+		os.RemoveAll(fetcher.localPath)
+	}
+}
+
+func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) {
+	var dataFeedNames []string
+	for y := 2002; y <= time.Now().Year(); y++ {
+		dataFeedNames = append(dataFeedNames, strconv.Itoa(y))
+	}
+
+	// Get hashes for these feeds.
+	for _, dataFeedName := range dataFeedNames {
+		hash, err := getHashFromMetaURL(fmt.Sprintf(dataFeedMetaURL, dataFeedName))
+		if err != nil {
+			log.Warningf("could get get NVD data feed hash '%s': %s", dataFeedName, err)
+
+			// It's not a big deal, no need interrupt, we're just going to download it again then.
+			continue
+		}
+
+		dataFeedHashes[dataFeedName] = hash
+	}
+
+	// Create io.Reader for every data feed.
+	dataFeedReaders := make(map[string]NestedReadCloser)
+	for _, dataFeedName := range dataFeedNames {
+		fileName := localPath + dataFeedName + ".xml"
+
+		if h, ok := dataFeedHashes[dataFeedName]; ok && h == dataFeedHashes[dataFeedName] {
+			// The hash is known, the disk should contains the feed. Try to read from it.
+			if localPath != "" {
+				if f, err := os.Open(fileName); err == nil {
+					dataFeedReaders[dataFeedName] = NestedReadCloser{
+						Reader:            f,
+						NestedReadClosers: []io.ReadCloser{f},
+					}
+					continue
+				}
+			}
+
+			// Download data feed.
+			r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName))
+			if err != nil {
+				log.Errorf("could not download NVD data feed file '%s': %s", dataFeedName, err)
+				return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload
+			}
+
+			// Un-gzip it.
+			gr, err := gzip.NewReader(r.Body)
+			if err != nil {
+				log.Errorf("could not read NVD data feed file '%s': %s", dataFeedName, err)
+				return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload
+			}
+
+			// Store it to a file at the same time if possible.
+			if f, err := os.Create(fileName); err == nil {
+				nrc := NestedReadCloser{
+					Reader:            io.TeeReader(gr, f),
+					NestedReadClosers: []io.ReadCloser{r.Body, gr, f},
+				}
+				dataFeedReaders[dataFeedName] = nrc
+			} else {
+				nrc := NestedReadCloser{
+					Reader:            gr,
+					NestedReadClosers: []io.ReadCloser{gr, r.Body},
+				}
+				dataFeedReaders[dataFeedName] = nrc
+
+				log.Warningf("could not store NVD data feed to filesystem: %s", err)
+			}
+		}
+	}
+
+	return dataFeedReaders, dataFeedHashes, nil
+}
+
+func getHashFromMetaURL(metaURL string) (string, error) {
+	r, err := http.Get(metaURL)
+	if err != nil {
+		return "", err
+	}
+	defer r.Body.Close()
+
+	scanner := bufio.NewScanner(r.Body)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "sha256:") {
+			return strings.TrimPrefix(line, "sha256:"), nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	return "", errors.New("invalid .meta file format")
+}

updater/metadata_fetchers/nvd/xml.go

@@ -0,0 +1,82 @@
+package nvd
+
+import (
+	"fmt"
+	"strings"
+)
+
+type nvd struct {
+	Entries []nvdEntry `xml:"entry"`
+}
+
+type nvdEntry struct {
+	Name string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"`
+	CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"`
+}
+
+type nvdCVSS struct {
+	BaseMetrics nvdCVSSBaseMetrics `xml:"http://scap.nist.gov/schema/cvss-v2/0.2 base_metrics"`
+}
+
+type nvdCVSSBaseMetrics struct {
+	Score            float64 `xml:"score"`
+	AccessVector     string  `xml:"access-vector"`
+	AccessComplexity string  `xml:"access-complexity"`
+	Authentication   string  `xml:"authentication"`
+	ConfImpact       string  `xml:"confidentiality-impact"`
+	IntegImpact      string  `xml:"integrity-impact"`
+	AvailImpact      string  `xml:"avaibility-impact"`
+}
+
+var vectorValuesToLetters map[string]string
+
+func init() {
+	vectorValuesToLetters = make(map[string]string)
+	vectorValuesToLetters["NETWORK"] = "N"
+	vectorValuesToLetters["ADJACENT_NETWORK"] = "A"
+	vectorValuesToLetters["LOCAL"] = "L"
+	vectorValuesToLetters["HIGH"] = "H"
+	vectorValuesToLetters["MEDIUM"] = "M"
+	vectorValuesToLetters["LOW"] = "L"
+	vectorValuesToLetters["NONE"] = "N"
+	vectorValuesToLetters["SINGLE_INSTANCE"] = "S"
+	vectorValuesToLetters["MULTIPLE_INSTANCES"] = "M"
+	vectorValuesToLetters["PARTIAL"] = "P"
+	vectorValuesToLetters["COMPLETE"] = "C"
+}
+
+func (n nvdEntry) Metadata() *NVDMetadata {
+	metadata := &NVDMetadata{
+		CVSSv2: NVDmetadataCVSSv2{
+			Vectors: n.CVSS.BaseMetrics.String(),
+			Score:   n.CVSS.BaseMetrics.Score,
+		},
+	}
+
+	if metadata.CVSSv2.Vectors == "" {
+		return nil
+	}
+	return metadata
+}
+
+func (n nvdCVSSBaseMetrics) String() string {
+	var str string
+	addVec(&str, "AV", n.AccessVector)
+	addVec(&str, "AC", n.AccessComplexity)
+	addVec(&str, "Au", n.Authentication)
+	addVec(&str, "C", n.ConfImpact)
+	addVec(&str, "I", n.IntegImpact)
+	addVec(&str, "A", n.AvailImpact)
+	str = strings.TrimSuffix(str, "/")
+	return str
+}
+
+func addVec(str *string, vec, val string) {
+	if val != "" {
+		if let, ok := vectorValuesToLetters[val]; ok {
+			*str = fmt.Sprintf("%s%s:%s/", *str, vec, let)
+		} else {
+			log.Warningf("unknown value '%v' for CVSSv2 vector '%s'", val, vec)
+		}
+	}
+}

updater/updater.go

@@ -19,6 +19,7 @@ package updater
 import (
 	"math/rand"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/coreos/clair/config"
@@ -144,6 +145,9 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
 	}
 
 	// Clean resources.
+	for _, metadataFetcher := range metadataFetchers {
+		metadataFetcher.Clean()
+	}
 	for _, fetcher := range fetchers {
 		fetcher.Clean()
 	}
@@ -161,10 +165,8 @@ func Update(datastore database.Datastore) {
 	// Fetch updates.
 	status, vulnerabilities, flags, notes := fetch(datastore)
 
-	// TODO(Quentin-M): Complete informations using NVD
-
 	// Insert vulnerabilities.
-	log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
+	log.Tracef("inserting %d vulnerabilities for update", len(vulnerabilities))
 	err := datastore.InsertVulnerabilities(vulnerabilities)
 	if err != nil {
 		promUpdaterErrorsTotal.Inc()
@@ -204,6 +206,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
 	flags := make(map[string]string)
 
 	// Fetch updates in parallel.
+	log.Info("fetching vulnerability updates")
 	var responseC = make(chan *FetcherResponse, 0)
 	for n, f := range fetchers {
 		go func(name string, fetcher Fetcher) {
@@ -233,7 +236,52 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
 	}
 
 	close(responseC)
-	return status, vulnerabilities, flags, notes
+	return status, addMetadata(datastore, vulnerabilities), flags, notes
+}
+
+// Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel.
+func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability {
+	if len(metadataFetchers) == 0 {
+		return vulnerabilities
+	}
+
+	log.Info("adding metadata to vulnerabilities")
+
+	// Wrap vulnerabilities in VulnerabilityWithLock.
+	// It ensures that only one metadata fetcher at a time can modify the Metadata map.
+	vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities))
+	for i := 0; i < len(vulnerabilities); i++ {
+		vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{
+			Vulnerability: &vulnerabilities[i],
+		})
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(metadataFetchers))
+
+	for n, f := range metadataFetchers {
+		go func(name string, metadataFetcher MetadataFetcher) {
+			defer wg.Done()
+
+			// Load the metadata fetcher.
+			if err := metadataFetcher.Load(datastore); err != nil {
+				promUpdaterErrorsTotal.Inc()
+				log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err)
+				return
+			}
+
+			// Add metadata to each vulnerability.
+			for _, vulnerability := range vulnerabilitiesWithLocks {
+				metadataFetcher.AddMetadata(vulnerability)
+			}
+
+			metadataFetcher.Unload()
+		}(n, f)
+	}
+
+	wg.Wait()
+
+	return vulnerabilities
 }
 
 func getLastUpdate(datastore database.Datastore) time.Time {

utils/string.go

@@ -14,22 +14,10 @@
 
 package utils
 
-import (
+import "regexp"
-	"crypto/sha1"
-	"encoding/hex"
-	"regexp"
-)
 
 var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`)
 
-// Hash returns an unique hash of the given string.
-func Hash(str string) string {
-	h := sha1.New()
-	h.Write([]byte(str))
-	bs := h.Sum(nil)
-	return hex.EncodeToString(bs)
-}
-
 // CleanURL removes all parameters from an URL.
 func CleanURL(str string) string {
 	return urlParametersRegexp.ReplaceAllString(str, "")

utils/types/cvss.go

@@ -1,225 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-//
-// import "fmt"
-//
-// // CVSSv2 represents the Common Vulnerability Scoring System (CVSS), that assesses the severity of
-// // vulnerabilities.
-// // It describes the CVSS score, but also a vector describing the components from which the score
-// // was calculated. This provides users of the score confidence in its correctness and provides
-// // insight into the nature of the vulnerability.
-// //
-// // Reference: https://nvd.nist.gov/CVSS/Vector-v2.aspx
-// type CVSSv2 struct {
-// 	// Base Vectors
-// 	AccessVector     CVSSValue
-// 	AccessComplexity CVSSValue
-// 	Authentication   CVSSValue
-// 	ConfImpact       CVSSValue
-// 	IntegImpact      CVSSValue
-// 	AvailImpact      CVSSValue
-// 	// Temporal Vectors
-// 	Exploitability   CVSSValue
-// 	RemediationLevel CVSSValue
-// 	ReportConfidence CVSSValue
-// 	// Environmental Vectors
-// 	CollateralDamagePotential        CVSSValue
-// 	TargetDistribution               CVSSValue
-// 	SystemConfidentialityRequirement CVSSValue
-// 	SystemIntegrityRequirement       CVSSValue
-// 	SystemAvailabilityRequirement    CVSSValue
-// }
-//
-// func NewCVSSv2(value string) (*CVSSv2, error) {
-//
-// }
-//
-// // CVSSValue is the comprehensible value for a CVSS metric.
-// type CVSSValue string
-//
-// // Metric acronym + Value abbreviation -> Comprehensible metric value.
-// var toValue map[string]func(string) (CVSSValue, error)
-//
-// func init() {
-// 	parsers = make(map[string]func(string) (CVSSValue, error), 14)
-// 	toValue["AV"] = av
-// 	toValue["AC"] = ac
-// 	toValue["Au"] = au
-// 	toValue["C"] = cAndIAndA
-// 	toValue["I"] = cAndIAndA
-// 	toValue["A"] = cAndIAndA
-// 	toValue["E"] = e
-// 	toValue["RL"] = rl
-// 	toValue["RC"] = rc
-// 	toValue["CDP"] = cdp
-// 	toValue["TD"] = td
-// 	toValue["CR"] = crAndIrAndAr
-// 	toValue["IR"] = crAndIrAndAr
-// 	toValue["AR"] = crAndIrAndAr
-// }
-//
-// func av(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "L":
-// 		return CVSSValue("Local access"), nil
-// 	case "A":
-// 		return CVSSValue("Adjacent Network"), nil
-// 	case "N":
-// 		return CVSSValue("Network"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for AV", v)
-// 	}
-// }
-//
-// func ac(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "H":
-// 		return CVSSValue("High"), nil
-// 	case "M":
-// 		return CVSSValue("Medium"), nil
-// 	case "L":
-// 		return CVSSValue("Low"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for AC", v)
-// 	}
-// }
-//
-// func au(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "N":
-// 		return CVSSValue("None required"), nil
-// 	case "S":
-// 		return CVSSValue("Requires single instance"), nil
-// 	case "M":
-// 		return CVSSValue("Requires multiple instances"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for Au", v)
-// 	}
-// }
-//
-// func cAndIAndA(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "N":
-// 		return CVSSValue("None"), nil
-// 	case "P":
-// 		return CVSSValue("Partial"), nil
-// 	case "C":
-// 		return CVSSValue("Complete"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for C/I/A", v)
-// 	}
-// }
-//
-// func e(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "U":
-// 		return CVSSValue("Unproven"), nil
-// 	case "POC":
-// 		return CVSSValue("Proof-of-concept"), nil
-// 	case "F":
-// 		return CVSSValue("Functional"), nil
-// 	case "H":
-// 		return CVSSValue("High"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for E", v)
-// 	}
-// }
-//
-// func rl(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "OF":
-// 		return CVSSValue("Official-fix"), nil
-// 	case "T":
-// 		return CVSSValue("Temporary-fix"), nil
-// 	case "W":
-// 		return CVSSValue("Workaround"), nil
-// 	case "U":
-// 		return CVSSValue("Unavailable"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for RL", v)
-// 	}
-// }
-//
-// func rc(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "UC":
-// 		return CVSSValue("Unconfirmed"), nil
-// 	case "UR":
-// 		return CVSSValue("Uncorroborated"), nil
-// 	case "C":
-// 		return CVSSValue("Confirmed"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for RC", v)
-// 	}
-// }
-//
-// func cdp(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "N":
-// 		return CVSSValue("None"), nil
-// 	case "L":
-// 		return CVSSValue("Low"), nil
-// 	case "LM":
-// 		return CVSSValue("Low-Medium"), nil
-// 	case "MH":
-// 		return CVSSValue("Medium-High"), nil
-// 	case "H":
-// 		return CVSSValue("High"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for CDP", v)
-// 	}
-// }
-//
-// func td(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "N":
-// 		return CVSSValue("None (0%)"), nil
-// 	case "L":
-// 		return CVSSValue("Low (1-25%)"), nil
-// 	case "M":
-// 		return CVSSValue("Medium (26-75%)"), nil
-// 	case "H":
-// 		return CVSSValue("High (76-100%)"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for TD", v)
-// 	}
-// }
-//
-// func crAndIrAndAr(v string) (CVSSValue, error) {
-// 	switch v {
-// 	case "L":
-// 		return CVSSValue("Low"), nil
-// 	case "M":
-// 		return CVSSValue("Medium"), nil
-// 	case "H":
-// 		return CVSSValue("High"), nil
-// 	case "ND":
-// 		return CVSSValue("Not Defined"), nil
-// 	default:
-// 		return "", fmt.Errorf("%v is not a valid value for CR/IR/AR", v)
-// 	}
-// }

utils/utils_test.go

@@ -56,9 +56,6 @@ func TestExec(t *testing.T) {
 
 // TestString tests the string.go file
 func TestString(t *testing.T) {
-	assert.Equal(t, Hash("abc123"), Hash("abc123"))
-	assert.NotEqual(t, Hash("abc123."), Hash("abc123"))
-
 	assert.False(t, Contains("", []string{}))
 	assert.True(t, Contains("a", []string{"a", "b"}))
 	assert.False(t, Contains("c", []string{"a", "b"}))