Merge pull request #645 from Katee/include-cvssv3

Switch to NVD JSON feed and include CVSSv3

ext/vulnmdsrc/nvd/json.go

@@ -0,0 +1,163 @@
+// Copyright 2018 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+	"fmt"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type nvd struct {
+	Entries []nvdEntry `json:"CVE_Items"`
+}
+
+type nvdEntry struct {
+	CVE               nvdCVE    `json:"cve"`
+	Impact            nvdImpact `json:"impact"`
+	PublishedDateTime string    `json:"publishedDate"`
+}
+
+type nvdCVE struct {
+	Metadata nvdCVEMetadata `json:"CVE_data_meta"`
+}
+
+type nvdCVEMetadata struct {
+	CVEID string `json:"ID"`
+}
+
+type nvdImpact struct {
+	BaseMetricV2 nvdBaseMetricV2 `json:"baseMetricV2"`
+	BaseMetricV3 nvdBaseMetricV3 `json:"baseMetricV3"`
+}
+
+type nvdBaseMetricV2 struct {
+	CVSSv2 nvdCVSSv2 `json:"cvssV2"`
+}
+
+type nvdCVSSv2 struct {
+	Score            float64 `json:"baseScore"`
+	AccessVector     string  `json:"accessVector"`
+	AccessComplexity string  `json:"accessComplexity"`
+	Authentication   string  `json:"authentication"`
+	ConfImpact       string  `json:"confidentialityImpact"`
+	IntegImpact      string  `json:"integrityImpact"`
+	AvailImpact      string  `json:"availabilityImpact"`
+}
+
+type nvdBaseMetricV3 struct {
+	CVSSv3              nvdCVSSv3 `json:"cvssV3"`
+	ExploitabilityScore float64   `json:"exploitabilityScore"`
+	ImpactScore         float64   `json:"impactScore"`
+}
+
+type nvdCVSSv3 struct {
+	Score              float64 `json:"baseScore"`
+	AttackVector       string  `json:"attackVector"`
+	AttackComplexity   string  `json:"attackComplexity"`
+	PrivilegesRequired string  `json:"privilegesRequired"`
+	UserInteraction    string  `json:"userInteraction"`
+	Scope              string  `json:"scope"`
+	ConfImpact         string  `json:"confidentialityImpact"`
+	IntegImpact        string  `json:"integrityImpact"`
+	AvailImpact        string  `json:"availabilityImpact"`
+}
+
+var vectorValuesToLetters = map[string]string{
+	"NETWORK":          "N",
+	"ADJACENT_NETWORK": "A",
+	"LOCAL":            "L",
+	"HIGH":             "H",
+	"MEDIUM":           "M",
+	"LOW":              "L",
+	"NONE":             "N",
+	"SINGLE":           "S",
+	"MULTIPLE":         "M",
+	"PARTIAL":          "P",
+	"COMPLETE":         "C",
+
+	// CVSSv3 only
+	"PHYSICAL":  "P",
+	"REQUIRED":  "R",
+	"CHANGED":   "C",
+	"UNCHANGED": "U",
+}
+
+func (n nvdEntry) Metadata() *NVDMetadata {
+	metadata := &NVDMetadata{
+		CVSSv2: NVDmetadataCVSSv2{
+			PublishedDateTime: n.PublishedDateTime,
+			Vectors:           n.Impact.BaseMetricV2.CVSSv2.String(),
+			Score:             n.Impact.BaseMetricV2.CVSSv2.Score,
+		},
+		CVSSv3: NVDmetadataCVSSv3{
+			Vectors:             n.Impact.BaseMetricV3.CVSSv3.String(),
+			Score:               n.Impact.BaseMetricV3.CVSSv3.Score,
+			ExploitabilityScore: n.Impact.BaseMetricV3.ExploitabilityScore,
+			ImpactScore:         n.Impact.BaseMetricV3.ImpactScore,
+		},
+	}
+
+	if metadata.CVSSv2.Vectors == "" {
+		return nil
+	}
+
+	return metadata
+}
+
+func (n nvdEntry) Name() string {
+	return n.CVE.Metadata.CVEID
+}
+
+func (n nvdCVSSv2) String() string {
+	var str string
+	addVec(&str, "AV", n.AccessVector)
+	addVec(&str, "AC", n.AccessComplexity)
+	addVec(&str, "Au", n.Authentication)
+	addVec(&str, "C", n.ConfImpact)
+	addVec(&str, "I", n.IntegImpact)
+	addVec(&str, "A", n.AvailImpact)
+	str = strings.TrimSuffix(str, "/")
+	return str
+}
+
+func (n nvdCVSSv3) String() string {
+	var str string
+	addVec(&str, "AV", n.AttackVector)
+	addVec(&str, "AC", n.AttackComplexity)
+	addVec(&str, "PR", n.PrivilegesRequired)
+	addVec(&str, "UI", n.UserInteraction)
+	addVec(&str, "S", n.Scope)
+	addVec(&str, "C", n.ConfImpact)
+	addVec(&str, "I", n.IntegImpact)
+	addVec(&str, "A", n.AvailImpact)
+	str = strings.TrimSuffix(str, "/")
+
+	if len(str) > 0 {
+		return fmt.Sprintf("CVSS:3.0/%s", str)
+	}
+	return str
+}
+
+func addVec(str *string, vec, val string) {
+	if val != "" {
+		if let, ok := vectorValuesToLetters[val]; ok {
+			*str = fmt.Sprintf("%s%s:%s/", *str, vec, let)
+		} else {
+			log.WithFields(log.Fields{"value": val, "vector": vec}).Warning("unknown value for CVSS vector")
+		}
+	}
+}

ext/vulnmdsrc/nvd/nvd.go

@@ -19,7 +19,7 @@ package nvd
 import (
 	"bufio"
 	"compress/gzip"
-	"encoding/xml"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -39,8 +39,8 @@ import (
 )
 
 const (
-	dataFeedURL     string = "https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-%s.xml.gz"
+	dataFeedURL     string = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-%s.json.gz"
-	dataFeedMetaURL string = "https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-%s.meta"
+	dataFeedMetaURL string = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-%s.meta"
 
 	appenderName string = "NVD"
 
@@ -55,6 +55,7 @@ type appender struct {
 
 type NVDMetadata struct {
 	CVSSv2 NVDmetadataCVSSv2
+	CVSSv3 NVDmetadataCVSSv3
 }
 
 type NVDmetadataCVSSv2 struct {
@@ -63,6 +64,13 @@ type NVDmetadataCVSSv2 struct {
 	Score             float64
 }
 
+type NVDmetadataCVSSv3 struct {
+	Vectors             string
+	Score               float64
+	ExploitabilityScore float64
+	ImpactScore         float64
+}
+
 func init() {
 	vulnmdsrc.RegisterAppender(appenderName, &appender{})
 }
@@ -95,20 +103,11 @@ func (a *appender) BuildCache(datastore database.Datastore) error {
 			log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not open NVD data file")
 			return commonerr.ErrCouldNotParse
 		}
-		var nvd nvd
-		r := bufio.NewReader(f)
-		if err = xml.NewDecoder(r).Decode(&nvd); err != nil {
-			f.Close()
-			log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not decode NVD data feed")
-			return commonerr.ErrCouldNotParse
-		}
 
-		// For each entry of this data feed:
+		r := bufio.NewReader(f)
-		for _, nvdEntry := range nvd.Entries {
+		if err := a.parseDataFeed(r); err != nil {
-			// Create metadata entry.
+			log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not parse NVD data file")
-			if metadata := nvdEntry.Metadata(); metadata != nil {
+			return err
-				a.metadata[nvdEntry.Name] = *metadata
-			}
 		}
 		f.Close()
 	}
@@ -116,6 +115,23 @@ func (a *appender) BuildCache(datastore database.Datastore) error {
 	return nil
 }
 
+func (a *appender) parseDataFeed(r io.Reader) error {
+	var nvd nvd
+
+	if err := json.NewDecoder(r).Decode(&nvd); err != nil {
+		return commonerr.ErrCouldNotParse
+	}
+
+	for _, nvdEntry := range nvd.Entries {
+		// Create metadata entry.
+		if metadata := nvdEntry.Metadata(); metadata != nil {
+			a.metadata[nvdEntry.Name()] = *metadata
+		}
+	}
+
+	return nil
+}
+
 func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error {
 	if nvdMetadata, ok := a.metadata[vulnName]; ok {
 		appendFunc(appenderName, nvdMetadata, SeverityFromCVSS(nvdMetadata.CVSSv2.Score))
@@ -154,7 +170,8 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin
 	// Create map containing the name and filename for every data feed.
 	dataFeedReaders := make(map[string]string)
 	for _, dataFeedName := range dataFeedNames {
-		fileName := filepath.Join(localPath, fmt.Sprintf("%s.xml", dataFeedName))
+		fileName := filepath.Join(localPath, fmt.Sprintf("%s.json", dataFeedName))
+
 		if h, ok := dataFeedHashes[dataFeedName]; ok && h == dataFeedHashes[dataFeedName] {
 			// The hash is known, the disk should contains the feed. Try to read from it.
 			if localPath != "" {
@@ -177,7 +194,6 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin
 }
 
 func downloadFeed(dataFeedName, fileName string) error {
-
 	// Download data feed.
 	r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName))
 	if err != nil {

ext/vulnmdsrc/nvd/nvd_test.go

@@ -0,0 +1,99 @@
+// Copyright 2018 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNVDParser(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	path := filepath.Join(filepath.Dir(filename))
+
+	dataFilePath := filepath.Join(path, "/testdata/nvd_test.json")
+	testData, err := os.Open(dataFilePath)
+	if err != nil {
+		t.Fatalf("Error opening %q: %v", dataFilePath, err)
+	}
+	defer testData.Close()
+
+	a := &appender{}
+	a.metadata = make(map[string]NVDMetadata)
+
+	err = a.parseDataFeed(testData)
+	if err != nil {
+		t.Fatalf("Error parsing %q: %v", dataFilePath, err)
+	}
+
+	var gotMetadata, wantMetadata NVDMetadata
+
+	// Items without CVSSv2 aren't returned.
+	assert.Len(t, a.metadata, 2)
+	gotMetadata, ok := a.metadata["CVE-2002-0001"]
+	assert.False(t, ok)
+
+	// Item with only CVSSv2.
+	gotMetadata, ok = a.metadata["CVE-2012-0001"]
+	assert.True(t, ok)
+	wantMetadata = NVDMetadata{
+		CVSSv2: NVDmetadataCVSSv2{
+			Vectors: "AV:N/AC:L/Au:S/C:P/I:N/A:N",
+			Score:   4.0,
+		},
+	}
+	assert.Equal(t, wantMetadata, gotMetadata)
+
+	// Item with both CVSSv2 and CVSSv3 has CVSSv2 information returned.
+	gotMetadata, ok = a.metadata["CVE-2018-0001"]
+	assert.True(t, ok)
+	wantMetadata = NVDMetadata{
+		CVSSv2: NVDmetadataCVSSv2{
+			Vectors: "AV:N/AC:L/Au:N/C:P/I:P/A:P",
+			Score:   7.5,
+		},
+		CVSSv3: NVDmetadataCVSSv3{
+			Vectors:             "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H",
+			Score:               9.8,
+			ExploitabilityScore: 3.9,
+			ImpactScore:         5.9,
+		},
+	}
+	assert.Equal(t, wantMetadata, gotMetadata)
+}
+
+func TestNVDParserErrors(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	path := filepath.Join(filepath.Dir(filename))
+
+	dataFilePath := filepath.Join(path, "/testdata/nvd_test_incorrect_format.json")
+	testData, err := os.Open(dataFilePath)
+	if err != nil {
+		t.Fatalf("Error opening %q: %v", dataFilePath, err)
+	}
+	defer testData.Close()
+
+	a := &appender{}
+	a.metadata = make(map[string]NVDMetadata)
+
+	err = a.parseDataFeed(testData)
+	if err == nil {
+		t.Fatalf("Expected error parsing NVD data file: %q", dataFilePath)
+	}
+}

ext/vulnmdsrc/nvd/testdata/nvd_test.json

@@ -0,0 +1,92 @@
+{
+  "CVE_Items" : [ {
+    "_comment": "A CVE without CVSSv2 or CVSSv3",
+    "cve" : {
+      "data_type" : "CVE",
+      "data_format" : "MITRE",
+      "data_version" : "4.0",
+      "CVE_data_meta" : {
+        "ID" : "CVE-2002-0001"
+      }
+    },
+    "impact" : { },
+    "publishedDate" : "2018-01-10T22:29Z"
+  }, {
+    "_comment": "A CVE with only CVSSv2",
+    "cve" : {
+      "CVE_data_meta" : {
+        "ID" : "CVE-2012-0001"
+      }
+    },
+    "impact" : {
+      "baseMetricV3" : { },
+      "baseMetricV2" : {
+        "cvssV2" : {
+          "version" : "2.0",
+          "vectorString" : "(AV:N/AC:L/Au:S/C:P/I:N/A:N)",
+          "accessVector" : "NETWORK",
+          "accessComplexity" : "LOW",
+          "authentication" : "SINGLE",
+          "confidentialityImpact" : "PARTIAL",
+          "integrityImpact" : "NONE",
+          "availabilityImpact" : "NONE",
+          "baseScore" : 4.0
+        },
+        "severity" : "MEDIUM",
+        "exploitabilityScore" : 8.0,
+        "impactScore" : 2.9,
+        "obtainAllPrivilege" : false,
+        "obtainUserPrivilege" : false,
+        "obtainOtherPrivilege" : false,
+        "userInteractionRequired" : false
+      }
+    }
+  }, {
+    "_comment": "A CVE with standard CVSSv2 and CVSSv3",
+    "cve" : {
+      "CVE_data_meta" : {
+        "ID" : "CVE-2018-0001"
+      }
+    },
+    "impact" : {
+      "baseMetricV3" : {
+        "cvssV3" : {
+          "version" : "3.0",
+          "vectorString" : "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H",
+          "attackVector" : "NETWORK",
+          "attackComplexity" : "LOW",
+          "privilegesRequired" : "NONE",
+          "userInteraction" : "NONE",
+          "scope" : "UNCHANGED",
+          "confidentialityImpact" : "HIGH",
+          "integrityImpact" : "HIGH",
+          "availabilityImpact" : "HIGH",
+          "baseScore" : 9.8,
+          "baseSeverity" : "CRITICAL"
+        },
+        "exploitabilityScore" : 3.9,
+        "impactScore" : 5.9
+      },
+      "baseMetricV2" : {
+        "cvssV2" : {
+          "version" : "2.0",
+          "vectorString" : "(AV:N/AC:L/Au:N/C:P/I:P/A:P)",
+          "accessVector" : "NETWORK",
+          "accessComplexity" : "LOW",
+          "authentication" : "NONE",
+          "confidentialityImpact" : "PARTIAL",
+          "integrityImpact" : "PARTIAL",
+          "availabilityImpact" : "PARTIAL",
+          "baseScore" : 7.5
+        },
+        "severity" : "HIGH",
+        "exploitabilityScore" : 10.0,
+        "impactScore" : 6.4,
+        "obtainAllPrivilege" : false,
+        "obtainUserPrivilege" : false,
+        "obtainOtherPrivilege" : false,
+        "userInteractionRequired" : false
+      }
+    }
+  } ]
+}

ext/vulnmdsrc/nvd/testdata/nvd_test_incorrect_format.json

@@ -0,0 +1 @@
+Not a JSON file.

ext/vulnmdsrc/nvd/xml.go

@@ -1,100 +0,0 @@
-// Copyright 2017 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package nvd
-
-import (
-	"fmt"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-)
-
-type nvd struct {
-	Entries []nvdEntry `xml:"entry"`
-}
-
-type nvdEntry struct {
-	Name              string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"`
-	CVSS              nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"`
-	PublishedDateTime string  `xml:"http://scap.nist.gov/schema/vulnerability/0.4 published-datetime"`
-}
-
-type nvdCVSS struct {
-	BaseMetrics nvdCVSSBaseMetrics `xml:"http://scap.nist.gov/schema/cvss-v2/0.2 base_metrics"`
-}
-
-type nvdCVSSBaseMetrics struct {
-	Score            float64 `xml:"score"`
-	AccessVector     string  `xml:"access-vector"`
-	AccessComplexity string  `xml:"access-complexity"`
-	Authentication   string  `xml:"authentication"`
-	ConfImpact       string  `xml:"confidentiality-impact"`
-	IntegImpact      string  `xml:"integrity-impact"`
-	AvailImpact      string  `xml:"availability-impact"`
-}
-
-var vectorValuesToLetters map[string]string
-
-func init() {
-	vectorValuesToLetters = make(map[string]string)
-	vectorValuesToLetters["NETWORK"] = "N"
-	vectorValuesToLetters["ADJACENT_NETWORK"] = "A"
-	vectorValuesToLetters["LOCAL"] = "L"
-	vectorValuesToLetters["HIGH"] = "H"
-	vectorValuesToLetters["MEDIUM"] = "M"
-	vectorValuesToLetters["LOW"] = "L"
-	vectorValuesToLetters["NONE"] = "N"
-	vectorValuesToLetters["SINGLE_INSTANCE"] = "S"
-	vectorValuesToLetters["MULTIPLE_INSTANCES"] = "M"
-	vectorValuesToLetters["PARTIAL"] = "P"
-	vectorValuesToLetters["COMPLETE"] = "C"
-}
-
-func (n nvdEntry) Metadata() *NVDMetadata {
-	metadata := &NVDMetadata{
-		CVSSv2: NVDmetadataCVSSv2{
-			PublishedDateTime: n.PublishedDateTime,
-			Vectors:           n.CVSS.BaseMetrics.String(),
-			Score:             n.CVSS.BaseMetrics.Score,
-		},
-	}
-
-	if metadata.CVSSv2.Vectors == "" {
-		return nil
-	}
-	return metadata
-}
-
-func (n nvdCVSSBaseMetrics) String() string {
-	var str string
-	addVec(&str, "AV", n.AccessVector)
-	addVec(&str, "AC", n.AccessComplexity)
-	addVec(&str, "Au", n.Authentication)
-	addVec(&str, "C", n.ConfImpact)
-	addVec(&str, "I", n.IntegImpact)
-	addVec(&str, "A", n.AvailImpact)
-	str = strings.TrimSuffix(str, "/")
-	return str
-}
-
-func addVec(str *string, vec, val string) {
-	if val != "" {
-		if let, ok := vectorValuesToLetters[val]; ok {
-			*str = fmt.Sprintf("%s%s:%s/", *str, vec, let)
-		} else {
-			log.WithFields(log.Fields{"value": val, "vector": vec}).Warning("unknown value for CVSSv2 vector")
-		}
-	}
-}