From aab46f5658cf5a75262945033cb41d93af5f2131 Mon Sep 17 00:00:00 2001 From: Kate Murphy Date: Tue, 16 Oct 2018 18:42:15 -0400 Subject: [PATCH] ext: Parse NVD JSON feed instead of XML The JSON feed provides some values that are not available in the XML feed such as CVSSv3. --- ext/vulnmdsrc/nvd/{xml.go => json.go} | 57 +++++++++++++++++---------- ext/vulnmdsrc/nvd/nvd.go | 15 +++---- 2 files changed, 45 insertions(+), 27 deletions(-) rename ext/vulnmdsrc/nvd/{xml.go => json.go} (62%) diff --git a/ext/vulnmdsrc/nvd/xml.go b/ext/vulnmdsrc/nvd/json.go similarity index 62% rename from ext/vulnmdsrc/nvd/xml.go rename to ext/vulnmdsrc/nvd/json.go index 31ca35a2..306bfc27 100644 --- a/ext/vulnmdsrc/nvd/xml.go +++ b/ext/vulnmdsrc/nvd/json.go @@ -22,27 +22,39 @@ import ( ) type nvd struct { - Entries []nvdEntry `xml:"entry"` + Entries []nvdEntry `json:"CVE_Items"` } type nvdEntry struct { - Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` - CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` - PublishedDateTime string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 published-datetime"` + CVE nvdCVE `json:"cve"` + Impact nvdImpact `json:"impact"` + PublishedDateTime string `json:"publishedDate"` } -type nvdCVSS struct { - BaseMetrics nvdCVSSBaseMetrics `xml:"http://scap.nist.gov/schema/cvss-v2/0.2 base_metrics"` +type nvdCVE struct { + Metadata nvdCVEMetadata `json:"CVE_data_meta"` } -type nvdCVSSBaseMetrics struct { - Score float64 `xml:"score"` - AccessVector string `xml:"access-vector"` - AccessComplexity string `xml:"access-complexity"` - Authentication string `xml:"authentication"` - ConfImpact string `xml:"confidentiality-impact"` - IntegImpact string `xml:"integrity-impact"` - AvailImpact string `xml:"availability-impact"` +type nvdCVEMetadata struct { + CVEID string `json:"ID"` +} + +type nvdImpact struct { + BaseMetricV2 nvdBaseMetricV2 `json:"baseMetricV2"` +} + +type nvdBaseMetricV2 struct { + CVSSv2 nvdCVSSv2 `json:"cvssV2"` +} + +type nvdCVSSv2 struct { + Score float64 `json:"baseScore"` + AccessVector string `json:"accessVector"` + AccessComplexity string `json:"accessComplexity"` + Authentication string `json:"authentication"` + ConfImpact string `json:"confidentialityImpact"` + IntegImpact string `json:"integrityImpact"` + AvailImpact string `json:"availabilityImpact"` } var vectorValuesToLetters map[string]string @@ -56,8 +68,8 @@ func init() { vectorValuesToLetters["MEDIUM"] = "M" vectorValuesToLetters["LOW"] = "L" vectorValuesToLetters["NONE"] = "N" - vectorValuesToLetters["SINGLE_INSTANCE"] = "S" - vectorValuesToLetters["MULTIPLE_INSTANCES"] = "M" + vectorValuesToLetters["SINGLE"] = "S" + vectorValuesToLetters["MULTIPLE"] = "M" vectorValuesToLetters["PARTIAL"] = "P" vectorValuesToLetters["COMPLETE"] = "C" } @@ -66,18 +78,23 @@ func (n nvdEntry) Metadata() *NVDMetadata { metadata := &NVDMetadata{ CVSSv2: NVDmetadataCVSSv2{ PublishedDateTime: n.PublishedDateTime, - Vectors: n.CVSS.BaseMetrics.String(), - Score: n.CVSS.BaseMetrics.Score, + Vectors: n.Impact.BaseMetricV2.CVSSv2.String(), + Score: n.Impact.BaseMetricV2.CVSSv2.Score, }, } if metadata.CVSSv2.Vectors == "" { return nil } + return metadata } -func (n nvdCVSSBaseMetrics) String() string { +func (n nvdEntry) Name() string { + return n.CVE.Metadata.CVEID +} + +func (n nvdCVSSv2) String() string { var str string addVec(&str, "AV", n.AccessVector) addVec(&str, "AC", n.AccessComplexity) @@ -94,7 +111,7 @@ func addVec(str *string, vec, val string) { if let, ok := vectorValuesToLetters[val]; ok { *str = fmt.Sprintf("%s%s:%s/", *str, vec, let) } else { - log.WithFields(log.Fields{"value": val, "vector": vec}).Warning("unknown value for CVSSv2 vector") + log.WithFields(log.Fields{"value": val, "vector": vec}).Warning("unknown value for CVSS vector") } } } diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 96f93587..19f8dc1d 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -19,7 +19,7 @@ package nvd import ( "bufio" "compress/gzip" - "encoding/xml" + "encoding/json" "errors" "fmt" "io" @@ -39,8 +39,8 @@ import ( ) const ( - dataFeedURL string = "https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-%s.xml.gz" - dataFeedMetaURL string = "https://nvd.nist.gov/feeds/xml/cve/2.0/nvdcve-2.0-%s.meta" + dataFeedURL string = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-%s.json.gz" + dataFeedMetaURL string = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-%s.meta" appenderName string = "NVD" @@ -96,8 +96,9 @@ func (a *appender) BuildCache(datastore database.Datastore) error { return commonerr.ErrCouldNotParse } var nvd nvd + r := bufio.NewReader(f) - if err = xml.NewDecoder(r).Decode(&nvd); err != nil { + if err := json.NewDecoder(r).Decode(&nvd); err != nil { f.Close() log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not decode NVD data feed") return commonerr.ErrCouldNotParse @@ -107,7 +108,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error { for _, nvdEntry := range nvd.Entries { // Create metadata entry. if metadata := nvdEntry.Metadata(); metadata != nil { - a.metadata[nvdEntry.Name] = *metadata + a.metadata[nvdEntry.Name()] = *metadata } } f.Close() @@ -154,7 +155,8 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin // Create map containing the name and filename for every data feed. dataFeedReaders := make(map[string]string) for _, dataFeedName := range dataFeedNames { - fileName := filepath.Join(localPath, fmt.Sprintf("%s.xml", dataFeedName)) + fileName := filepath.Join(localPath, fmt.Sprintf("%s.json", dataFeedName)) + if h, ok := dataFeedHashes[dataFeedName]; ok && h == dataFeedHashes[dataFeedName] { // The hash is known, the disk should contains the feed. Try to read from it. if localPath != "" { @@ -177,7 +179,6 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } func downloadFeed(dataFeedName, fileName string) error { - // Download data feed. r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) if err != nil {