From df1dd5c1491da6436d762fd0f3255b6a2e53d154 Mon Sep 17 00:00:00 2001 From: ErikThoreson Date: Thu, 12 Jul 2018 16:40:05 -0500 Subject: [PATCH] adding publisher datetime and updating nvd feed download --- ext/vulnmdsrc/nvd/nested_read_closer.go | 33 --------------- ext/vulnmdsrc/nvd/nvd.go | 53 +++++++++++++------------ ext/vulnmdsrc/nvd/xml.go | 10 +++-- 3 files changed, 33 insertions(+), 63 deletions(-) delete mode 100644 ext/vulnmdsrc/nvd/nested_read_closer.go diff --git a/ext/vulnmdsrc/nvd/nested_read_closer.go b/ext/vulnmdsrc/nvd/nested_read_closer.go deleted file mode 100644 index 23322655..00000000 --- a/ext/vulnmdsrc/nvd/nested_read_closer.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nvd - -import "io" - -// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed -// io.ReadCloser. -// It allows chaining io.ReadCloser together and still keep the ability to close them all in a -// simple manner. -type NestedReadCloser struct { - io.Reader - NestedReadClosers []io.ReadCloser -} - -// Close closes the gzip.Reader and the underlying io.ReadCloser. -func (nrc *NestedReadCloser) Close() { - for _, nestedReadCloser := range nrc.NestedReadClosers { - nestedReadCloser.Close() - } -} diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 5be08b97..17896ee6 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -38,8 +38,8 @@ import ( ) const ( - dataFeedURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz" - dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" + dataFeedURL string = "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz" + dataFeedMetaURL string = "https://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" appenderName string = "NVD" @@ -57,8 +57,9 @@ type NVDMetadata struct { } type NVDmetadataCVSSv2 struct { - Vectors string - Score float64 + PublishedDateTime string + Vectors string + Score float64 } func init() { @@ -87,9 +88,16 @@ func (a *appender) BuildCache(datastore database.Datastore) error { a.dataFeedHashes = dataFeedHashes // Parse data feeds. - for dataFeedName, dataFeedReader := range dataFeedReaders { + for dataFeedName, dataFileName := range dataFeedReaders { + f, err := os.Open(dataFileName) + if err != nil { + log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not open NVD data file") + return commonerr.ErrCouldNotParse + } var nvd nvd - if err = xml.NewDecoder(dataFeedReader).Decode(&nvd); err != nil { + r := bufio.NewReader(f) + if err = xml.NewDecoder(r).Decode(&nvd); err != nil { + f.Close() log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not decode NVD data feed") return commonerr.ErrCouldNotParse } @@ -101,8 +109,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error { a.metadata[nvdEntry.Name] = *metadata } } - - dataFeedReader.Close() + f.Close() } return nil @@ -124,7 +131,7 @@ func (a *appender) Clean() { os.RemoveAll(a.localPath) } -func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) { +func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]string, map[string]string, error) { var dataFeedNames []string for y := 2002; y <= time.Now().Year(); y++ { dataFeedNames = append(dataFeedNames, strconv.Itoa(y)) @@ -143,19 +150,16 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin dataFeedHashes[dataFeedName] = hash } - // Create io.Reader for every data feed. - dataFeedReaders := make(map[string]NestedReadCloser) + // Create map containing the name and filename for every data feed. + dataFeedReaders := make(map[string]string) for _, dataFeedName := range dataFeedNames { fileName := localPath + dataFeedName + ".xml" - if h, ok := dataFeedHashes[dataFeedName]; ok && h == dataFeedHashes[dataFeedName] { // The hash is known, the disk should contains the feed. Try to read from it. if localPath != "" { if f, err := os.Open(fileName); err == nil { - dataFeedReaders[dataFeedName] = NestedReadCloser{ - Reader: f, - NestedReadClosers: []io.ReadCloser{f}, - } + f.Close() + dataFeedReaders[dataFeedName] = fileName continue } } @@ -176,20 +180,17 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin // Store it to a file at the same time if possible. if f, err := os.Create(fileName); err == nil { - nrc := NestedReadCloser{ - Reader: io.TeeReader(gr, f), - NestedReadClosers: []io.ReadCloser{r.Body, gr, f}, + _, err = io.Copy(f, gr) + if err != nil { + log.WithError(err).Warning("could not stream NVD data feed to filesystem") } - dataFeedReaders[dataFeedName] = nrc + dataFeedReaders[dataFeedName] = fileName + f.Close() } else { - nrc := NestedReadCloser{ - Reader: gr, - NestedReadClosers: []io.ReadCloser{gr, r.Body}, - } - dataFeedReaders[dataFeedName] = nrc - log.WithError(err).Warning("could not store NVD data feed to filesystem") } + + r.Body.Close() } } diff --git a/ext/vulnmdsrc/nvd/xml.go b/ext/vulnmdsrc/nvd/xml.go index 99e1420d..31ca35a2 100644 --- a/ext/vulnmdsrc/nvd/xml.go +++ b/ext/vulnmdsrc/nvd/xml.go @@ -26,8 +26,9 @@ type nvd struct { } type nvdEntry struct { - Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` - CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` + Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` + CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` + PublishedDateTime string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 published-datetime"` } type nvdCVSS struct { @@ -64,8 +65,9 @@ func init() { func (n nvdEntry) Metadata() *NVDMetadata { metadata := &NVDMetadata{ CVSSv2: NVDmetadataCVSSv2{ - Vectors: n.CVSS.BaseMetrics.String(), - Score: n.CVSS.BaseMetrics.Score, + PublishedDateTime: n.PublishedDateTime, + Vectors: n.CVSS.BaseMetrics.String(), + Score: n.CVSS.BaseMetrics.Score, }, }