From 49cbdd7a7c5973fa38dcbd0821cbba332f51aef3 Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Thu, 6 Sep 2018 15:41:40 -0400 Subject: [PATCH 1/5] Using httputil for NVD nvd was missed when moving to httputil, this fixes it --- ext/vulnmdsrc/nvd/nvd.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index a36dec83..750559fe 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -24,7 +24,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" "strconv" @@ -36,6 +35,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/httputil" ) const ( @@ -166,11 +166,17 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } // Download data feed. - r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName)) + r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) if err != nil { log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload } + // r is closed in BuildCache() + + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to download NVD data feed") + return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload + } // Un-gzip it. gr, err := gzip.NewReader(r.Body) @@ -199,12 +205,16 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } func getHashFromMetaURL(metaURL string) (string, error) { - r, err := http.Get(metaURL) + r, err := httputil.GetWithUserAgent(metaURL) if err != nil { return "", err } defer r.Body.Close() + if !httputil.Status2xx(r) { + return "", errors.New("Unsuccesuful status code: " + string(r.StatusCode)) + } + scanner := bufio.NewScanner(r.Body) for scanner.Scan() { line := scanner.Text() From 3959f416fa1234a870b1ae076b82f9074d011511 Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Fri, 7 Sep 2018 10:47:11 -0400 Subject: [PATCH 2/5] Fix up error and changing close to defer close --- ext/vulnmdsrc/nvd/nvd.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 750559fe..0f4ec190 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -171,7 +171,7 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload } - // r is closed in BuildCache() + defer r.Body.Close() if !httputil.Status2xx(r) { log.WithField("StatusCode", r.StatusCode).Error("Failed to download NVD data feed") @@ -196,8 +196,6 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } else { log.WithError(err).Warning("could not store NVD data feed to filesystem") } - - r.Body.Close() } } @@ -212,7 +210,7 @@ func getHashFromMetaURL(metaURL string) (string, error) { defer r.Body.Close() if !httputil.Status2xx(r) { - return "", errors.New("Unsuccesuful status code: " + string(r.StatusCode)) + return "", errors.New(metaURL + " failed status code: " + string(r.StatusCode)) } scanner := bufio.NewScanner(r.Body) From f34f94320af00a070da924582aa24bea310eed7d Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Fri, 7 Sep 2018 17:13:31 -0400 Subject: [PATCH 3/5] Embed nvd's downloading and storing of meta data into a function to help with resource management --- ext/vulnmdsrc/nvd/nvd.go | 60 +++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 0f4ec190..0506e6fd 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -165,36 +165,46 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } } - // Download data feed. - r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) - if err != nil { - log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") - return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload - } - defer r.Body.Close() + downloadAndSave := func() error { - if !httputil.Status2xx(r) { - log.WithField("StatusCode", r.StatusCode).Error("Failed to download NVD data feed") - return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload - } + // Download data feed. + r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) + if err != nil { + log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") + return commonerr.ErrCouldNotDownload + } + defer r.Body.Close() - // Un-gzip it. - gr, err := gzip.NewReader(r.Body) - if err != nil { - log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not read NVD data feed") - return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload - } + if !httputil.Status2xx(r) { + log.WithField("StatusCode", r.StatusCode).Error("Failed to download NVD data feed") + return commonerr.ErrCouldNotDownload + } - // Store it to a file at the same time if possible. - if f, err := os.Create(fileName); err == nil { - _, err = io.Copy(f, gr) + // Un-gzip it. + gr, err := gzip.NewReader(r.Body) if err != nil { - log.WithError(err).Warning("could not stream NVD data feed to filesystem") + log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not read NVD data feed") + return commonerr.ErrCouldNotDownload } - dataFeedReaders[dataFeedName] = fileName - f.Close() - } else { - log.WithError(err).Warning("could not store NVD data feed to filesystem") + + // Store it to a file at the same time if possible. + if f, err := os.Create(fileName); err == nil { + _, err = io.Copy(f, gr) + if err != nil { + log.WithError(err).Warning("could not stream NVD data feed to filesystem") + } + defer f.Close() + dataFeedReaders[dataFeedName] = fileName + + } else { + log.WithError(err).Warning("could not store NVD data feed to filesystem") + } + return nil + } + + err := downloadAndSave() + if err != nil { + return dataFeedReaders, dataFeedHashes, err } } } From 56b4f23ae20ec7060c0a7390255532d416333a20 Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Mon, 10 Sep 2018 14:42:22 -0400 Subject: [PATCH 4/5] Move downloadFeed out to a seperate function --- ext/vulnmdsrc/nvd/nvd.go | 78 +++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 0506e6fd..ff5d9bf3 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -165,51 +165,55 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin } } - downloadAndSave := func() error { + err := downloadFeed(dataFeedName, fileName) + if err != nil { + return dataFeedReaders, dataFeedHashes, err + } + dataFeedReaders[dataFeedName] = fileName + } + } - // Download data feed. - r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) - if err != nil { - log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") - return commonerr.ErrCouldNotDownload - } - defer r.Body.Close() + return dataFeedReaders, dataFeedHashes, nil +} - if !httputil.Status2xx(r) { - log.WithField("StatusCode", r.StatusCode).Error("Failed to download NVD data feed") - return commonerr.ErrCouldNotDownload - } +func downloadFeed(dataFeedName, fileName string) error { - // Un-gzip it. - gr, err := gzip.NewReader(r.Body) - if err != nil { - log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not read NVD data feed") - return commonerr.ErrCouldNotDownload - } + // Download data feed. + r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName)) + if err != nil { + log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed") + return commonerr.ErrCouldNotDownload + } + defer r.Body.Close() - // Store it to a file at the same time if possible. - if f, err := os.Create(fileName); err == nil { - _, err = io.Copy(f, gr) - if err != nil { - log.WithError(err).Warning("could not stream NVD data feed to filesystem") - } - defer f.Close() - dataFeedReaders[dataFeedName] = fileName + if !httputil.Status2xx(r) { + log.WithFields(log.Fields{"StatusCode": r.StatusCode, "DataFeedName": dataFeedName}).Error("Failed to download NVD data feed") + return commonerr.ErrCouldNotDownload + } - } else { - log.WithError(err).Warning("could not store NVD data feed to filesystem") - } - return nil - } + // Un-gzip it. + gr, err := gzip.NewReader(r.Body) + if err != nil { + log.WithError(err).WithFields(log.Fields{"StatusCode": r.StatusCode, "DataFeedName": dataFeedName}).Error("could not read NVD data feed") + return commonerr.ErrCouldNotDownload + } - err := downloadAndSave() - if err != nil { - return dataFeedReaders, dataFeedHashes, err - } - } + // Store it to a file at the same time if possible. + f, err := os.Create(fileName) + if err != nil { + + log.WithError(err).WithField("Filename", fileName).Warning("could not store NVD data feed to filesystem") + return commonerr.ErrFilesystem } + defer f.Close() - return dataFeedReaders, dataFeedHashes, nil + _, err = io.Copy(f, gr) + if err != nil { + log.WithError(err).WithField("Filename", fileName).Warning("could not stream NVD data feed to filesystem") + return commonerr.ErrFilesystem + } + + return nil } func getHashFromMetaURL(metaURL string) (string, error) { From 30848d9eb76edad83c93058699c990a7e04ddc97 Mon Sep 17 00:00:00 2001 From: Jean Michel MacKay Date: Tue, 11 Sep 2018 15:28:26 -0400 Subject: [PATCH 5/5] Fixed extra newline --- ext/vulnmdsrc/nvd/nvd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index ff5d9bf3..96f93587 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -201,7 +201,6 @@ func downloadFeed(dataFeedName, fileName string) error { // Store it to a file at the same time if possible. f, err := os.Create(fileName) if err != nil { - log.WithError(err).WithField("Filename", fileName).Warning("could not store NVD data feed to filesystem") return commonerr.ErrFilesystem }