arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #610 from MackJM/wip/master_nvd_httputil

Browse Source 
Using httputil for NVD
This commit is contained in:
Jimmy Zelinskie 2018-09-19 14:25:44 -04:00 committed by GitHub
commit 44ae4bc959
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 48 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
ext/vulnmdsrc/nvd/nvd.go
View File

@ -24,7 +24,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strconv"
@ -36,6 +35,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/vulnmdsrc"
"github.com/coreos/clair/pkg/commonerr"
"github.com/coreos/clair/pkg/httputil"
)
const (
@ -165,46 +165,67 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin
}
}
// Download data feed.
r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName))
err := downloadFeed(dataFeedName, fileName)
if err != nil {
log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed")
return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload
}
// Un-gzip it.
gr, err := gzip.NewReader(r.Body)
if err != nil {
log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not read NVD data feed")
return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload
}
// Store it to a file at the same time if possible.
if f, err := os.Create(fileName); err == nil {
_, err = io.Copy(f, gr)
if err != nil {
log.WithError(err).Warning("could not stream NVD data feed to filesystem")
return dataFeedReaders, dataFeedHashes, err
}
dataFeedReaders[dataFeedName] = fileName
f.Close()
} else {
log.WithError(err).Warning("could not store NVD data feed to filesystem")
}
r.Body.Close()
}
}
return dataFeedReaders, dataFeedHashes, nil
}
func downloadFeed(dataFeedName, fileName string) error {
// Download data feed.
r, err := httputil.GetWithUserAgent(fmt.Sprintf(dataFeedURL, dataFeedName))
if err != nil {
log.WithError(err).WithField(logDataFeedName, dataFeedName).Error("could not download NVD data feed")
return commonerr.ErrCouldNotDownload
}
defer r.Body.Close()
if !httputil.Status2xx(r) {
log.WithFields(log.Fields{"StatusCode": r.StatusCode, "DataFeedName": dataFeedName}).Error("Failed to download NVD data feed")
return commonerr.ErrCouldNotDownload
}
// Un-gzip it.
gr, err := gzip.NewReader(r.Body)
if err != nil {
log.WithError(err).WithFields(log.Fields{"StatusCode": r.StatusCode, "DataFeedName": dataFeedName}).Error("could not read NVD data feed")
return commonerr.ErrCouldNotDownload
}
// Store it to a file at the same time if possible.
f, err := os.Create(fileName)
if err != nil {
log.WithError(err).WithField("Filename", fileName).Warning("could not store NVD data feed to filesystem")
return commonerr.ErrFilesystem
}
defer f.Close()
_, err = io.Copy(f, gr)
if err != nil {
log.WithError(err).WithField("Filename", fileName).Warning("could not stream NVD data feed to filesystem")
return commonerr.ErrFilesystem
}
return nil
}
func getHashFromMetaURL(metaURL string) (string, error) {
r, err := http.Get(metaURL)
r, err := httputil.GetWithUserAgent(metaURL)
if err != nil {
return "", err
}
defer r.Body.Close()
if !httputil.Status2xx(r) {
return "", errors.New(metaURL + " failed status code: " + string(r.StatusCode))
}
scanner := bufio.NewScanner(r.Body)
for scanner.Scan() {
line := scanner.Text()