arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

vulnsrc/ubuntu: use new git-based ubuntu tracker

Browse Source
This commit is contained in:
Jimmy Zelinskie 2018-07-05 19:11:50 -04:00
parent c031f8ea0c
commit 456af5f48c
1 changed files with 128 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

238
ext/vulnsrc/ubuntu/ubuntu.go
View File

@ -18,14 +18,13 @@ package ubuntu
import ( import (
"bufio" "bufio"
"bytes" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"os/exec" "os/exec"
"regexp" "regexp"
"strconv"
"strings" "strings"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -38,8 +37,7 @@ import (
) )
const ( const (
trackerURI = "https://launchpad.net/ubuntu-cve-tracker" trackerURI = "https://git.launchpad.net/ubuntu-cve-tracker"
trackerRepository = "https://launchpad.net/ubuntu-cve-tracker"
updaterFlag = "ubuntuUpdater" updaterFlag = "ubuntuUpdater"
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s" cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
) )
@ -74,6 +72,8 @@ var (
affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`) affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames() affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()
errUnknownRelease = errors.New("found packages with CVEs for a verison of Ubuntu that Clair doesn't know about")
) )
type updater struct { type updater struct {
@ -84,211 +84,179 @@ func init() {
vulnsrc.RegisterUpdater("ubuntu", &updater{}) vulnsrc.RegisterUpdater("ubuntu", &updater{})
} }
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.WithField("package", "Ubuntu").Info("Start fetching vulnerabilities") log.WithField("package", "Ubuntu").Info("Start fetching vulnerabilities")
// Pull the bzr repository. // Pull the master branch.
if err = u.pullRepository(); err != nil { var commit string
return resp, err commit, err = u.pullRepository()
}
// Get revision number.
revisionNumber, err := getRevisionNumber(u.repositoryLocalPath)
if err != nil { if err != nil {
return resp, err return resp, err
} }
tx, err := datastore.Begin() // Open a database transaction.
tx, err := db.Begin()
if err != nil { if err != nil {
return resp, err return resp, err
} }
defer tx.Rollback()
// Get the latest revision number we successfully applied in the database. // Ask the database for the latest commit we successfully applied.
dbRevisionNumber, ok, err := tx.FindKeyValue("ubuntuUpdater") var dbCommit string
var ok bool
dbCommit, ok, err = tx.FindKeyValue(updaterFlag)
if err != nil { if err != nil {
return resp, err return
} }
if err := tx.Rollback(); err != nil {
return resp, err
}
if !ok { if !ok {
dbRevisionNumber = "" dbCommit = ""
}
// Set the updaterFlag to equal the commit processed.
resp.FlagName = updaterFlag
resp.FlagValue = commit
// Short-circuit if there have been no updates.
if commit == dbCommit {
log.WithField("package", "ubuntu").Debug("no update")
return
} }
// Get the list of vulnerabilities that we have to update. // Get the list of vulnerabilities that we have to update.
modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath) var modifiedCVE map[string]struct{}
modifiedCVE, err = collectModifiedVulnerabilities(commit, dbCommit, u.repositoryLocalPath)
if err != nil { if err != nil {
return resp, err return
} }
notes := make(map[string]struct{}) // Get the list of vulnerabilities.
for cvePath := range modifiedCVE { resp.Vulnerabilities, resp.Notes, err = collectVulnerabilitiesAndNotes(u.repositoryLocalPath, modifiedCVE)
// Open the CVE file.
file, err := os.Open(u.repositoryLocalPath + "/" + cvePath)
if err != nil { if err != nil {
// This can happen when a file is modified and then moved in another return
// commit.
continue
} }
// Parse the vulnerability. // The only notes we take are if we encountered unknown Ubuntu release.
v, unknownReleases, err := parseUbuntuCVE(file) // We don't want the commit to be considered as managed in that case.
if err != nil { if len(resp.Notes) != 0 {
return resp, err resp.FlagValue = dbCommit
}
// Add the vulnerability to the response.
resp.Vulnerabilities = append(resp.Vulnerabilities, v)
// Store any unknown releases as notes.
for k := range unknownReleases {
note := fmt.Sprintf("Ubuntu %s is not mapped to any version number (eg. trusty->14.04). Please update me.", k)
notes[note] = struct{}{}
// If we encountered unknown Ubuntu release, we don't want the revision
// number to be considered as managed.
dbRevisionNumberInt, _ := strconv.Atoi(dbRevisionNumber)
revisionNumber = dbRevisionNumberInt
}
// Close the file manually.
//
// We do that instead of using defer because defer works on a function-level scope.
// We would open many files and close them all at once at the end of the function,
// which could lead to exceed fs.file-max.
file.Close()
}
// Add flag and notes.
resp.FlagName = updaterFlag
resp.FlagValue = strconv.Itoa(revisionNumber)
for note := range notes {
resp.Notes = append(resp.Notes, note)
} }
return return
} }
func (u *updater) Clean() { func (u *updater) Clean() {
if u.repositoryLocalPath != "" {
os.RemoveAll(u.repositoryLocalPath) os.RemoveAll(u.repositoryLocalPath)
} }
}
func (u *updater) pullRepository() (err error) { func (u *updater) pullRepository() (commit string, err error) {
// Determine whether we should branch or pull. // Determine whether we should branch or pull.
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
// Create a temporary folder to store the repository. // Create a temporary folder to store the repository.
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return vulnsrc.ErrFilesystem return "", vulnsrc.ErrFilesystem
} }
cmd := exec.Command("git", "clone", trackerURI, ".")
// Branch repository.
cmd := exec.Command("bzr", "branch", "--use-existing-dir", trackerRepository, ".")
cmd.Dir = u.repositoryLocalPath cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil { if out, err := cmd.CombinedOutput(); err != nil {
log.WithError(err).WithField("output", string(out)).Error("could not branch Ubuntu repository") u.Clean()
return commonerr.ErrCouldNotDownload log.WithError(err).WithField("output", string(out)).Error("could not clone ubuntu-cve-tracker repository")
return "", commonerr.ErrCouldNotDownload
} }
} else {
return nil // The repository already exists and it needs to be refreshed via a pull.
} cmd := exec.Command("git", "pull")
// Pull repository.
cmd := exec.Command("bzr", "pull", "--overwrite")
cmd.Dir = u.repositoryLocalPath cmd.Dir = u.repositoryLocalPath
if out, err := cmd.CombinedOutput(); err != nil { if _, err := cmd.CombinedOutput(); err != nil {
os.RemoveAll(u.repositoryLocalPath) return "", vulnsrc.ErrGitFailure
log.WithError(err).WithField("output", string(out)).Error("could not pull Ubuntu repository") }
return commonerr.ErrCouldNotDownload
} }
return nil cmd := exec.Command("git", "rev-parse", "HEAD")
} cmd.Dir = u.repositoryLocalPath
func getRevisionNumber(pathToRepo string) (int, error) {
cmd := exec.Command("bzr", "revno")
cmd.Dir = pathToRepo
out, err := cmd.CombinedOutput() out, err := cmd.CombinedOutput()
if err != nil { if err != nil {
log.WithError(err).WithField("output", string(out)).Error("could not get Ubuntu repository's revision number") return "", vulnsrc.ErrGitFailure
return 0, commonerr.ErrCouldNotDownload
} }
revno, err := strconv.Atoi(strings.TrimSpace(string(out))) commit = strings.TrimSpace(string(out))
if err != nil { return
log.WithError(err).WithField("output", string(out)).Error("could not parse Ubuntu repository's revision number")
return 0, commonerr.ErrCouldNotDownload
} }
return revno, nil func collectModifiedVulnerabilities(commit, dbCommit, repositoryLocalPath string) (map[string]struct{}, error) {
}
func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPath string) (map[string]struct{}, error) {
modifiedCVE := make(map[string]struct{}) modifiedCVE := make(map[string]struct{})
for _, dirName := range []string{"active", "retired"} {
if err := processDirectory(repositoryLocalPath, dirName, modifiedCVE); err != nil {
return nil, err
}
}
return modifiedCVE, nil
}
// Handle a brand new database. func processDirectory(repositoryLocalPath, dirName string, modifiedCVE map[string]struct{}) error {
if dbRevision == "" { // Open the directory.
for _, folder := range []string{"active", "retired"} { d, err := os.Open(repositoryLocalPath + "/" + dirName)
d, err := os.Open(repositoryLocalPath + "/" + folder)
if err != nil { if err != nil {
log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder") log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder")
return nil, vulnsrc.ErrFilesystem return vulnsrc.ErrFilesystem
} }
defer d.Close()
// Get the FileInfo of all the files in the directory. // Get the FileInfo of all the files in the directory.
names, err := d.Readdirnames(-1) names, err := d.Readdirnames(-1)
if err != nil { if err != nil {
log.WithError(err).Error("could not read Ubuntu vulnerabilities repository's folder") log.WithError(err).Error("could not read Ubuntu vulnerabilities repository's folder")
return nil, vulnsrc.ErrFilesystem return vulnsrc.ErrFilesystem
} }
// Add the vulnerabilities to the list. // Add the vulnerabilities to the list.
for _, name := range names { for _, name := range names {
if strings.HasPrefix(name, "CVE-") { if strings.HasPrefix(name, "CVE-") {
modifiedCVE[folder+"/"+name] = struct{}{} modifiedCVE[dirName+"/"+name] = struct{}{}
} }
} }
// Close the file manually. return nil
//
// We do that instead of using defer because defer works on a function-level scope.
// We would open many files and close them all at once at the end of the function,
// which could lead to exceed fs.file-max.
d.Close()
} }
return modifiedCVE, nil func collectVulnerabilitiesAndNotes(repositoryLocalPath string, modifiedCVE map[string]struct{}) ([]database.VulnerabilityWithAffected, []string, error) {
} vulns := make([]database.VulnerabilityWithAffected, 0)
noteSet := make(map[string]struct{})
// Handle an up to date database. for cvePath := range modifiedCVE {
dbRevisionInt, _ := strconv.Atoi(dbRevision) // Open the CVE file.
if revision == dbRevisionInt { file, err := os.Open(repositoryLocalPath + "/" + cvePath)
log.WithField("package", "Ubuntu").Debug("no update")
return modifiedCVE, nil
}
// Handle a database that needs upgrading.
cmd := exec.Command("bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0")
cmd.Dir = repositoryLocalPath
out, err := cmd.CombinedOutput()
if err != nil { if err != nil {
log.WithError(err).WithField("output", string(out)).Error("could not get Ubuntu vulnerabilities repository logs") // This can happen when a file is modified then moved in another commit.
return nil, commonerr.ErrCouldNotDownload continue
} }
scanner := bufio.NewScanner(bytes.NewReader(out)) // Parse the vulnerability.
for scanner.Scan() { v, unknownReleases, err := parseUbuntuCVE(file)
text := strings.TrimSpace(scanner.Text()) if err != nil {
if strings.Contains(text, "CVE-") && (strings.HasPrefix(text, "active/") || strings.HasPrefix(text, "retired/")) { file.Close()
if strings.Contains(text, " => ") { return nil, nil, err
text = text[strings.Index(text, " => ")+4:]
}
modifiedCVE[text] = struct{}{}
}
} }
return modifiedCVE, nil // Add the vulnerability to the response.
vulns = append(vulns, v)
// Store any unknown releases as notes.
for k := range unknownReleases {
noteSet[errUnknownRelease.Error()+": "+k] = struct{}{}
}
file.Close()
}
// Convert the note set into a slice.
var notes []string
for note := range noteSet {
notes = append(notes, note)
}
return vulns, notes, nil
} }
func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.VulnerabilityWithAffected, unknownReleases map[string]struct{}, err error) { func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.VulnerabilityWithAffected, unknownReleases map[string]struct{}, err error) {