You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
382 lines
11 KiB
382 lines
11 KiB
// Copyright 2017 clair authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
// Package ubuntu implements a vulnerability source updater using the
|
|
// Ubuntu CVE Tracker.
|
|
package ubuntu
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"regexp"
|
|
"strings"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
"github.com/coreos/clair/database"
|
|
"github.com/coreos/clair/ext/versionfmt"
|
|
"github.com/coreos/clair/ext/versionfmt/dpkg"
|
|
"github.com/coreos/clair/ext/vulnsrc"
|
|
"github.com/coreos/clair/pkg/commonerr"
|
|
)
|
|
|
|
const (
|
|
trackerGitURL = "https://git.launchpad.net/ubuntu-cve-tracker"
|
|
updaterFlag = "ubuntuUpdater"
|
|
cveURL = "http://people.ubuntu.com/~ubuntu-security/cve/%s"
|
|
)
|
|
|
|
var (
|
|
ubuntuIgnoredReleases = map[string]struct{}{
|
|
"upstream": {},
|
|
"devel": {},
|
|
|
|
"dapper": {},
|
|
"edgy": {},
|
|
"feisty": {},
|
|
"gutsy": {},
|
|
"hardy": {},
|
|
"intrepid": {},
|
|
"jaunty": {},
|
|
"karmic": {},
|
|
"lucid": {},
|
|
"maverick": {},
|
|
"natty": {},
|
|
"oneiric": {},
|
|
"saucy": {},
|
|
|
|
"vivid/ubuntu-core": {},
|
|
"vivid/stable-phone-overlay": {},
|
|
|
|
// Syntax error
|
|
"Patches": {},
|
|
// Product
|
|
"product": {},
|
|
}
|
|
|
|
affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
|
|
affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()
|
|
)
|
|
|
|
type updater struct {
|
|
repositoryLocalPath string
|
|
}
|
|
|
|
func init() {
|
|
vulnsrc.RegisterUpdater("ubuntu", &updater{})
|
|
}
|
|
|
|
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
|
|
log.WithField("package", "Ubuntu").Info("Start fetching vulnerabilities")
|
|
|
|
// Pull the master branch.
|
|
var commit string
|
|
commit, err = u.pullRepository()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// Get the latest revision number we successfully applied in the database.
|
|
dbCommit, err := datastore.GetKeyValue(updaterFlag)
|
|
if err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
// Short-circuit if there have been no updates.
|
|
if commit == dbCommit {
|
|
log.WithField("package", "ubuntu").Debug("no update")
|
|
return
|
|
}
|
|
|
|
// Get the list of vulnerabilities that we have to update.
|
|
modifiedCVE, err := collectModifiedVulnerabilities(u.repositoryLocalPath)
|
|
if err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
notes := make(map[string]struct{})
|
|
for cvePath := range modifiedCVE {
|
|
// Open the CVE file.
|
|
file, err := os.Open(u.repositoryLocalPath + "/" + cvePath)
|
|
if err != nil {
|
|
// This can happen when a file is modified and then moved in another
|
|
// commit.
|
|
continue
|
|
}
|
|
|
|
// Parse the vulnerability.
|
|
v, unknownReleases, err := parseUbuntuCVE(file)
|
|
if err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
// Add the vulnerability to the response.
|
|
resp.Vulnerabilities = append(resp.Vulnerabilities, v)
|
|
|
|
// Store any unknown releases as notes.
|
|
for k := range unknownReleases {
|
|
note := fmt.Sprintf("Ubuntu %s is not mapped to any version number (eg. trusty->14.04). Please update me.", k)
|
|
notes[note] = struct{}{}
|
|
|
|
// If we encountered unknown Ubuntu release, we don't want the revision
|
|
// number to be considered as managed.
|
|
commit = dbCommit
|
|
}
|
|
|
|
// Close the file manually.
|
|
//
|
|
// We do that instead of using defer because defer works on a function-level scope.
|
|
// We would open many files and close them all at once at the end of the function,
|
|
// which could lead to exceed fs.file-max.
|
|
file.Close()
|
|
}
|
|
|
|
// Add flag and notes.
|
|
resp.FlagName = updaterFlag
|
|
resp.FlagValue = commit
|
|
for note := range notes {
|
|
resp.Notes = append(resp.Notes, note)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (u *updater) Clean() {
|
|
os.RemoveAll(u.repositoryLocalPath)
|
|
}
|
|
|
|
func (u *updater) pullRepository() (commit string, err error) {
|
|
// If the repository doesn't exist, clone it.
|
|
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
|
|
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
|
|
return "", vulnsrc.ErrFilesystem
|
|
}
|
|
|
|
cmd := exec.Command("git", "clone", trackerGitURL, ".")
|
|
cmd.Dir = u.repositoryLocalPath
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
u.Clean()
|
|
log.WithError(err).WithField("output", string(out)).Error("could not pull ubuntu-cve-tracker repository")
|
|
return "", commonerr.ErrCouldNotDownload
|
|
}
|
|
} else {
|
|
// The repository already exists and it needs to be refreshed via a pull.
|
|
cmd := exec.Command("git", "pull")
|
|
cmd.Dir = u.repositoryLocalPath
|
|
if _, err := cmd.CombinedOutput(); err != nil {
|
|
return "", vulnsrc.ErrGitFailure
|
|
}
|
|
}
|
|
|
|
cmd := exec.Command("git", "rev-parse", "HEAD")
|
|
cmd.Dir = u.repositoryLocalPath
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
return "", vulnsrc.ErrGitFailure
|
|
}
|
|
|
|
commit = strings.TrimSpace(string(out))
|
|
return
|
|
}
|
|
|
|
func collectModifiedVulnerabilities(repositoryLocalPath string) (map[string]struct{}, error) {
|
|
modifiedCVE := make(map[string]struct{})
|
|
|
|
// Handle a brand new database.
|
|
|
|
for _, folder := range []string{"active", "retired"} {
|
|
d, err := os.Open(repositoryLocalPath + "/" + folder)
|
|
if err != nil {
|
|
log.WithError(err).Error("could not open Ubuntu vulnerabilities repository's folder")
|
|
return nil, vulnsrc.ErrFilesystem
|
|
}
|
|
|
|
// Get the FileInfo of all the files in the directory.
|
|
names, err := d.Readdirnames(-1)
|
|
if err != nil {
|
|
log.WithError(err).Error("could not read Ubuntu vulnerabilities repository's folder")
|
|
return nil, vulnsrc.ErrFilesystem
|
|
}
|
|
|
|
// Add the vulnerabilities to the list.
|
|
for _, name := range names {
|
|
if strings.HasPrefix(name, "CVE-") {
|
|
modifiedCVE[folder+"/"+name] = struct{}{}
|
|
}
|
|
}
|
|
|
|
// Close the file manually.
|
|
//
|
|
// We do that instead of using defer because defer works on a function-level scope.
|
|
// We would open many files and close them all at once at the end of the function,
|
|
// which could lead to exceed fs.file-max.
|
|
d.Close()
|
|
}
|
|
|
|
return modifiedCVE, nil
|
|
|
|
}
|
|
|
|
func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) {
|
|
unknownReleases = make(map[string]struct{})
|
|
readingDescription := false
|
|
scanner := bufio.NewScanner(fileContent)
|
|
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
|
|
// Skip any comments.
|
|
if strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
|
|
// Parse the name.
|
|
if strings.HasPrefix(line, "Candidate:") {
|
|
vulnerability.Name = strings.TrimSpace(strings.TrimPrefix(line, "Candidate:"))
|
|
vulnerability.Link = fmt.Sprintf(cveURL, vulnerability.Name)
|
|
continue
|
|
}
|
|
|
|
// Parse the priority.
|
|
if strings.HasPrefix(line, "Priority:") {
|
|
priority := strings.TrimSpace(strings.TrimPrefix(line, "Priority:"))
|
|
|
|
// Handle syntax error: Priority: medium (heap-protector)
|
|
if strings.Contains(priority, " ") {
|
|
priority = priority[:strings.Index(priority, " ")]
|
|
}
|
|
|
|
vulnerability.Severity = SeverityFromPriority(priority)
|
|
continue
|
|
}
|
|
|
|
// Parse the description.
|
|
if strings.HasPrefix(line, "Description:") {
|
|
readingDescription = true
|
|
vulnerability.Description = strings.TrimSpace(strings.TrimPrefix(line, "Description:")) // In case there is a formatting error and the description starts on the same line
|
|
continue
|
|
}
|
|
if readingDescription {
|
|
if strings.HasPrefix(line, "Ubuntu-Description:") || strings.HasPrefix(line, "Notes:") || strings.HasPrefix(line, "Bugs:") || strings.HasPrefix(line, "Priority:") || strings.HasPrefix(line, "Discovered-by:") || strings.HasPrefix(line, "Assigned-to:") {
|
|
readingDescription = false
|
|
} else {
|
|
vulnerability.Description = vulnerability.Description + " " + line
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Try to parse the package that the vulnerability affects.
|
|
affectsCaptureArr := affectsCaptureRegexp.FindAllStringSubmatch(line, -1)
|
|
if len(affectsCaptureArr) > 0 {
|
|
affectsCapture := affectsCaptureArr[0]
|
|
|
|
md := map[string]string{}
|
|
for i, n := range affectsCapture {
|
|
md[affectsCaptureRegexpNames[i]] = strings.TrimSpace(n)
|
|
}
|
|
|
|
// Ignore Linux kernels.
|
|
if strings.HasPrefix(md["package"], "linux") {
|
|
continue
|
|
}
|
|
|
|
// Only consider the package if its status is needed, active, deferred, not-affected or
|
|
// released. Ignore DNE (package does not exist), needs-triage, ignored, pending.
|
|
if md["status"] == "needed" || md["status"] == "active" || md["status"] == "deferred" || md["status"] == "released" || md["status"] == "not-affected" {
|
|
md["release"] = strings.Split(md["release"], "/")[0]
|
|
if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
|
|
continue
|
|
}
|
|
if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
|
|
unknownReleases[md["release"]] = struct{}{}
|
|
continue
|
|
}
|
|
|
|
var version string
|
|
if md["status"] == "released" {
|
|
if md["note"] != "" {
|
|
var err error
|
|
err = versionfmt.Valid(dpkg.ParserName, md["note"])
|
|
if err != nil {
|
|
log.WithError(err).WithField("version", md["note"]).Warning("could not parse package version. skipping")
|
|
}
|
|
version = md["note"]
|
|
}
|
|
} else if md["status"] == "not-affected" {
|
|
version = versionfmt.MinVersion
|
|
} else {
|
|
version = versionfmt.MaxVersion
|
|
}
|
|
if version == "" {
|
|
continue
|
|
}
|
|
|
|
// Create and add the new package.
|
|
featureVersion := database.FeatureVersion{
|
|
Feature: database.Feature{
|
|
Namespace: database.Namespace{
|
|
Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]],
|
|
VersionFormat: dpkg.ParserName,
|
|
},
|
|
Name: md["package"],
|
|
},
|
|
Version: version,
|
|
}
|
|
vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Trim extra spaces in the description
|
|
vulnerability.Description = strings.TrimSpace(vulnerability.Description)
|
|
|
|
// If no link has been provided (CVE-2006-NNN0 for instance), add the link to the tracker
|
|
if vulnerability.Link == "" {
|
|
vulnerability.Link = trackerGitURL
|
|
}
|
|
|
|
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
|
|
if vulnerability.Severity == "" {
|
|
vulnerability.Severity = database.UnknownSeverity
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// SeverityFromPriority converts an priority from the Ubuntu CVE Tracker into
|
|
// a database.Severity.
|
|
func SeverityFromPriority(priority string) database.Severity {
|
|
switch priority {
|
|
case "untriaged":
|
|
return database.UnknownSeverity
|
|
case "negligible":
|
|
return database.NegligibleSeverity
|
|
case "low":
|
|
return database.LowSeverity
|
|
case "medium":
|
|
return database.MediumSeverity
|
|
case "high":
|
|
return database.HighSeverity
|
|
case "critical":
|
|
return database.CriticalSeverity
|
|
default:
|
|
log.Warning("could not determine a vulnerability severity from: %s", priority)
|
|
return database.UnknownSeverity
|
|
}
|
|
}
|