arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updater: port updater and its fetchers

Browse Source
This commit is contained in:
Quentin Machu 2016-01-19 19:17:08 -05:00 committed by Jimmy Zelinskie
parent fd6fdbd3f9
commit 77387af2ac
15 changed files with 140 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/handlers.go
View File

@ -46,8 +46,8 @@ func GETVersions(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _
} }
// GETHealth sums up the health of all the registered services. // GETHealth sums up the health of all the registered services.
func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _ *Env) { func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, e *Env) {
globalHealth, statuses := health.Healthcheck() globalHealth, statuses := health.Healthcheck(e.Datastore)
httpStatus := http.StatusOK httpStatus := http.StatusOK
if !globalHealth { if !globalHealth {

5
clair.go
View File

@ -25,6 +25,7 @@ import (
"github.com/coreos/clair/api" "github.com/coreos/clair/api"
"github.com/coreos/clair/config" "github.com/coreos/clair/config"
"github.com/coreos/clair/database/pgsql" "github.com/coreos/clair/database/pgsql"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/utils" "github.com/coreos/clair/utils"
"github.com/coreos/pkg/capnslog" "github.com/coreos/pkg/capnslog"
) )
@ -55,8 +56,8 @@ func Boot(config *config.Config) {
go api.RunHealth(config.API, &api.Env{Datastore: db}, st) go api.RunHealth(config.API, &api.Env{Datastore: db}, st)
// Start updater // Start updater
// st.Begin() st.Begin()
// go updater.Run(config.Updater, st) go updater.Run(config.Updater, db, st)
// Wait for interruption and shutdown gracefully. // Wait for interruption and shutdown gracefully.
waitForSignals(os.Interrupt) waitForSignals(os.Interrupt)

1
database/pgsql/feature.go
View File

@ -146,7 +146,6 @@ func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.Fea
if featureVersion.Version.Compare(fixedInVersion) < 0 { if featureVersion.Version.Compare(fixedInVersion) < 0 {
// The version of the FeatureVersion we are inserting is lower than the fixed version on this // The version of the FeatureVersion we are inserting is lower than the fixed version on this
// Vulnerability, thus, this FeatureVersion is affected by it. // Vulnerability, thus, this FeatureVersion is affected by it.
// TODO(Quentin-M): Prepare.
_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID, _, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID,
featureVersion.ID, fixedInID) featureVersion.ID, fixedInID)
if err != nil { if err != nil {

6
database/pgsql/pgsql.go
View File

@ -99,7 +99,7 @@ func createDatabase(dataSource, databaseName string) error {
defer db.Close() defer db.Close()
// Create database. // Create database.
_, err = db.Exec("CREATE DATABASE " + databaseName + ";") _, err = db.Exec("CREATE DATABASE " + databaseName)
if err != nil { if err != nil {
return fmt.Errorf("could not create database: %v", err) return fmt.Errorf("could not create database: %v", err)
} }
@ -118,7 +118,7 @@ func dropDatabase(dataSource, databaseName string) error {
defer db.Close() defer db.Close()
// Drop database. // Drop database.
_, err = db.Exec("DROP DATABASE " + databaseName + ";") _, err = db.Exec("DROP DATABASE " + databaseName)
if err != nil { if err != nil {
return fmt.Errorf("could not drop database: %v", err) return fmt.Errorf("could not drop database: %v", err)
} }
@ -167,7 +167,7 @@ func OpenForTest(name string, withTestData bool) (*pgSQLTest, error) {
d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/testdata/data.sql") d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/testdata/data.sql")
_, err = db.(*pgSQL).Exec(string(d)) _, err = db.(*pgSQL).Exec(string(d))
if err != nil { if err != nil {
dropDatabase(dataSource, dbName) dropDatabase(dataSource+"dbname=postgres", dbName)
log.Error(err) log.Error(err)
return nil, database.ErrCantOpen return nil, database.ErrCantOpen
} }

43
database/pgsql/vulnerability.go
View File

@ -2,6 +2,7 @@ package pgsql
import ( import (
"database/sql" "database/sql"
"fmt"
"github.com/coreos/clair/database" "github.com/coreos/clair/database"
"github.com/coreos/clair/utils" "github.com/coreos/clair/utils"
@ -69,6 +70,7 @@ func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerabili
for _, vulnerability := range vulnerabilities { for _, vulnerability := range vulnerabilities {
err := pgSQL.insertVulnerability(vulnerability) err := pgSQL.insertVulnerability(vulnerability)
if err != nil { if err != nil {
fmt.Printf("%#v\n", vulnerability)
return err return err
} }
} }
@ -121,16 +123,18 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
return nil return nil
} }
// Insert or find the new FeatureVersions. // Insert or find the new Feature.
// We already have the Feature IDs in updatedFixedInFeatureVersions because diffFixedIn fills them // We already have the Feature IDs in updatedFixedInFeatureVersions because diffFixedIn fills them
// in using the existing vulnerability's FixedIn FeatureVersions. Note that even if FixedIn // in using the existing vulnerability's FixedIn FeatureVersions. Note that even if FixedIn
// is type FeatureVersion, the actual stored ID in these structs are the Feature IDs. // is type FeatureVersion, the actual stored ID in these structs are the Feature IDs.
// //
// Also, we enforce the namespace of the FeatureVersion in case it was empty. There is a test // Also, we enforce the namespace of the FeatureVersion in case it was empty. There is a test
// above to ensure that the passed Namespace is either the same as the vulnerability or empty. // above to ensure that the passed Namespace is either the same as the vulnerability or empty.
//
// TODO(Quentin-M): Batch me.
for i := 0; i < len(newFixedInFeatureVersions); i++ { for i := 0; i < len(newFixedInFeatureVersions); i++ {
newFixedInFeatureVersions[i].Feature.Namespace.Name = vulnerability.Namespace.Name newFixedInFeatureVersions[i].Feature.Namespace.Name = vulnerability.Namespace.Name
newFixedInFeatureVersions[i].ID, err = pgSQL.insertFeatureVersion(newFixedInFeatureVersions[i]) newFixedInFeatureVersions[i].Feature.ID, err = pgSQL.insertFeature(newFixedInFeatureVersions[i].Feature)
if err != nil { if err != nil {
return err return err
} }
@ -229,15 +233,23 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
var fixedInID int var fixedInID int
for _, fv := range newFixedInFeatureVersions { for _, fv := range newFixedInFeatureVersions {
if fv.Version == types.MinVersion {
// We don't want to mark a Feature as fixed in MinVersion. MinVersion only makes sense when a
// Feature is already marked as fixed in some version, in which case we would be in the
// "updatedFixedInFeatureVersions" loop and removes the fixed in mark.
continue
}
// Insert Vulnerability_FixedIn_Feature. // Insert Vulnerability_FixedIn_Feature.
err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID, err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.Feature.ID,
&fv.Version).Scan(&fixedInID) &fv.Version).Scan(&fixedInID)
if err != nil { if err != nil {
return handleError("i_vulnerability_fixedin_feature", err) return handleError("i_vulnerability_fixedin_feature", err)
} }
// Insert Vulnerability_Affects_FeatureVersion. // Insert Vulnerability_Affects_FeatureVersion.
err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version) err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID,
fv.Version)
if err != nil { if err != nil {
return err return err
} }
@ -246,8 +258,8 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
for _, fv := range updatedFixedInFeatureVersions { for _, fv := range updatedFixedInFeatureVersions {
if fv.Version != types.MinVersion { if fv.Version != types.MinVersion {
// Update Vulnerability_FixedIn_Feature. // Update Vulnerability_FixedIn_Feature.
err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID, err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID,
&fv.Version).Scan(&fixedInID) fv.Feature.ID, &fv.Version).Scan(&fixedInID)
if err != nil { if err != nil {
return handleError("u_vulnerability_fixedin_feature", err) return handleError("u_vulnerability_fixedin_feature", err)
} }
@ -259,7 +271,8 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
} }
// Insert Vulnerability_Affects_FeatureVersion. // Insert Vulnerability_Affects_FeatureVersion.
err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version) err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID,
fv.Version)
if err != nil { if err != nil {
return err return err
} }
@ -267,15 +280,17 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
// Updating FixedIn by saying that the fixed version is the lowest possible version, it // Updating FixedIn by saying that the fixed version is the lowest possible version, it
// basically means that the vulnerability doesn't affect the feature (anymore). // basically means that the vulnerability doesn't affect the feature (anymore).
// Drop it from Vulnerability_FixedIn_Feature and Vulnerability_Affects_FeatureVersion. // Drop it from Vulnerability_FixedIn_Feature and Vulnerability_Affects_FeatureVersion.
err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID). err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID,
Scan(&fixedInID) fv.Feature.ID).Scan(&fixedInID)
if err != nil { if err != nil && err != sql.ErrNoRows {
return handleError("r_vulnerability_fixedin_feature", err) return handleError("r_vulnerability_fixedin_feature", err)
} }
_, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID) if err == nil {
if err != nil { _, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID)
return handleError("r_vulnerability_affects_featureversion", err) if err != nil {
return handleError("r_vulnerability_affects_featureversion", err)
}
} }
} }
} }
@ -304,7 +319,7 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
} }
if featureVersionVersion.Compare(fixedInVersion) < 0 { if featureVersionVersion.Compare(fixedInVersion) < 0 {
_, err := tx.Exec("i_vulnerability_affects_featureversion", vulnerabilityID, featureVersionID, _, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID, featureVersionID,
fixedInID) fixedInID)
if err != nil { if err != nil {
return handleError("i_vulnerability_affects_featureversion", err) return handleError("i_vulnerability_affects_featureversion", err)

8
health/health.go
View File

@ -19,6 +19,8 @@ package health
import ( import (
"fmt" "fmt"
"sync" "sync"
"github.com/coreos/clair/database"
) )
// Status defines a way to know the health status of a service // Status defines a way to know the health status of a service
@ -33,7 +35,7 @@ type Status struct {
} }
// A Healthchecker function is a method returning the Status of the tested service // A Healthchecker function is a method returning the Status of the tested service
type Healthchecker func() Status type Healthchecker func(database.Datastore) Status
var ( var (
healthcheckersLock sync.Mutex healthcheckersLock sync.Mutex
@ -59,12 +61,12 @@ func RegisterHealthchecker(name string, f Healthchecker) {
} }
// Healthcheck calls every registered Healthchecker and summarize their output // Healthcheck calls every registered Healthchecker and summarize their output
func Healthcheck() (bool, map[string]interface{}) { func Healthcheck(datastore database.Datastore) (bool, map[string]interface{}) {
globalHealth := true globalHealth := true
statuses := make(map[string]interface{}) statuses := make(map[string]interface{})
for serviceName, serviceChecker := range healthcheckers { for serviceName, serviceChecker := range healthcheckers {
status := serviceChecker() status := serviceChecker(datastore)
globalHealth = globalHealth && (!status.IsEssential || status.IsHealthy) globalHealth = globalHealth && (!status.IsEssential || status.IsHealthy)
statuses[serviceName] = struct { statuses[serviceName] = struct {

5
updater/fetchers.go
View File

@ -20,7 +20,7 @@ var fetchers = make(map[string]Fetcher)
// Fetcher represents anything that can fetch vulnerabilities. // Fetcher represents anything that can fetch vulnerabilities.
type Fetcher interface { type Fetcher interface {
FetchUpdate() (FetcherResponse, error) FetchUpdate(database.Datastore) (FetcherResponse, error)
} }
// FetcherResponse represents the sum of results of an update. // FetcherResponse represents the sum of results of an update.
@ -28,8 +28,7 @@ type FetcherResponse struct {
FlagName string FlagName string
FlagValue string FlagValue string
Notes []string Notes []string
Vulnerabilities []*database.Vulnerability Vulnerabilities []database.Vulnerability
Packages []*database.Package
} }
// RegisterFetcher makes a Fetcher available by the provided name. // RegisterFetcher makes a Fetcher available by the provided name.

53
updater/fetchers/debian/debian.go
View File

@ -27,6 +27,7 @@ import (
"github.com/coreos/clair/updater" "github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors" cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types" "github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
) )
const ( const (
@ -35,6 +36,8 @@ const (
debianUpdaterFlag = "debianUpdater" debianUpdaterFlag = "debianUpdater"
) )
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian")
type jsonData map[string]map[string]jsonVuln type jsonData map[string]map[string]jsonVuln
type jsonVuln struct { type jsonVuln struct {
@ -57,7 +60,7 @@ func init() {
} }
// FetchUpdate fetches vulnerability updates from the Debian Security Tracker. // FetchUpdate fetches vulnerability updates from the Debian Security Tracker.
func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
log.Info("fetching Debian vulnerabilities") log.Info("fetching Debian vulnerabilities")
// Download JSON. // Download JSON.
@ -68,7 +71,7 @@ func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err e
} }
// Get the SHA-1 of the latest update's JSON data // Get the SHA-1 of the latest update's JSON data
latestHash, err := database.GetFlagValue(debianUpdaterFlag) latestHash, err := datastore.GetKeyValue(debianUpdaterFlag)
if err != nil { if err != nil {
return resp, err return resp, err
} }
@ -103,7 +106,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
err = json.NewDecoder(teedJSONReader).Decode(&data) err = json.NewDecoder(teedJSONReader).Decode(&data)
if err != nil { if err != nil {
log.Errorf("could not unmarshal Debian's JSON: %s", err) log.Errorf("could not unmarshal Debian's JSON: %s", err)
return resp, ErrCouldNotParse return resp, cerrors.ErrCouldNotParse
} }
// Calculate the hash and skip updating if the hash has been seen before. // Calculate the hash and skip updating if the hash has been seen before.
@ -115,7 +118,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
// Extract vulnerability data from Debian's JSON schema. // Extract vulnerability data from Debian's JSON schema.
var unknownReleases map[string]struct{} var unknownReleases map[string]struct{}
resp.Vulnerabilities, resp.Packages, unknownReleases = parseDebianJSON(&data) resp.Vulnerabilities, unknownReleases = parseDebianJSON(&data)
// Log unknown releases // Log unknown releases
for k := range unknownReleases { for k := range unknownReleases {
@ -127,7 +130,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
return resp, nil return resp, nil
} }
func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) { func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, unknownReleases map[string]struct{}) {
mvulnerabilities := make(map[string]*database.Vulnerability) mvulnerabilities := make(map[string]*database.Vulnerability)
unknownReleases = make(map[string]struct{}) unknownReleases = make(map[string]struct{})
@ -140,18 +143,21 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability,
continue continue
} }
// Skip if the release is not affected. // Skip if the status is not determined.
if releaseNode.FixedVersion == "0" || releaseNode.Status == "undetermined" { if releaseNode.Status == "undetermined" {
continue continue
} }
// Get or create the vulnerability. // Get or create the vulnerability.
vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName] namespaceName := "debian:" + database.DebianReleasesMapping[releaseName]
index := namespaceName + ":" + vulnName
vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[index]
if !vulnerabilityAlreadyExists { if !vulnerabilityAlreadyExists {
vulnerability = &database.Vulnerability{ vulnerability = &database.Vulnerability{
ID: vulnName, Name: vulnName,
Namespace: database.Namespace{Name: namespaceName},
Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""),
Priority: types.Unknown, Severity: types.Unknown,
Description: vulnNode.Description, Description: vulnNode.Description,
} }
} }
@ -159,15 +165,18 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability,
// Set the priority of the vulnerability. // Set the priority of the vulnerability.
// In the JSON, a vulnerability has one urgency per package it affects. // In the JSON, a vulnerability has one urgency per package it affects.
// The highest urgency should be the one set. // The highest urgency should be the one set.
urgency := urgencyToPriority(releaseNode.Urgency) urgency := urgencyToSeverity(releaseNode.Urgency)
if urgency.Compare(vulnerability.Priority) > 0 { if urgency.Compare(vulnerability.Severity) > 0 {
vulnerability.Priority = urgency vulnerability.Severity = urgency
} }
// Determine the version of the package the vulnerability affects. // Determine the version of the package the vulnerability affects.
var version types.Version var version types.Version
var err error var err error
if releaseNode.Status == "open" { if releaseNode.FixedVersion == "0" {
// This means that the package is not affected by this vulnerability.
version = types.MinVersion
} else if releaseNode.Status == "open" {
// Open means that the package is currently vulnerable in the latest // Open means that the package is currently vulnerable in the latest
// version of this Debian release. // version of this Debian release.
version = types.MaxVersion version = types.MaxVersion
@ -181,30 +190,28 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability,
} }
} }
// Create and add the package. // Create and add the feature version.
pkg := &database.Package{ pkg := database.FeatureVersion{
OS: "debian:" + database.DebianReleasesMapping[releaseName], Feature: database.Feature{Name: pkgName},
Name: pkgName,
Version: version, Version: version,
} }
vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode()) vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
packages = append(packages, pkg)
// Store the vulnerability. // Store the vulnerability.
mvulnerabilities[vulnName] = vulnerability mvulnerabilities[index] = vulnerability
} }
} }
} }
// Convert the vulnerabilities map to a slice // Convert the vulnerabilities map to a slice
for _, v := range mvulnerabilities { for _, v := range mvulnerabilities {
vulnerabilities = append(vulnerabilities, v) vulnerabilities = append(vulnerabilities, *v)
} }
return return
} }
func urgencyToPriority(urgency string) types.Priority { func urgencyToSeverity(urgency string) types.Priority {
switch urgency { switch urgency {
case "not yet assigned": case "not yet assigned":
return types.Unknown return types.Unknown

4
updater/fetchers/debian/debian_test.go
View File

@ -35,7 +35,7 @@ func TestDebianParser(t *testing.T) {
for _, vulnerability := range response.Vulnerabilities { for _, vulnerability := range response.Vulnerabilities {
if vulnerability.ID == "CVE-2015-1323" { if vulnerability.ID == "CVE-2015-1323" {
assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2015-1323", vulnerability.Link) assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2015-1323", vulnerability.Link)
assert.Equal(t, types.Low, vulnerability.Priority) assert.Equal(t, types.Low, vulnerability.Severity)
assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description)
expectedPackages := []*database.Package{ expectedPackages := []*database.Package{
@ -57,7 +57,7 @@ func TestDebianParser(t *testing.T) {
} }
} else if vulnerability.ID == "CVE-2003-0779" { } else if vulnerability.ID == "CVE-2003-0779" {
assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link) assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link)
assert.Equal(t, types.High, vulnerability.Priority) assert.Equal(t, types.High, vulnerability.Severity)
assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description)
expectedPackages := []*database.Package{ expectedPackages := []*database.Package{

11
updater/fetchers/rhel/rhel.go
View File

@ -27,6 +27,7 @@ import (
"github.com/coreos/clair/updater" "github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors" cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types" "github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
) )
const ( const (
@ -48,6 +49,8 @@ var (
} }
rhsaRegexp = regexp.MustCompile(`com.redhat.rhsa-(\d+).xml`) rhsaRegexp = regexp.MustCompile(`com.redhat.rhsa-(\d+).xml`)
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/rhel")
) )
type oval struct { type oval struct {
@ -85,11 +88,11 @@ func init() {
} }
// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions. // FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions.
func (f *RHELFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
log.Info("fetching Red Hat vulnerabilities") log.Info("fetching Red Hat vulnerabilities")
// Get the first RHSA we have to manage. // Get the first RHSA we have to manage.
flagValue, err := database.GetFlagValue(rhelUpdaterFlag) flagValue, err := datastore.GetKeyValue(rhelUpdaterFlag)
if err != nil { if err != nil {
return resp, err return resp, err
} }
@ -155,7 +158,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []*database.Vulnerability,
err = xml.NewDecoder(ovalReader).Decode(&ov) err = xml.NewDecoder(ovalReader).Decode(&ov)
if err != nil { if err != nil {
log.Errorf("could not decode RHEL's XML: %s.", err) log.Errorf("could not decode RHEL's XML: %s.", err)
err = ErrCouldNotParse err = cerrors.ErrCouldNotParse
return return
} }
@ -332,7 +335,7 @@ func link(def definition) (link string) {
return return
} }
func priority(def definition) types.Priority { func priority(def definition) types.Severity {
// Parse the priority. // Parse the priority.
priority := strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) priority := strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1])

4
updater/fetchers/rhel/rhel_test.go
View File

@ -35,7 +35,7 @@ func TestRHELParser(t *testing.T) {
if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) {
assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].ID) assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].ID)
assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link)
assert.Equal(t, types.Medium, vulnerabilities[0].Priority) assert.Equal(t, types.Medium, vulnerabilities[0].Severity)
assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description)
expectedPackages := []*database.Package{ expectedPackages := []*database.Package{
@ -68,7 +68,7 @@ func TestRHELParser(t *testing.T) {
if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) {
assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].ID) assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].ID)
assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link)
assert.Equal(t, types.Critical, vulnerabilities[0].Priority) assert.Equal(t, types.Critical, vulnerabilities[0].Severity)
assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description)
expectedPackages := []*database.Package{ expectedPackages := []*database.Package{

15
updater/fetchers/ubuntu/ubuntu.go
View File

@ -30,6 +30,7 @@ import (
"github.com/coreos/clair/utils" "github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors" cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types" "github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
) )
const ( const (
@ -69,6 +70,8 @@ var (
affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`) affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames() affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu")
) )
// UbuntuFetcher implements updater.Fetcher and get vulnerability updates from // UbuntuFetcher implements updater.Fetcher and get vulnerability updates from
@ -80,7 +83,7 @@ func init() {
} }
// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker. // FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
func (fetcher *UbuntuFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
log.Info("fetching Ubuntu vulnerabilities") log.Info("fetching Ubuntu vulnerabilities")
// Check to see if the repository does not already exist. // Check to see if the repository does not already exist.
@ -114,7 +117,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate() (resp updater.FetcherResponse, err e
} }
// Get the latest revision number we successfully applied in the database. // Get the latest revision number we successfully applied in the database.
dbRevisionNumber, err := database.GetFlagValue("ubuntuUpdater") dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater")
if err != nil { if err != nil {
return resp, err return resp, err
} }
@ -305,7 +308,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerabilit
priority = priority[:strings.Index(priority, " ")] priority = priority[:strings.Index(priority, " ")]
} }
vulnerability.Priority = ubuntuPriorityToPriority(priority) vulnerability.Severity = ubuntuPriorityToPriority(priority)
continue continue
} }
@ -387,14 +390,14 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerabilit
} }
// If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown // If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown
if vulnerability.Priority == "" { if vulnerability.Severity == "" {
vulnerability.Priority = types.Unknown vulnerability.Severity = types.Unknown
} }
return return
} }
func ubuntuPriorityToPriority(priority string) types.Priority { func ubuntuPriorityToPriority(priority string) types.Severity {
switch priority { switch priority {
case "untriaged": case "untriaged":
return types.Unknown return types.Unknown

2
updater/fetchers/ubuntu/ubuntu_test.go
View File

@ -35,7 +35,7 @@ func TestUbuntuParser(t *testing.T) {
vulnerability, packages, unknownReleases, err := parseUbuntuCVE(testData) vulnerability, packages, unknownReleases, err := parseUbuntuCVE(testData)
if assert.Nil(t, err) { if assert.Nil(t, err) {
assert.Equal(t, "CVE-2015-4471", vulnerability.ID) assert.Equal(t, "CVE-2015-4471", vulnerability.ID)
assert.Equal(t, types.Medium, vulnerability.Priority) assert.Equal(t, types.Medium, vulnerability.Severity)
assert.Equal(t, "Off-by-one error in the lzxd_decompress function in lzxd.c in libmspack before 0.5 allows remote attackers to cause a denial of service (buffer under-read and application crash) via a crafted CAB archive.", vulnerability.Description) assert.Equal(t, "Off-by-one error in the lzxd_decompress function in lzxd.c in libmspack before 0.5 allows remote attackers to cause a denial of service (buffer under-read and application crash) via a crafted CAB archive.", vulnerability.Description)
// Unknown release (line 28) // Unknown release (line 28)

170
updater/updater.go
View File

@ -18,7 +18,6 @@ package updater
import ( import (
"encoding/json" "encoding/json"
"fmt"
"math/rand" "math/rand"
"strconv" "strconv"
"time" "time"
@ -32,16 +31,18 @@ import (
) )
const ( const (
flagName = "updater" flagName = "updater/last"
notesFlagName = "updater/notes" notesFlagName = "updater/notes"
refreshLockDuration = time.Minute * 8
lockName = "updater"
lockDuration = refreshLockDuration + time.Minute*2 lockDuration = refreshLockDuration + time.Minute*2
refreshLockDuration = time.Minute * 8
) )
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater") var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater")
// Run updates the vulnerability database at regular intervals. // Run updates the vulnerability database at regular intervals.
func Run(config *config.UpdaterConfig, st *utils.Stopper) { func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) {
defer st.End() defer st.End()
// Do not run the updater if there is no config or if the interval is 0. // Do not run the updater if there is no config or if the interval is 0.
@ -62,7 +63,7 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) {
// occurs. // occurs.
var nextUpdate time.Time var nextUpdate time.Time
var stop bool var stop bool
if lastUpdate := getLastUpdate(); !lastUpdate.IsZero() { if lastUpdate := getLastUpdate(datastore); !lastUpdate.IsZero() {
nextUpdate = lastUpdate.Add(config.Interval) nextUpdate = lastUpdate.Add(config.Interval)
} else { } else {
nextUpdate = time.Now().UTC() nextUpdate = time.Now().UTC()
@ -72,12 +73,12 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) {
if nextUpdate.Before(time.Now().UTC()) { if nextUpdate.Before(time.Now().UTC()) {
// Attempt to get a lock on the the update. // Attempt to get a lock on the the update.
log.Debug("attempting to obtain update lock") log.Debug("attempting to obtain update lock")
hasLock, hasLockUntil := database.Lock(flagName, lockDuration, whoAmI) hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false)
if hasLock { if hasLock {
// Launch update in a new go routine. // Launch update in a new go routine.
doneC := make(chan bool, 1) doneC := make(chan bool, 1)
go func() { go func() {
Update() Update(datastore)
doneC <- true doneC <- true
}() }()
@ -87,21 +88,21 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) {
done = true done = true
case <-time.After(refreshLockDuration): case <-time.After(refreshLockDuration):
// Refresh the lock until the update is done. // Refresh the lock until the update is done.
database.Lock(flagName, lockDuration, whoAmI) datastore.Lock(lockName, whoAmI, lockDuration, true)
case <-st.Chan(): case <-st.Chan():
stop = true stop = true
} }
} }
// Unlock the update. // Unlock the update.
database.Unlock(flagName, whoAmI) datastore.Unlock(lockName, whoAmI)
if stop { if stop {
break break
} }
continue continue
} else { } else {
lockOwner, lockExpiration, err := database.FindLock(flagName) lockOwner, lockExpiration, err := datastore.FindLock(lockName)
if err != nil { if err != nil {
log.Debug("update lock is already taken") log.Debug("update lock is already taken")
nextUpdate = hasLockUntil nextUpdate = hasLockUntil
@ -128,69 +129,50 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) {
// Update fetches all the vulnerabilities from the registered fetchers, upserts // Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications. // them into the database and then sends notifications.
func Update() { func Update(datastore database.Datastore) {
log.Info("updating vulnerabilities") log.Info("updating vulnerabilities")
// Fetch updates. // Fetch updates.
status, responses := fetch() status, vulnerabilities, flags, notes := fetch(datastore)
// Merge responses.
vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses)
if err != nil {
log.Errorf("an error occured when merging update responses: %s", err)
return
}
responses = nil
// TODO(Quentin-M): Complete informations using NVD // TODO(Quentin-M): Complete informations using NVD
// Insert packages.
log.Tracef("beginning insertion of %d packages for update", len(packages))
err = database.InsertPackages(packages)
if err != nil {
log.Errorf("an error occured when inserting packages for update: %s", err)
return
}
packages = nil
// Insert vulnerabilities. // Insert vulnerabilities.
log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities))
notifications, err := database.InsertVulnerabilities(vulnerabilities) err := datastore.InsertVulnerabilities(vulnerabilities)
if err != nil { if err != nil {
log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
return return
} }
vulnerabilities = nil vulnerabilities = nil
// Insert notifications into the database.
err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper())
if err != nil {
log.Errorf("an error occured when inserting notifications for update: %s", err)
return
}
notifications = nil
// Update flags and notes. // Update flags and notes.
for flagName, flagValue := range flags { for flagName, flagValue := range flags {
database.UpdateFlag(flagName, flagValue) datastore.InsertKeyValue(flagName, flagValue)
} }
database.UpdateFlag(notesFlagName, notes)
bnotes, _ := json.Marshal(notes)
datastore.InsertKeyValue(notesFlagName, string(bnotes))
// Update last successful update if every fetchers worked properly. // Update last successful update if every fetchers worked properly.
if status { if status {
database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) datastore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10))
} }
log.Info("update finished") log.Info("update finished")
} }
// fetch get data from the registered fetchers, in parallel. // fetch get data from the registered fetchers, in parallel.
func fetch() (status bool, responses []*FetcherResponse) { func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[string]string, []string) {
var vulnerabilities []database.Vulnerability
var notes []string
status := true
flags := make(map[string]string)
// Fetch updates in parallel. // Fetch updates in parallel.
status = true
var responseC = make(chan *FetcherResponse, 0) var responseC = make(chan *FetcherResponse, 0)
for n, f := range fetchers { for n, f := range fetchers {
go func(name string, fetcher Fetcher) { go func(name string, fetcher Fetcher) {
response, err := fetcher.FetchUpdate() response, err := fetcher.FetchUpdate(datastore)
if err != nil { if err != nil {
log.Errorf("an error occured when fetching update '%s': %s.", name, err) log.Errorf("an error occured when fetching update '%s': %s.", name, err)
status = false status = false
@ -206,93 +188,21 @@ func fetch() (status bool, responses []*FetcherResponse) {
for i := 0; i < len(fetchers); i++ { for i := 0; i < len(fetchers); i++ {
resp := <-responseC resp := <-responseC
if resp != nil { if resp != nil {
responses = append(responses, resp) vulnerabilities = append(vulnerabilities, resp.Vulnerabilities...)
notes = append(notes, resp.Notes...)
if resp.FlagName != "" && resp.FlagValue != "" {
flags[resp.FlagName] = resp.FlagValue
}
} }
} }
close(responseC) close(responseC)
return return status, vulnerabilities, flags, notes
}
// merge put all the responses together (vulnerabilities, packages, flags, notes), ensure the
// uniqueness of vulnerabilities and packages and verify that every vulnerability's fixedInNodes
// have their corresponding package definition.
func mergeAndVerify(responses []*FetcherResponse) (svulnerabilities []*database.Vulnerability, spackages []*database.Package, flags map[string]string, snotes string, err error) {
vulnerabilities := make(map[string]*database.Vulnerability)
packages := make(map[string]*database.Package)
flags = make(map[string]string)
var notes []string
// Merge responses.
for _, response := range responses {
// Notes
notes = append(notes, response.Notes...)
// Flags
if response.FlagName != "" && response.FlagValue != "" {
flags[response.FlagName] = response.FlagValue
}
// Packages
for _, p := range response.Packages {
node := p.GetNode()
if _, ok := packages[node]; !ok {
packages[node] = p
}
}
// Vulnerabilities
for _, v := range response.Vulnerabilities {
if vulnerability, ok := vulnerabilities[v.ID]; !ok {
vulnerabilities[v.ID] = v
} else {
mergeVulnerability(vulnerability, v)
}
}
}
// Verify that the packages used in the vulnerabilities are specified.
for _, v := range vulnerabilities {
for _, node := range v.FixedInNodes {
if _, ok := packages[node]; !ok {
err = fmt.Errorf("vulnerability %s is fixed by an unspecified package", v.ID)
return
}
}
}
// Convert data and return
for _, v := range vulnerabilities {
svulnerabilities = append(svulnerabilities, v)
}
for _, p := range packages {
spackages = append(spackages, p)
}
bnotes, _ := json.Marshal(notes)
snotes = string(bnotes)
return
}
// mergeVulnerability updates the target vulnerability structure using the specified one.
func mergeVulnerability(target, source *database.Vulnerability) {
if source.Link != "" {
target.Link = source.Link
}
if source.Description != "" {
target.Description = source.Description
}
if source.Priority.Compare(target.Priority) > 0 {
target.Priority = source.Priority
}
for _, node := range source.FixedInNodes {
if !utils.Contains(node, target.FixedInNodes) {
target.FixedInNodes = append(target.FixedInNodes, node)
}
}
} }
// Healthcheck returns the health of the updater service. // Healthcheck returns the health of the updater service.
func Healthcheck() health.Status { func Healthcheck(datastore database.Datastore) health.Status {
notes := getNotes() notes := getNotes(datastore)
return health.Status{ return health.Status{
IsEssential: false, IsEssential: false,
@ -301,14 +211,14 @@ func Healthcheck() health.Status {
LatestSuccessfulUpdate time.Time LatestSuccessfulUpdate time.Time
Notes []string `json:",omitempty"` Notes []string `json:",omitempty"`
}{ }{
LatestSuccessfulUpdate: getLastUpdate(), LatestSuccessfulUpdate: getLastUpdate(datastore),
Notes: notes, Notes: notes,
}, },
} }
} }
func getLastUpdate() time.Time { func getLastUpdate(datastore database.Datastore) time.Time {
if lastUpdateTSS, err := database.GetFlagValue(flagName); err == nil && lastUpdateTSS != "" { if lastUpdateTSS, err := datastore.GetKeyValue(flagName); err == nil && lastUpdateTSS != "" {
if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil { if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil {
return time.Unix(lastUpdateTS, 0).UTC() return time.Unix(lastUpdateTS, 0).UTC()
} }
@ -316,8 +226,8 @@ func getLastUpdate() time.Time {
return time.Time{} return time.Time{}
} }
func getNotes() (notes []string) { func getNotes(datastore database.Datastore) (notes []string) {
if jsonNotes, err := database.GetFlagValue(notesFlagName); err == nil && jsonNotes != "" { if jsonNotes, err := datastore.GetKeyValue(notesFlagName); err == nil && jsonNotes != "" {
json.Unmarshal([]byte(jsonNotes), notes) json.Unmarshal([]byte(jsonNotes), notes)
} }
return return

5
utils/errors/errors.go
View File

@ -20,10 +20,15 @@ import "errors"
var ( var (
// ErrFilesystem occurs when a filesystem interaction fails. // ErrFilesystem occurs when a filesystem interaction fails.
ErrFilesystem = errors.New("something went wrong when interacting with the fs") ErrFilesystem = errors.New("something went wrong when interacting with the fs")
// ErrCouldNotDownload occurs when a download fails. // ErrCouldNotDownload occurs when a download fails.
ErrCouldNotDownload = errors.New("could not download requested ressource") ErrCouldNotDownload = errors.New("could not download requested ressource")
// ErrNotFound occurs when a resource could not be found. // ErrNotFound occurs when a resource could not be found.
ErrNotFound = errors.New("the resource cannot be found") ErrNotFound = errors.New("the resource cannot be found")
// ErrCouldNotParse is returned when a fetcher fails to parse the update data.
ErrCouldNotParse = errors.New("updater/fetchers: could not parse")
) )
// ErrBadRequest occurs when a method has been passed an inappropriate argument. // ErrBadRequest occurs when a method has been passed an inappropriate argument.