diff --git a/api/handlers.go b/api/handlers.go index ae927703..f467260b 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -46,8 +46,8 @@ func GETVersions(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _ } // GETHealth sums up the health of all the registered services. -func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _ *Env) { - globalHealth, statuses := health.Healthcheck() +func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, e *Env) { + globalHealth, statuses := health.Healthcheck(e.Datastore) httpStatus := http.StatusOK if !globalHealth { diff --git a/clair.go b/clair.go index 1cc4bc4c..fd0a473f 100644 --- a/clair.go +++ b/clair.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/clair/api" "github.com/coreos/clair/config" "github.com/coreos/clair/database/pgsql" + "github.com/coreos/clair/updater" "github.com/coreos/clair/utils" "github.com/coreos/pkg/capnslog" ) @@ -55,8 +56,8 @@ func Boot(config *config.Config) { go api.RunHealth(config.API, &api.Env{Datastore: db}, st) // Start updater - // st.Begin() - // go updater.Run(config.Updater, st) + st.Begin() + go updater.Run(config.Updater, db, st) // Wait for interruption and shutdown gracefully. waitForSignals(os.Interrupt) diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index 8414540b..4261fcf1 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -146,7 +146,6 @@ func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.Fea if featureVersion.Version.Compare(fixedInVersion) < 0 { // The version of the FeatureVersion we are inserting is lower than the fixed version on this // Vulnerability, thus, this FeatureVersion is affected by it. - // TODO(Quentin-M): Prepare. _, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID, featureVersion.ID, fixedInID) if err != nil { diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index d0ff887e..bfea2a8e 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -99,7 +99,7 @@ func createDatabase(dataSource, databaseName string) error { defer db.Close() // Create database. - _, err = db.Exec("CREATE DATABASE " + databaseName + ";") + _, err = db.Exec("CREATE DATABASE " + databaseName) if err != nil { return fmt.Errorf("could not create database: %v", err) } @@ -118,7 +118,7 @@ func dropDatabase(dataSource, databaseName string) error { defer db.Close() // Drop database. - _, err = db.Exec("DROP DATABASE " + databaseName + ";") + _, err = db.Exec("DROP DATABASE " + databaseName) if err != nil { return fmt.Errorf("could not drop database: %v", err) } @@ -167,7 +167,7 @@ func OpenForTest(name string, withTestData bool) (*pgSQLTest, error) { d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/testdata/data.sql") _, err = db.(*pgSQL).Exec(string(d)) if err != nil { - dropDatabase(dataSource, dbName) + dropDatabase(dataSource+"dbname=postgres", dbName) log.Error(err) return nil, database.ErrCantOpen } diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 4588bfed..919ed7e5 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -2,6 +2,7 @@ package pgsql import ( "database/sql" + "fmt" "github.com/coreos/clair/database" "github.com/coreos/clair/utils" @@ -69,6 +70,7 @@ func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerabili for _, vulnerability := range vulnerabilities { err := pgSQL.insertVulnerability(vulnerability) if err != nil { + fmt.Printf("%#v\n", vulnerability) return err } } @@ -121,16 +123,18 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er return nil } - // Insert or find the new FeatureVersions. + // Insert or find the new Feature. // We already have the Feature IDs in updatedFixedInFeatureVersions because diffFixedIn fills them // in using the existing vulnerability's FixedIn FeatureVersions. Note that even if FixedIn // is type FeatureVersion, the actual stored ID in these structs are the Feature IDs. // // Also, we enforce the namespace of the FeatureVersion in case it was empty. There is a test // above to ensure that the passed Namespace is either the same as the vulnerability or empty. + // + // TODO(Quentin-M): Batch me. for i := 0; i < len(newFixedInFeatureVersions); i++ { newFixedInFeatureVersions[i].Feature.Namespace.Name = vulnerability.Namespace.Name - newFixedInFeatureVersions[i].ID, err = pgSQL.insertFeatureVersion(newFixedInFeatureVersions[i]) + newFixedInFeatureVersions[i].Feature.ID, err = pgSQL.insertFeature(newFixedInFeatureVersions[i].Feature) if err != nil { return err } @@ -229,15 +233,23 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability var fixedInID int for _, fv := range newFixedInFeatureVersions { + if fv.Version == types.MinVersion { + // We don't want to mark a Feature as fixed in MinVersion. MinVersion only makes sense when a + // Feature is already marked as fixed in some version, in which case we would be in the + // "updatedFixedInFeatureVersions" loop and removes the fixed in mark. + continue + } + // Insert Vulnerability_FixedIn_Feature. - err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID, + err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.Feature.ID, &fv.Version).Scan(&fixedInID) if err != nil { return handleError("i_vulnerability_fixedin_feature", err) } // Insert Vulnerability_Affects_FeatureVersion. - err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version) + err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID, + fv.Version) if err != nil { return err } @@ -246,8 +258,8 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability for _, fv := range updatedFixedInFeatureVersions { if fv.Version != types.MinVersion { // Update Vulnerability_FixedIn_Feature. - err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID, - &fv.Version).Scan(&fixedInID) + err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID, + fv.Feature.ID, &fv.Version).Scan(&fixedInID) if err != nil { return handleError("u_vulnerability_fixedin_feature", err) } @@ -259,7 +271,8 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability } // Insert Vulnerability_Affects_FeatureVersion. - err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version) + err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID, + fv.Version) if err != nil { return err } @@ -267,15 +280,17 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability // Updating FixedIn by saying that the fixed version is the lowest possible version, it // basically means that the vulnerability doesn't affect the feature (anymore). // Drop it from Vulnerability_FixedIn_Feature and Vulnerability_Affects_FeatureVersion. - err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID). - Scan(&fixedInID) - if err != nil { + err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID, + fv.Feature.ID).Scan(&fixedInID) + if err != nil && err != sql.ErrNoRows { return handleError("r_vulnerability_fixedin_feature", err) } - _, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID) - if err != nil { - return handleError("r_vulnerability_affects_featureversion", err) + if err == nil { + _, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID) + if err != nil { + return handleError("r_vulnerability_affects_featureversion", err) + } } } } @@ -304,7 +319,7 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID, } if featureVersionVersion.Compare(fixedInVersion) < 0 { - _, err := tx.Exec("i_vulnerability_affects_featureversion", vulnerabilityID, featureVersionID, + _, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID, featureVersionID, fixedInID) if err != nil { return handleError("i_vulnerability_affects_featureversion", err) diff --git a/health/health.go b/health/health.go index 0ec34725..2dbfbefd 100644 --- a/health/health.go +++ b/health/health.go @@ -19,6 +19,8 @@ package health import ( "fmt" "sync" + + "github.com/coreos/clair/database" ) // Status defines a way to know the health status of a service @@ -33,7 +35,7 @@ type Status struct { } // A Healthchecker function is a method returning the Status of the tested service -type Healthchecker func() Status +type Healthchecker func(database.Datastore) Status var ( healthcheckersLock sync.Mutex @@ -59,12 +61,12 @@ func RegisterHealthchecker(name string, f Healthchecker) { } // Healthcheck calls every registered Healthchecker and summarize their output -func Healthcheck() (bool, map[string]interface{}) { +func Healthcheck(datastore database.Datastore) (bool, map[string]interface{}) { globalHealth := true statuses := make(map[string]interface{}) for serviceName, serviceChecker := range healthcheckers { - status := serviceChecker() + status := serviceChecker(datastore) globalHealth = globalHealth && (!status.IsEssential || status.IsHealthy) statuses[serviceName] = struct { diff --git a/updater/fetchers.go b/updater/fetchers.go index d6295628..af09eeb9 100644 --- a/updater/fetchers.go +++ b/updater/fetchers.go @@ -20,7 +20,7 @@ var fetchers = make(map[string]Fetcher) // Fetcher represents anything that can fetch vulnerabilities. type Fetcher interface { - FetchUpdate() (FetcherResponse, error) + FetchUpdate(database.Datastore) (FetcherResponse, error) } // FetcherResponse represents the sum of results of an update. @@ -28,8 +28,7 @@ type FetcherResponse struct { FlagName string FlagValue string Notes []string - Vulnerabilities []*database.Vulnerability - Packages []*database.Package + Vulnerabilities []database.Vulnerability } // RegisterFetcher makes a Fetcher available by the provided name. diff --git a/updater/fetchers/debian/debian.go b/updater/fetchers/debian/debian.go index abdf2be6..a8e6fb9e 100644 --- a/updater/fetchers/debian/debian.go +++ b/updater/fetchers/debian/debian.go @@ -27,6 +27,7 @@ import ( "github.com/coreos/clair/updater" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" + "github.com/coreos/pkg/capnslog" ) const ( @@ -35,6 +36,8 @@ const ( debianUpdaterFlag = "debianUpdater" ) +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian") + type jsonData map[string]map[string]jsonVuln type jsonVuln struct { @@ -57,7 +60,7 @@ func init() { } // FetchUpdate fetches vulnerability updates from the Debian Security Tracker. -func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { +func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { log.Info("fetching Debian vulnerabilities") // Download JSON. @@ -68,7 +71,7 @@ func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err e } // Get the SHA-1 of the latest update's JSON data - latestHash, err := database.GetFlagValue(debianUpdaterFlag) + latestHash, err := datastore.GetKeyValue(debianUpdaterFlag) if err != nil { return resp, err } @@ -103,7 +106,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F err = json.NewDecoder(teedJSONReader).Decode(&data) if err != nil { log.Errorf("could not unmarshal Debian's JSON: %s", err) - return resp, ErrCouldNotParse + return resp, cerrors.ErrCouldNotParse } // Calculate the hash and skip updating if the hash has been seen before. @@ -115,7 +118,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F // Extract vulnerability data from Debian's JSON schema. var unknownReleases map[string]struct{} - resp.Vulnerabilities, resp.Packages, unknownReleases = parseDebianJSON(&data) + resp.Vulnerabilities, unknownReleases = parseDebianJSON(&data) // Log unknown releases for k := range unknownReleases { @@ -127,7 +130,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F return resp, nil } -func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) { +func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, unknownReleases map[string]struct{}) { mvulnerabilities := make(map[string]*database.Vulnerability) unknownReleases = make(map[string]struct{}) @@ -140,18 +143,21 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, continue } - // Skip if the release is not affected. - if releaseNode.FixedVersion == "0" || releaseNode.Status == "undetermined" { + // Skip if the status is not determined. + if releaseNode.Status == "undetermined" { continue } // Get or create the vulnerability. - vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName] + namespaceName := "debian:" + database.DebianReleasesMapping[releaseName] + index := namespaceName + ":" + vulnName + vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[index] if !vulnerabilityAlreadyExists { vulnerability = &database.Vulnerability{ - ID: vulnName, + Name: vulnName, + Namespace: database.Namespace{Name: namespaceName}, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), - Priority: types.Unknown, + Severity: types.Unknown, Description: vulnNode.Description, } } @@ -159,15 +165,18 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, // Set the priority of the vulnerability. // In the JSON, a vulnerability has one urgency per package it affects. // The highest urgency should be the one set. - urgency := urgencyToPriority(releaseNode.Urgency) - if urgency.Compare(vulnerability.Priority) > 0 { - vulnerability.Priority = urgency + urgency := urgencyToSeverity(releaseNode.Urgency) + if urgency.Compare(vulnerability.Severity) > 0 { + vulnerability.Severity = urgency } // Determine the version of the package the vulnerability affects. var version types.Version var err error - if releaseNode.Status == "open" { + if releaseNode.FixedVersion == "0" { + // This means that the package is not affected by this vulnerability. + version = types.MinVersion + } else if releaseNode.Status == "open" { // Open means that the package is currently vulnerable in the latest // version of this Debian release. version = types.MaxVersion @@ -181,30 +190,28 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, } } - // Create and add the package. - pkg := &database.Package{ - OS: "debian:" + database.DebianReleasesMapping[releaseName], - Name: pkgName, + // Create and add the feature version. + pkg := database.FeatureVersion{ + Feature: database.Feature{Name: pkgName}, Version: version, } - vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode()) - packages = append(packages, pkg) + vulnerability.FixedIn = append(vulnerability.FixedIn, pkg) // Store the vulnerability. - mvulnerabilities[vulnName] = vulnerability + mvulnerabilities[index] = vulnerability } } } // Convert the vulnerabilities map to a slice for _, v := range mvulnerabilities { - vulnerabilities = append(vulnerabilities, v) + vulnerabilities = append(vulnerabilities, *v) } return } -func urgencyToPriority(urgency string) types.Priority { +func urgencyToSeverity(urgency string) types.Priority { switch urgency { case "not yet assigned": return types.Unknown diff --git a/updater/fetchers/debian/debian_test.go b/updater/fetchers/debian/debian_test.go index 6d0e5639..47e81ba4 100644 --- a/updater/fetchers/debian/debian_test.go +++ b/updater/fetchers/debian/debian_test.go @@ -35,7 +35,7 @@ func TestDebianParser(t *testing.T) { for _, vulnerability := range response.Vulnerabilities { if vulnerability.ID == "CVE-2015-1323" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2015-1323", vulnerability.Link) - assert.Equal(t, types.Low, vulnerability.Priority) + assert.Equal(t, types.Low, vulnerability.Severity) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) expectedPackages := []*database.Package{ @@ -57,7 +57,7 @@ func TestDebianParser(t *testing.T) { } } else if vulnerability.ID == "CVE-2003-0779" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link) - assert.Equal(t, types.High, vulnerability.Priority) + assert.Equal(t, types.High, vulnerability.Severity) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) expectedPackages := []*database.Package{ diff --git a/updater/fetchers/rhel/rhel.go b/updater/fetchers/rhel/rhel.go index e05955d9..ee80ed2d 100644 --- a/updater/fetchers/rhel/rhel.go +++ b/updater/fetchers/rhel/rhel.go @@ -27,6 +27,7 @@ import ( "github.com/coreos/clair/updater" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" + "github.com/coreos/pkg/capnslog" ) const ( @@ -48,6 +49,8 @@ var ( } rhsaRegexp = regexp.MustCompile(`com.redhat.rhsa-(\d+).xml`) + + log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/rhel") ) type oval struct { @@ -85,11 +88,11 @@ func init() { } // FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions. -func (f *RHELFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { +func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { log.Info("fetching Red Hat vulnerabilities") // Get the first RHSA we have to manage. - flagValue, err := database.GetFlagValue(rhelUpdaterFlag) + flagValue, err := datastore.GetKeyValue(rhelUpdaterFlag) if err != nil { return resp, err } @@ -155,7 +158,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []*database.Vulnerability, err = xml.NewDecoder(ovalReader).Decode(&ov) if err != nil { log.Errorf("could not decode RHEL's XML: %s.", err) - err = ErrCouldNotParse + err = cerrors.ErrCouldNotParse return } @@ -332,7 +335,7 @@ func link(def definition) (link string) { return } -func priority(def definition) types.Priority { +func priority(def definition) types.Severity { // Parse the priority. priority := strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) diff --git a/updater/fetchers/rhel/rhel_test.go b/updater/fetchers/rhel/rhel_test.go index b4184d3a..2c21a4de 100644 --- a/updater/fetchers/rhel/rhel_test.go +++ b/updater/fetchers/rhel/rhel_test.go @@ -35,7 +35,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].ID) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link) - assert.Equal(t, types.Medium, vulnerabilities[0].Priority) + assert.Equal(t, types.Medium, vulnerabilities[0].Severity) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) expectedPackages := []*database.Package{ @@ -68,7 +68,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].ID) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link) - assert.Equal(t, types.Critical, vulnerabilities[0].Priority) + assert.Equal(t, types.Critical, vulnerabilities[0].Severity) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) expectedPackages := []*database.Package{ diff --git a/updater/fetchers/ubuntu/ubuntu.go b/updater/fetchers/ubuntu/ubuntu.go index 5876bb52..e1fd4322 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/updater/fetchers/ubuntu/ubuntu.go @@ -30,6 +30,7 @@ import ( "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" + "github.com/coreos/pkg/capnslog" ) const ( @@ -69,6 +70,8 @@ var ( affectsCaptureRegexp = regexp.MustCompile(`(?P.*)_(?P.*): (?P[^\s]*)( \(+(?P[^()]*)\)+)?`) affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames() + + log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu") ) // UbuntuFetcher implements updater.Fetcher and get vulnerability updates from @@ -80,7 +83,7 @@ func init() { } // FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker. -func (fetcher *UbuntuFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { +func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { log.Info("fetching Ubuntu vulnerabilities") // Check to see if the repository does not already exist. @@ -114,7 +117,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate() (resp updater.FetcherResponse, err e } // Get the latest revision number we successfully applied in the database. - dbRevisionNumber, err := database.GetFlagValue("ubuntuUpdater") + dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater") if err != nil { return resp, err } @@ -305,7 +308,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerabilit priority = priority[:strings.Index(priority, " ")] } - vulnerability.Priority = ubuntuPriorityToPriority(priority) + vulnerability.Severity = ubuntuPriorityToPriority(priority) continue } @@ -387,14 +390,14 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerabilit } // If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown - if vulnerability.Priority == "" { - vulnerability.Priority = types.Unknown + if vulnerability.Severity == "" { + vulnerability.Severity = types.Unknown } return } -func ubuntuPriorityToPriority(priority string) types.Priority { +func ubuntuPriorityToPriority(priority string) types.Severity { switch priority { case "untriaged": return types.Unknown diff --git a/updater/fetchers/ubuntu/ubuntu_test.go b/updater/fetchers/ubuntu/ubuntu_test.go index 4827ed8f..9bfd379b 100644 --- a/updater/fetchers/ubuntu/ubuntu_test.go +++ b/updater/fetchers/ubuntu/ubuntu_test.go @@ -35,7 +35,7 @@ func TestUbuntuParser(t *testing.T) { vulnerability, packages, unknownReleases, err := parseUbuntuCVE(testData) if assert.Nil(t, err) { assert.Equal(t, "CVE-2015-4471", vulnerability.ID) - assert.Equal(t, types.Medium, vulnerability.Priority) + assert.Equal(t, types.Medium, vulnerability.Severity) assert.Equal(t, "Off-by-one error in the lzxd_decompress function in lzxd.c in libmspack before 0.5 allows remote attackers to cause a denial of service (buffer under-read and application crash) via a crafted CAB archive.", vulnerability.Description) // Unknown release (line 28) diff --git a/updater/updater.go b/updater/updater.go index 929e2e95..9da57f1e 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -18,7 +18,6 @@ package updater import ( "encoding/json" - "fmt" "math/rand" "strconv" "time" @@ -32,16 +31,18 @@ import ( ) const ( - flagName = "updater" - notesFlagName = "updater/notes" - refreshLockDuration = time.Minute * 8 + flagName = "updater/last" + notesFlagName = "updater/notes" + + lockName = "updater" lockDuration = refreshLockDuration + time.Minute*2 + refreshLockDuration = time.Minute * 8 ) var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater") // Run updates the vulnerability database at regular intervals. -func Run(config *config.UpdaterConfig, st *utils.Stopper) { +func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. @@ -62,7 +63,7 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) { // occurs. var nextUpdate time.Time var stop bool - if lastUpdate := getLastUpdate(); !lastUpdate.IsZero() { + if lastUpdate := getLastUpdate(datastore); !lastUpdate.IsZero() { nextUpdate = lastUpdate.Add(config.Interval) } else { nextUpdate = time.Now().UTC() @@ -72,12 +73,12 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) { if nextUpdate.Before(time.Now().UTC()) { // Attempt to get a lock on the the update. log.Debug("attempting to obtain update lock") - hasLock, hasLockUntil := database.Lock(flagName, lockDuration, whoAmI) + hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false) if hasLock { // Launch update in a new go routine. doneC := make(chan bool, 1) go func() { - Update() + Update(datastore) doneC <- true }() @@ -87,21 +88,21 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) { done = true case <-time.After(refreshLockDuration): // Refresh the lock until the update is done. - database.Lock(flagName, lockDuration, whoAmI) + datastore.Lock(lockName, whoAmI, lockDuration, true) case <-st.Chan(): stop = true } } // Unlock the update. - database.Unlock(flagName, whoAmI) + datastore.Unlock(lockName, whoAmI) if stop { break } continue } else { - lockOwner, lockExpiration, err := database.FindLock(flagName) + lockOwner, lockExpiration, err := datastore.FindLock(lockName) if err != nil { log.Debug("update lock is already taken") nextUpdate = hasLockUntil @@ -128,69 +129,50 @@ func Run(config *config.UpdaterConfig, st *utils.Stopper) { // Update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. -func Update() { +func Update(datastore database.Datastore) { log.Info("updating vulnerabilities") // Fetch updates. - status, responses := fetch() - - // Merge responses. - vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses) - if err != nil { - log.Errorf("an error occured when merging update responses: %s", err) - return - } - responses = nil + status, vulnerabilities, flags, notes := fetch(datastore) // TODO(Quentin-M): Complete informations using NVD - // Insert packages. - log.Tracef("beginning insertion of %d packages for update", len(packages)) - err = database.InsertPackages(packages) - if err != nil { - log.Errorf("an error occured when inserting packages for update: %s", err) - return - } - packages = nil - // Insert vulnerabilities. log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) - notifications, err := database.InsertVulnerabilities(vulnerabilities) + err := datastore.InsertVulnerabilities(vulnerabilities) if err != nil { log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) return } vulnerabilities = nil - // Insert notifications into the database. - err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) - if err != nil { - log.Errorf("an error occured when inserting notifications for update: %s", err) - return - } - notifications = nil - // Update flags and notes. for flagName, flagValue := range flags { - database.UpdateFlag(flagName, flagValue) + datastore.InsertKeyValue(flagName, flagValue) } - database.UpdateFlag(notesFlagName, notes) + + bnotes, _ := json.Marshal(notes) + datastore.InsertKeyValue(notesFlagName, string(bnotes)) // Update last successful update if every fetchers worked properly. if status { - database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) + datastore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) } log.Info("update finished") } // fetch get data from the registered fetchers, in parallel. -func fetch() (status bool, responses []*FetcherResponse) { +func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[string]string, []string) { + var vulnerabilities []database.Vulnerability + var notes []string + status := true + flags := make(map[string]string) + // Fetch updates in parallel. - status = true var responseC = make(chan *FetcherResponse, 0) for n, f := range fetchers { go func(name string, fetcher Fetcher) { - response, err := fetcher.FetchUpdate() + response, err := fetcher.FetchUpdate(datastore) if err != nil { log.Errorf("an error occured when fetching update '%s': %s.", name, err) status = false @@ -206,93 +188,21 @@ func fetch() (status bool, responses []*FetcherResponse) { for i := 0; i < len(fetchers); i++ { resp := <-responseC if resp != nil { - responses = append(responses, resp) - } - } - - close(responseC) - return -} - -// merge put all the responses together (vulnerabilities, packages, flags, notes), ensure the -// uniqueness of vulnerabilities and packages and verify that every vulnerability's fixedInNodes -// have their corresponding package definition. -func mergeAndVerify(responses []*FetcherResponse) (svulnerabilities []*database.Vulnerability, spackages []*database.Package, flags map[string]string, snotes string, err error) { - vulnerabilities := make(map[string]*database.Vulnerability) - packages := make(map[string]*database.Package) - flags = make(map[string]string) - var notes []string - - // Merge responses. - for _, response := range responses { - // Notes - notes = append(notes, response.Notes...) - // Flags - if response.FlagName != "" && response.FlagValue != "" { - flags[response.FlagName] = response.FlagValue - } - // Packages - for _, p := range response.Packages { - node := p.GetNode() - if _, ok := packages[node]; !ok { - packages[node] = p - } - } - // Vulnerabilities - for _, v := range response.Vulnerabilities { - if vulnerability, ok := vulnerabilities[v.ID]; !ok { - vulnerabilities[v.ID] = v - } else { - mergeVulnerability(vulnerability, v) + vulnerabilities = append(vulnerabilities, resp.Vulnerabilities...) + notes = append(notes, resp.Notes...) + if resp.FlagName != "" && resp.FlagValue != "" { + flags[resp.FlagName] = resp.FlagValue } } } - // Verify that the packages used in the vulnerabilities are specified. - for _, v := range vulnerabilities { - for _, node := range v.FixedInNodes { - if _, ok := packages[node]; !ok { - err = fmt.Errorf("vulnerability %s is fixed by an unspecified package", v.ID) - return - } - } - } - - // Convert data and return - for _, v := range vulnerabilities { - svulnerabilities = append(svulnerabilities, v) - } - for _, p := range packages { - spackages = append(spackages, p) - } - - bnotes, _ := json.Marshal(notes) - snotes = string(bnotes) - - return -} - -// mergeVulnerability updates the target vulnerability structure using the specified one. -func mergeVulnerability(target, source *database.Vulnerability) { - if source.Link != "" { - target.Link = source.Link - } - if source.Description != "" { - target.Description = source.Description - } - if source.Priority.Compare(target.Priority) > 0 { - target.Priority = source.Priority - } - for _, node := range source.FixedInNodes { - if !utils.Contains(node, target.FixedInNodes) { - target.FixedInNodes = append(target.FixedInNodes, node) - } - } + close(responseC) + return status, vulnerabilities, flags, notes } // Healthcheck returns the health of the updater service. -func Healthcheck() health.Status { - notes := getNotes() +func Healthcheck(datastore database.Datastore) health.Status { + notes := getNotes(datastore) return health.Status{ IsEssential: false, @@ -301,14 +211,14 @@ func Healthcheck() health.Status { LatestSuccessfulUpdate time.Time Notes []string `json:",omitempty"` }{ - LatestSuccessfulUpdate: getLastUpdate(), + LatestSuccessfulUpdate: getLastUpdate(datastore), Notes: notes, }, } } -func getLastUpdate() time.Time { - if lastUpdateTSS, err := database.GetFlagValue(flagName); err == nil && lastUpdateTSS != "" { +func getLastUpdate(datastore database.Datastore) time.Time { + if lastUpdateTSS, err := datastore.GetKeyValue(flagName); err == nil && lastUpdateTSS != "" { if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil { return time.Unix(lastUpdateTS, 0).UTC() } @@ -316,8 +226,8 @@ func getLastUpdate() time.Time { return time.Time{} } -func getNotes() (notes []string) { - if jsonNotes, err := database.GetFlagValue(notesFlagName); err == nil && jsonNotes != "" { +func getNotes(datastore database.Datastore) (notes []string) { + if jsonNotes, err := datastore.GetKeyValue(notesFlagName); err == nil && jsonNotes != "" { json.Unmarshal([]byte(jsonNotes), notes) } return diff --git a/utils/errors/errors.go b/utils/errors/errors.go index 2d391be4..78dc4f1a 100644 --- a/utils/errors/errors.go +++ b/utils/errors/errors.go @@ -20,10 +20,15 @@ import "errors" var ( // ErrFilesystem occurs when a filesystem interaction fails. ErrFilesystem = errors.New("something went wrong when interacting with the fs") + // ErrCouldNotDownload occurs when a download fails. ErrCouldNotDownload = errors.New("could not download requested ressource") + // ErrNotFound occurs when a resource could not be found. ErrNotFound = errors.New("the resource cannot be found") + + // ErrCouldNotParse is returned when a fetcher fails to parse the update data. + ErrCouldNotParse = errors.New("updater/fetchers: could not parse") ) // ErrBadRequest occurs when a method has been passed an inappropriate argument.