From a7b683d4ba8420a1ad18c5392517774f82ce14d6 Mon Sep 17 00:00:00 2001 From: Quentin Machu Date: Tue, 1 Dec 2015 14:58:17 -0500 Subject: [PATCH] updater: Refactor and merge fetcher responses Fixes #17 and lays the groundwork for #19. --- updater/fetchers.go | 18 +-- updater/fetchers/debian.go | 28 ++-- updater/fetchers/debian_test.go | 34 ++-- updater/fetchers/rhel.go | 28 ++-- updater/fetchers/rhel_test.go | 38 +++-- updater/fetchers/ubuntu.go | 16 +- updater/fetchers/ubuntu_test.go | 21 ++- updater/updater.go | 273 ++++++++++++++++++-------------- 8 files changed, 257 insertions(+), 199 deletions(-) diff --git a/updater/fetchers.go b/updater/fetchers.go index 36f6f307..d6295628 100644 --- a/updater/fetchers.go +++ b/updater/fetchers.go @@ -14,10 +14,7 @@ package updater -import ( - "github.com/coreos/clair/database" - "github.com/coreos/clair/utils/types" -) +import "github.com/coreos/clair/database" var fetchers = make(map[string]Fetcher) @@ -31,17 +28,8 @@ type FetcherResponse struct { FlagName string FlagValue string Notes []string - Vulnerabilities []FetcherVulnerability -} - -// FetcherVulnerability represents an individual vulnerability processed from -// an update. -type FetcherVulnerability struct { - ID string - Link string - Description string - Priority types.Priority - FixedIn []*database.Package + Vulnerabilities []*database.Vulnerability + Packages []*database.Package } // RegisterFetcher makes a Fetcher available by the provided name. diff --git a/updater/fetchers/debian.go b/updater/fetchers/debian.go index c653c3c5..f63f73c4 100644 --- a/updater/fetchers/debian.go +++ b/updater/fetchers/debian.go @@ -24,8 +24,8 @@ import ( "strings" "github.com/coreos/clair/database" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/updater" + cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -114,7 +114,8 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F } // Extract vulnerability data from Debian's JSON schema. - vulnerabilities, unknownReleases := parseDebianJSON(&data) + var unknownReleases map[string]struct{} + resp.Vulnerabilities, resp.Packages, unknownReleases = parseDebianJSON(&data) // Log unknown releases for k := range unknownReleases { @@ -123,16 +124,11 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F log.Warning(note) } - // Convert the vulnerabilities map to a slice in the response - for _, v := range vulnerabilities { - resp.Vulnerabilities = append(resp.Vulnerabilities, v) - } - return resp, nil } -func parseDebianJSON(data *jsonData) (vulnerabilities map[string]updater.FetcherVulnerability, unknownReleases map[string]struct{}) { - vulnerabilities = make(map[string]updater.FetcherVulnerability) +func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) { + mvulnerabilities := make(map[string]*database.Vulnerability) unknownReleases = make(map[string]struct{}) for pkgName, pkgNode := range *data { @@ -150,9 +146,9 @@ func parseDebianJSON(data *jsonData) (vulnerabilities map[string]updater.Fetcher } // Get or create the vulnerability. - vulnerability, vulnerabilityAlreadyExists := vulnerabilities[vulnName] + vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName] if !vulnerabilityAlreadyExists { - vulnerability = updater.FetcherVulnerability{ + vulnerability = &database.Vulnerability{ ID: vulnName, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), Priority: types.Unknown, @@ -191,14 +187,20 @@ func parseDebianJSON(data *jsonData) (vulnerabilities map[string]updater.Fetcher Name: pkgName, Version: version, } - vulnerability.FixedIn = append(vulnerability.FixedIn, pkg) + vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode()) + packages = append(packages, pkg) // Store the vulnerability. - vulnerabilities[vulnName] = vulnerability + mvulnerabilities[vulnName] = vulnerability } } } + // Convert the vulnerabilities map to a slice + for _, v := range mvulnerabilities { + vulnerabilities = append(vulnerabilities, v) + } + return } diff --git a/updater/fetchers/debian_test.go b/updater/fetchers/debian_test.go index e03c398c..f1243c3d 100644 --- a/updater/fetchers/debian_test.go +++ b/updater/fetchers/debian_test.go @@ -38,39 +38,49 @@ func TestDebianParser(t *testing.T) { assert.Equal(t, types.Low, vulnerability.Priority) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) - if assert.Len(t, vulnerability.FixedIn, 2) { - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + expectedPackages := []*database.Package{ + &database.Package{ OS: "debian:8", Name: "aptdaemon", Version: types.MaxVersion, - }) - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + }, + &database.Package{ OS: "debian:unstable", Name: "aptdaemon", Version: types.NewVersionUnsafe("1.1.1+bzr982-1"), - }) + }, + } + + for _, expectedPackage := range expectedPackages { + assert.Contains(t, response.Packages, expectedPackage) + assert.Contains(t, vulnerability.FixedInNodes, expectedPackage.GetNode()) } } else if vulnerability.ID == "CVE-2003-0779" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link) assert.Equal(t, types.High, vulnerability.Priority) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) - if assert.Len(t, vulnerability.FixedIn, 3) { - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + expectedPackages := []*database.Package{ + &database.Package{ OS: "debian:8", Name: "aptdaemon", Version: types.NewVersionUnsafe("0.7.0"), - }) - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + }, + &database.Package{ OS: "debian:unstable", Name: "aptdaemon", Version: types.NewVersionUnsafe("0.7.0"), - }) - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + }, + &database.Package{ OS: "debian:8", Name: "asterisk", Version: types.NewVersionUnsafe("0.5.56"), - }) + }, + } + + for _, expectedPackage := range expectedPackages { + assert.Contains(t, response.Packages, expectedPackage) + assert.Contains(t, vulnerability.FixedInNodes, expectedPackage.GetNode()) } } else { assert.Fail(t, "Wrong vulnerability name: ", vulnerability.ID) diff --git a/updater/fetchers/rhel.go b/updater/fetchers/rhel.go index 21efc84d..fd038131 100644 --- a/updater/fetchers/rhel.go +++ b/updater/fetchers/rhel.go @@ -24,8 +24,8 @@ import ( "strings" "github.com/coreos/clair/database" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/updater" + cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -128,17 +128,14 @@ func (f *RHELFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { } // Parse the XML. - vs, err := parseRHSA(r.Body) + vs, pkgs, err := parseRHSA(r.Body) if err != nil { return resp, err } // Collect vulnerabilities. - for _, v := range vs { - if len(v.FixedIn) > 0 { - resp.Vulnerabilities = append(resp.Vulnerabilities, v) - } - } + resp.Vulnerabilities = append(resp.Vulnerabilities, vs...) + resp.Packages = append(resp.Packages, pkgs...) } // Set the flag if we found anything. @@ -152,7 +149,7 @@ func (f *RHELFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) { return resp, nil } -func parseRHSA(ovalReader io.Reader) (vulnerabilities []updater.FetcherVulnerability, err error) { +func parseRHSA(ovalReader io.Reader) (vulnerabilities []*database.Vulnerability, packages []*database.Package, err error) { // Decode the XML. var ov oval err = xml.NewDecoder(ovalReader).Decode(&ov) @@ -163,18 +160,21 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []updater.FetcherVulnerabi } // Iterate over the definitions and collect any vulnerabilities that affect - // more than one package. + // at least one package. for _, definition := range ov.Definitions { - packages := toPackages(definition.Criteria) - if len(packages) > 0 { - vuln := updater.FetcherVulnerability{ + pkgs := toPackages(definition.Criteria) + if len(pkgs) > 0 { + vulnerability := &database.Vulnerability{ ID: name(definition), Link: link(definition), Priority: priority(definition), Description: description(definition), - FixedIn: packages, } - vulnerabilities = append(vulnerabilities, vuln) + for _, p := range pkgs { + vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, p.GetNode()) + } + vulnerabilities = append(vulnerabilities, vulnerability) + packages = append(packages, pkgs...) } } diff --git a/updater/fetchers/rhel_test.go b/updater/fetchers/rhel_test.go index 0a18faec..4db6078f 100644 --- a/updater/fetchers/rhel_test.go +++ b/updater/fetchers/rhel_test.go @@ -31,52 +31,62 @@ func TestRHELParser(t *testing.T) { // Test parsing testdata/fetcher_rhel_test.1.xml testFile, _ := os.Open(path + "/testdata/fetcher_rhel_test.1.xml") - vulnerabilities, err := parseRHSA(testFile) + vulnerabilities, packages, err := parseRHSA(testFile) if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].ID) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link) assert.Equal(t, types.Medium, vulnerabilities[0].Priority) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) - if assert.Len(t, vulnerabilities[0].FixedIn, 3) { - assert.Contains(t, vulnerabilities[0].FixedIn, &database.Package{ + expectedPackages := []*database.Package{ + &database.Package{ OS: "centos:7", Name: "xerces-c", Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), - }) - assert.Contains(t, vulnerabilities[0].FixedIn, &database.Package{ + }, + &database.Package{ OS: "centos:7", Name: "xerces-c-devel", Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), - }) - assert.Contains(t, vulnerabilities[0].FixedIn, &database.Package{ + }, + &database.Package{ OS: "centos:7", Name: "xerces-c-doc", Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), - }) + }, } + + for _, expectedPackage := range expectedPackages { + assert.Contains(t, packages, expectedPackage) + assert.Contains(t, vulnerabilities[0].FixedInNodes, expectedPackage.GetNode()) + } } // Test parsing testdata/fetcher_rhel_test.2.xml testFile, _ = os.Open(path + "/testdata/fetcher_rhel_test.2.xml") - vulnerabilities, err = parseRHSA(testFile) + vulnerabilities, packages, err = parseRHSA(testFile) if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].ID) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link) assert.Equal(t, types.Critical, vulnerabilities[0].Priority) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) - if assert.Len(t, vulnerabilities[0].FixedIn, 2) { - assert.Contains(t, vulnerabilities[0].FixedIn, &database.Package{ + expectedPackages := []*database.Package{ + &database.Package{ OS: "centos:6", Name: "firefox", Version: types.NewVersionUnsafe("38.1.0-1.el6_6"), - }) - assert.Contains(t, vulnerabilities[0].FixedIn, &database.Package{ + }, + &database.Package{ OS: "centos:7", Name: "firefox", Version: types.NewVersionUnsafe("38.1.0-1.el7_1"), - }) + }, } + + for _, expectedPackage := range expectedPackages { + assert.Contains(t, packages, expectedPackage) + assert.Contains(t, vulnerabilities[0].FixedInNodes, expectedPackage.GetNode()) + } } } diff --git a/updater/fetchers/ubuntu.go b/updater/fetchers/ubuntu.go index 92e88a50..4f7b9be4 100644 --- a/updater/fetchers/ubuntu.go +++ b/updater/fetchers/ubuntu.go @@ -133,13 +133,14 @@ func (fetcher *UbuntuFetcher) FetchUpdate() (resp updater.FetcherResponse, err e } defer file.Close() - v, unknownReleases, err := parseUbuntuCVE(file) + v, pkgs, unknownReleases, err := parseUbuntuCVE(file) if err != nil { return resp, err } - if len(v.FixedIn) > 0 { + if len(v.FixedInNodes) > 0 { resp.Vulnerabilities = append(resp.Vulnerabilities, v) + resp.Packages = append(resp.Packages, pkgs...) } // Log any unknown releases. @@ -255,7 +256,8 @@ func getRevisionNumber(pathToRepo string) (int, error) { return revno, nil } -func parseUbuntuCVE(fileContent io.Reader) (vulnerability updater.FetcherVulnerability, unknownReleases map[string]struct{}, err error) { +func parseUbuntuCVE(fileContent io.Reader) (vulnerability *database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}, err error) { + vulnerability = &database.Vulnerability{} unknownReleases = make(map[string]struct{}) readingDescription := false scanner := bufio.NewScanner(fileContent) @@ -351,7 +353,13 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability updater.FetcherVulnera } // Create and add the new package. - vulnerability.FixedIn = append(vulnerability.FixedIn, &database.Package{OS: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]], Name: md["package"], Version: version}) + pkg := &database.Package{ + OS: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]], + Name: md["package"], + Version: version, + } + packages = append(packages, pkg) + vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode()) } } } diff --git a/updater/fetchers/ubuntu_test.go b/updater/fetchers/ubuntu_test.go index 446e2152..1ad9e4fa 100644 --- a/updater/fetchers/ubuntu_test.go +++ b/updater/fetchers/ubuntu_test.go @@ -32,7 +32,7 @@ func TestUbuntuParser(t *testing.T) { // Test parsing testdata/fetcher_ testData, _ := os.Open(path + "/testdata/fetcher_ubuntu_test.txt") defer testData.Close() - vulnerability, unknownReleases, err := parseUbuntuCVE(testData) + vulnerability, packages, unknownReleases, err := parseUbuntuCVE(testData) if assert.Nil(t, err) { assert.Equal(t, "CVE-2015-4471", vulnerability.ID) assert.Equal(t, types.Medium, vulnerability.Priority) @@ -42,22 +42,27 @@ func TestUbuntuParser(t *testing.T) { _, hasUnkownRelease := unknownReleases["unknown"] assert.True(t, hasUnkownRelease) - if assert.Len(t, vulnerability.FixedIn, 3) { - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + expectedPackages := []*database.Package{ + &database.Package{ OS: "ubuntu:14.04", Name: "libmspack", Version: types.MaxVersion, - }) - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + }, + &database.Package{ OS: "ubuntu:15.04", Name: "libmspack", Version: types.NewVersionUnsafe("0.4-3"), - }) - assert.Contains(t, vulnerability.FixedIn, &database.Package{ + }, + &database.Package{ OS: "ubuntu:15.10", Name: "libmspack-anotherpkg", Version: types.NewVersionUnsafe("0.1"), - }) + }, + } + + for _, expectedPackage := range expectedPackages { + assert.Contains(t, packages, expectedPackage) + assert.Contains(t, vulnerability.FixedInNodes, expectedPackage.GetNode()) } } } diff --git a/updater/updater.go b/updater/updater.go index 1ad71b0d..5f76df3d 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -17,6 +17,8 @@ package updater import ( + "encoding/json" + "fmt" "math/rand" "strconv" "time" @@ -30,23 +32,12 @@ import ( const ( flagName = "updater" + notesFlagName = "updater/notes" refreshLockDuration = time.Minute * 8 lockDuration = refreshLockDuration + time.Minute*2 - - // healthMaxConsecutiveLocalFailures defines the number of times the updater - // can fail before we should tag it as unhealthy - healthMaxConsecutiveLocalFailures = 5 ) -var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater") - - healthLatestSuccessfulUpdate time.Time - healthLockOwner string - healthIdentifier string - healthConsecutiveLocalFailures int - healthNotes []string -) +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater") func init() { health.RegisterHealthchecker("updater", Healthcheck) @@ -63,19 +54,17 @@ func Run(interval time.Duration, st *utils.Stopper) { } whoAmI := uuid.New() - healthIdentifier = whoAmI log.Infof("updater service started. lock identifier: %s", whoAmI) for { // Set the next update time to (last update time + interval) or now if there // is no last update time stored in database (first update) or if an error // occurs - nextUpdate := time.Now().UTC() - if lastUpdateTSS, err := database.GetFlagValue(flagName); err == nil && lastUpdateTSS != "" { - if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil { - healthLatestSuccessfulUpdate = time.Unix(lastUpdateTS, 0) - nextUpdate = time.Unix(lastUpdateTS, 0).Add(interval) - } + var nextUpdate time.Time + if lastUpdate := getLastUpdate(); !lastUpdate.IsZero() { + nextUpdate = lastUpdate.Add(interval) + } else { + nextUpdate = time.Now().UTC() } // If the next update timer is in the past, then try to update. @@ -84,8 +73,6 @@ func Run(interval time.Duration, st *utils.Stopper) { log.Debug("attempting to obtain update lock") hasLock, hasLockUntil := database.Lock(flagName, lockDuration, whoAmI) if hasLock { - healthLockOwner = healthIdentifier - // Launch update in a new go routine. doneC := make(chan bool, 1) go func() { @@ -105,6 +92,8 @@ func Run(interval time.Duration, st *utils.Stopper) { // Unlock the update. database.Unlock(flagName, whoAmI) + + continue } else { lockOwner, lockExpiration, err := database.LockInfo(flagName) if err != nil { @@ -113,7 +102,6 @@ func Run(interval time.Duration, st *utils.Stopper) { } else { log.Debugf("update lock is already taken by %s until %v", lockOwner, lockExpiration) nextUpdate = lockExpiration - healthLockOwner = lockOwner } } } @@ -137,8 +125,62 @@ func Run(interval time.Duration, st *utils.Stopper) { func Update() { log.Info("updating vulnerabilities") + // Fetch updates. + status, responses := fetch() + + // Merge responses. + vulnerabilities, packages, flags, notes, err := mergeAndVerify(responses) + if err != nil { + log.Errorf("an error occured when merging update responses: %s", err) + return + } + responses = nil + + // TODO(Quentin-M): Complete informations using NVD + + // Insert packages. + log.Tracef("beginning insertion of %d packages for update", len(packages)) + err = database.InsertPackages(packages) + if err != nil { + log.Errorf("an error occured when inserting packages for update: %s", err) + return + } + packages = nil + + // Insert vulnerabilities. + log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) + notifications, err := database.InsertVulnerabilities(vulnerabilities) + if err != nil { + log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) + return + } + vulnerabilities = nil + + // Insert notifications into the database. + err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) + if err != nil { + log.Errorf("an error occured when inserting notifications for update: %s", err) + return + } + notifications = nil + + // Update flags and notes. + for flagName, flagValue := range flags { + database.UpdateFlag(flagName, flagValue) + } + database.UpdateFlag(notesFlagName, notes) + + // Update last successful update if every fetchers worked properly. + if status { + database.UpdateFlag(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) + } + log.Info("update finished") +} + +// fetch get data from the registered fetchers, in parallel. +func fetch() (status bool, responses []*FetcherResponse) { // Fetch updates in parallel. - var status = true + status = true var responseC = make(chan *FetcherResponse, 0) for n, f := range fetchers { go func(name string, fetcher Fetcher) { @@ -155,129 +197,122 @@ func Update() { } // Collect results of updates. - var responses []*FetcherResponse - var notes []string - for i := 0; i < len(fetchers); { - select { - case resp := <-responseC: - if resp != nil { - responses = append(responses, resp) - notes = append(notes, resp.Notes...) - } - i++ + for i := 0; i < len(fetchers); i++ { + resp := <-responseC + if resp != nil { + responses = append(responses, resp) } } close(responseC) + return +} - // TODO(Quentin-M): Merge responses together - // TODO(Quentin-M): Complete informations using NVD +// merge put all the responses together (vulnerabilities, packages, flags, notes), ensure the +// uniqueness of vulnerabilities and packages and verify that every vulnerability's fixedInNodes +// have their corresponding package definition. +func mergeAndVerify(responses []*FetcherResponse) (svulnerabilities []*database.Vulnerability, spackages []*database.Package, flags map[string]string, snotes string, err error) { + vulnerabilities := make(map[string]*database.Vulnerability) + packages := make(map[string]*database.Package) + flags = make(map[string]string) + var notes []string - // Store flags out of the response struct. - flags := make(map[string]string) + // Merge responses. for _, response := range responses { + // Notes + notes = append(notes, response.Notes...) + // Flags if response.FlagName != "" && response.FlagValue != "" { flags[response.FlagName] = response.FlagValue } - } - - // Update health notes. - healthNotes = notes - - // Build list of packages. - var packages []*database.Package - for _, response := range responses { - for _, v := range response.Vulnerabilities { - packages = append(packages, v.FixedIn...) - } - } - - // Insert packages into the database. - log.Tracef("beginning insertion of %d packages for update", len(packages)) - t := time.Now() - err := database.InsertPackages(packages) - log.Tracef("inserting %d packages took %v", len(packages), time.Since(t)) - if err != nil { - log.Errorf("an error occured when inserting packages for update: %s", err) - updateHealth(false) - return - } - packages = nil - - // Build a list of vulnerabilties. - var vulnerabilities []*database.Vulnerability - for _, response := range responses { - for _, v := range response.Vulnerabilities { - var packageNodes []string - for _, pkg := range v.FixedIn { - packageNodes = append(packageNodes, pkg.Node) + // Packages + for _, p := range response.Packages { + node := p.GetNode() + if _, ok := packages[node]; !ok { + packages[node] = p + } + } + // Vulnerabilities + for _, v := range response.Vulnerabilities { + if vulnerability, ok := vulnerabilities[v.ID]; !ok { + vulnerabilities[v.ID] = v + } else { + mergeVulnerability(vulnerability, v) } - vulnerabilities = append(vulnerabilities, &database.Vulnerability{ID: v.ID, Link: v.Link, Priority: v.Priority, Description: v.Description, FixedInNodes: packageNodes}) } } - responses = nil - // Insert vulnerabilities into the database. - log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) - t = time.Now() - notifications, err := database.InsertVulnerabilities(vulnerabilities) - log.Tracef("inserting %d vulnerabilities took %v", len(vulnerabilities), time.Since(t)) - if err != nil { - log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) - updateHealth(false) - return - } - vulnerabilities = nil - - // Insert notifications into the database. - err = database.InsertNotifications(notifications, database.GetDefaultNotificationWrapper()) - if err != nil { - log.Errorf("an error occured when inserting notifications for update: %s", err) - updateHealth(false) - return - } - notifications = nil - - // Update flags in the database. - for flagName, flagValue := range flags { - database.UpdateFlag(flagName, flagValue) + // Verify that the packages used in the vulnerabilities are specified. + for _, v := range vulnerabilities { + for _, node := range v.FixedInNodes { + if _, ok := packages[node]; !ok { + err = fmt.Errorf("vulnerability %s is fixed by an unspecified package", v.ID) + return + } + } } - // Update health depending on the status of the fetchers. - updateHealth(status) - if status { - now := time.Now().UTC() - database.UpdateFlag(flagName, strconv.FormatInt(now.Unix(), 10)) - healthLatestSuccessfulUpdate = now + // Convert data and return + for _, v := range vulnerabilities { + svulnerabilities = append(svulnerabilities, v) } - log.Info("update finished") + for _, p := range packages { + spackages = append(spackages, p) + } + + bnotes, _ := json.Marshal(notes) + snotes = string(bnotes) + + return } -func updateHealth(s bool) { - if s == false { - healthConsecutiveLocalFailures++ - } else { - healthConsecutiveLocalFailures = 0 +// mergeVulnerability updates the target vulnerability structure using the specified one. +func mergeVulnerability(target, source *database.Vulnerability) { + if source.Link != "" { + target.Link = source.Link + } + if source.Description != "" { + target.Description = source.Description + } + if source.Priority.Compare(target.Priority) > 0 { + target.Priority = source.Priority + } + for _, node := range source.FixedInNodes { + if !utils.Contains(node, target.FixedInNodes) { + target.FixedInNodes = append(target.FixedInNodes, node) + } } } // Healthcheck returns the health of the updater service. func Healthcheck() health.Status { + notes := getNotes() + return health.Status{ IsEssential: false, - IsHealthy: healthConsecutiveLocalFailures < healthMaxConsecutiveLocalFailures, + IsHealthy: len(notes) == 0, Details: struct { - HealthIdentifier string - HealthLockOwner string - LatestSuccessfulUpdate time.Time - ConsecutiveLocalFailures int - Notes []string `json:",omitempty"` + LatestSuccessfulUpdate time.Time + Notes []string `json:",omitempty"` }{ - HealthIdentifier: healthIdentifier, - HealthLockOwner: healthLockOwner, - LatestSuccessfulUpdate: healthLatestSuccessfulUpdate, - ConsecutiveLocalFailures: healthConsecutiveLocalFailures, - Notes: healthNotes, + LatestSuccessfulUpdate: getLastUpdate(), + Notes: notes, }, } } + +func getLastUpdate() time.Time { + if lastUpdateTSS, err := database.GetFlagValue(flagName); err == nil && lastUpdateTSS != "" { + if lastUpdateTS, err := strconv.ParseInt(lastUpdateTSS, 10, 64); err == nil { + return time.Unix(lastUpdateTS, 0).UTC() + } + } + return time.Time{} +} + +func getNotes() (notes []string) { + if jsonNotes, err := database.GetFlagValue(notesFlagName); err == nil && jsonNotes != "" { + json.Unmarshal([]byte(jsonNotes), notes) + } + return +}