From 4a990372fff35f606184e276976f0279e4ea5a56 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 3 Jan 2017 21:44:32 -0500 Subject: [PATCH] refactor: move updaters and notifier into ext --- cmd/clair/main.go | 17 ++-- ext/notification/driver.go | 65 +++++++++++++ .../notification/webhook}/webhook.go | 33 ++++--- ext/vulnmdsrc/driver.go | 68 +++++++++++++ ext/vulnmdsrc/nvd/nested_read_closer.go | 33 +++++++ .../vulnmdsrc}/nvd/nvd.go | 95 ++++++++++--------- .../vulnmdsrc}/nvd/xml.go | 14 +++ .../fetchers => ext/vulnsrc}/alpine/alpine.go | 61 ++++++------ .../vulnsrc}/alpine/alpine_test.go | 0 .../vulnsrc}/alpine/testdata/v33_main.yaml | 0 .../vulnsrc}/alpine/testdata/v34_main.yaml | 0 .../fetchers => ext/vulnsrc}/debian/debian.go | 24 +++-- .../vulnsrc}/debian/debian_test.go | 0 .../debian/testdata/fetcher_debian_test.json | 0 ext/vulnsrc/driver.go | 73 ++++++++++++++ .../fetchers => ext/vulnsrc}/oracle/oracle.go | 22 ++--- .../vulnsrc}/oracle/oracle_test.go | 0 .../oracle/testdata/fetcher_oracle_test.1.xml | 0 .../oracle/testdata/fetcher_oracle_test.2.xml | 0 .../fetchers => ext/vulnsrc}/rhel/rhel.go | 22 ++--- .../vulnsrc}/rhel/rhel_test.go | 0 .../rhel/testdata/fetcher_rhel_test.1.xml | 0 .../rhel/testdata/fetcher_rhel_test.2.xml | 0 .../ubuntu/testdata/fetcher_ubuntu_test.txt | 0 .../fetchers => ext/vulnsrc}/ubuntu/ubuntu.go | 56 +++++------ .../vulnsrc}/ubuntu/ubuntu_test.go | 0 notifier/notifier.go | 66 ++++--------- updater/fetchers.go | 56 ----------- updater/metadata_fetchers.go | 64 ------------- .../nvd/nested_read_closer.go | 19 ---- updater/updater.go | 91 ++++++++++++------ 31 files changed, 487 insertions(+), 392 deletions(-) create mode 100644 ext/notification/driver.go rename {notifier/notifiers => ext/notification/webhook}/webhook.go (79%) create mode 100644 ext/vulnmdsrc/driver.go create mode 100644 ext/vulnmdsrc/nvd/nested_read_closer.go rename {updater/metadata_fetchers => ext/vulnmdsrc}/nvd/nvd.go (72%) rename {updater/metadata_fetchers => ext/vulnmdsrc}/nvd/xml.go (78%) rename {updater/fetchers => ext/vulnsrc}/alpine/alpine.go (80%) rename {updater/fetchers => ext/vulnsrc}/alpine/alpine_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/alpine/testdata/v33_main.yaml (100%) rename {updater/fetchers => ext/vulnsrc}/alpine/testdata/v34_main.yaml (100%) rename {updater/fetchers => ext/vulnsrc}/debian/debian.go (91%) rename {updater/fetchers => ext/vulnsrc}/debian/debian_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/debian/testdata/fetcher_debian_test.json (100%) create mode 100644 ext/vulnsrc/driver.go rename {updater/fetchers => ext/vulnsrc}/oracle/oracle.go (93%) rename {updater/fetchers => ext/vulnsrc}/oracle/oracle_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/oracle/testdata/fetcher_oracle_test.1.xml (100%) rename {updater/fetchers => ext/vulnsrc}/oracle/testdata/fetcher_oracle_test.2.xml (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/rhel.go (94%) rename {updater/fetchers => ext/vulnsrc}/rhel/rhel_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/testdata/fetcher_rhel_test.1.xml (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/testdata/fetcher_rhel_test.2.xml (100%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/testdata/fetcher_ubuntu_test.txt (100%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/ubuntu.go (86%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/ubuntu_test.go (100%) delete mode 100644 updater/fetchers.go delete mode 100644 updater/metadata_fetchers.go delete mode 100644 updater/metadata_fetchers/nvd/nested_read_closer.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 1e3735d8..8d3e35f5 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -25,15 +25,14 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/config" - // Register components - _ "github.com/coreos/clair/notifier/notifiers" - - _ "github.com/coreos/clair/updater/fetchers/alpine" - _ "github.com/coreos/clair/updater/fetchers/debian" - _ "github.com/coreos/clair/updater/fetchers/oracle" - _ "github.com/coreos/clair/updater/fetchers/rhel" - _ "github.com/coreos/clair/updater/fetchers/ubuntu" - _ "github.com/coreos/clair/updater/metadata_fetchers/nvd" + // Register extensions. + _ "github.com/coreos/clair/ext/notification/webhook" + _ "github.com/coreos/clair/ext/vulnmdsrc/nvd" + _ "github.com/coreos/clair/ext/vulnsrc/alpine" + _ "github.com/coreos/clair/ext/vulnsrc/debian" + _ "github.com/coreos/clair/ext/vulnsrc/oracle" + _ "github.com/coreos/clair/ext/vulnsrc/rhel" + _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" _ "github.com/coreos/clair/worker/detectors/data/aci" _ "github.com/coreos/clair/worker/detectors/data/docker" diff --git a/ext/notification/driver.go b/ext/notification/driver.go new file mode 100644 index 00000000..2f3b6429 --- /dev/null +++ b/ext/notification/driver.go @@ -0,0 +1,65 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notification fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. + +// Package notification exposes functions to dynamically register methods to +// deliver notifications from the Clair database. +package notification + +import ( + "github.com/coreos/pkg/capnslog" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/database" +) + +var ( + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/notification") + + // Senders is the list of registered Senders. + Senders = make(map[string]Sender) +) + +// Sender represents anything that can transmit notifications. +type Sender interface { + // Configure attempts to initialize the notifier with the provided configuration. + // It returns whether the notifier is enabled or not. + Configure(*config.NotifierConfig) (bool, error) + + // Send informs the existence of the specified notification. + Send(notification database.VulnerabilityNotification) error +} + +// RegisterSender makes a Sender available by the provided name. +// +// If RegisterSender is called twice with the same name, the name is blank, or +// if the provided Sender is nil, this function panics. +func RegisterSender(name string, s Sender) { + if name == "" { + panic("notification: could not register a Sender with an empty name") + } + + if s == nil { + panic("notification: could not register a nil Sender") + } + + if _, dup := Senders[name]; dup { + panic("notification: RegisterSender called twice for " + name) + } + + Senders[name] = s +} diff --git a/notifier/notifiers/webhook.go b/ext/notification/webhook/webhook.go similarity index 79% rename from notifier/notifiers/webhook.go rename to ext/notification/webhook/webhook.go index 28b76604..6a991cf4 100644 --- a/notifier/notifiers/webhook.go +++ b/ext/notification/webhook/webhook.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package notifiers implements several kinds of notifier.Notifier -package notifiers +// Package webhook implements a notification sender for HTTP JSON webhooks. +package webhook import ( "bytes" @@ -31,19 +31,18 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" - "github.com/coreos/clair/notifier" + "github.com/coreos/clair/ext/notification" ) const timeout = 5 * time.Second -// A WebhookNotifier dispatches notifications to a webhook endpoint. -type WebhookNotifier struct { +type sender struct { endpoint string client *http.Client } -// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier. -type WebhookNotifierConfiguration struct { +// Config represents the configuration of a Webhook Sender. +type Config struct { Endpoint string ServerName string CertFile string @@ -53,12 +52,12 @@ type WebhookNotifierConfiguration struct { } func init() { - notifier.RegisterNotifier("webhook", &WebhookNotifier{}) + notification.RegisterSender("webhook", &sender{}) } -func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) { +func (s *sender) Configure(config *config.NotifierConfig) (bool, error) { // Get configuration - var httpConfig WebhookNotifierConfiguration + var httpConfig Config if config == nil { return false, nil } @@ -81,11 +80,11 @@ func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) if _, err := url.ParseRequestURI(httpConfig.Endpoint); err != nil { return false, fmt.Errorf("could not parse endpoint URL: %s\n", err) } - h.endpoint = httpConfig.Endpoint + s.endpoint = httpConfig.Endpoint // Setup HTTP client. transport := &http.Transport{} - h.client = &http.Client{ + s.client = &http.Client{ Transport: transport, Timeout: timeout, } @@ -114,7 +113,7 @@ type notificationEnvelope struct { } } -func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error { +func (s *sender) Send(notification database.VulnerabilityNotification) error { // Marshal notification. jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}}) if err != nil { @@ -122,7 +121,7 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) } // Send notification via HTTP POST. - resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification)) + resp, err := s.client.Post(s.endpoint, "application/json", bytes.NewBuffer(jsonNotification)) if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) { if resp != nil { return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode) @@ -134,11 +133,11 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) return nil } -// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration. +// loadTLSClientConfig initializes a *tls.Config using the given Config. // // If no certificates are given, (nil, nil) is returned. // The CA certificate is optional and falls back to the system default. -func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) { +func loadTLSClientConfig(cfg *Config) (*tls.Config, error) { if cfg.CertFile == "" || cfg.KeyFile == "" { return nil, nil } diff --git a/ext/vulnmdsrc/driver.go b/ext/vulnmdsrc/driver.go new file mode 100644 index 00000000..658125de --- /dev/null +++ b/ext/vulnmdsrc/driver.go @@ -0,0 +1,68 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vulnmdsrc exposes functions to dynamically register vulnerability +// metadata sources used to update a Clair database. +package vulnmdsrc + +import ( + "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" +) + +// Appenders is the list of registered Appenders. +var Appenders = make(map[string]Appender) + +// AppendFunc is the type of a callback provided to an Appender. +type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority) + +// Appender represents anything that can fetch vulnerability metadata and +// append it to a Vulnerability. +type Appender interface { + // BuildCache loads metadata into memory such that it can be quickly accessed + // for future calls to Append. + BuildCache(database.Datastore) error + + // AddMetadata adds metadata to the given database.Vulnerability. + // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. + // Append + Append(vulnName string, callback AppendFunc) error + + // PurgeCache deallocates metadata from memory after all calls to Append are + // finished. + PurgeCache() + + // Clean deletes any allocated resources. + // It is invoked when Clair stops. + Clean() +} + +// RegisterAppender makes an Appender available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func RegisterAppender(name string, a Appender) { + if name == "" { + panic("updater: could not register an Appender with an empty name") + } + + if a == nil { + panic("vulnmdsrc: could not register a nil Appender") + } + + if _, dup := Appenders[name]; dup { + panic("vulnmdsrc: RegisterAppender called twice for " + name) + } + + Appenders[name] = a +} diff --git a/ext/vulnmdsrc/nvd/nested_read_closer.go b/ext/vulnmdsrc/nvd/nested_read_closer.go new file mode 100644 index 00000000..23322655 --- /dev/null +++ b/ext/vulnmdsrc/nvd/nested_read_closer.go @@ -0,0 +1,33 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvd + +import "io" + +// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed +// io.ReadCloser. +// It allows chaining io.ReadCloser together and still keep the ability to close them all in a +// simple manner. +type NestedReadCloser struct { + io.Reader + NestedReadClosers []io.ReadCloser +} + +// Close closes the gzip.Reader and the underlying io.ReadCloser. +func (nrc *NestedReadCloser) Close() { + for _, nestedReadCloser := range nrc.NestedReadClosers { + nestedReadCloser.Close() + } +} diff --git a/updater/metadata_fetchers/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go similarity index 72% rename from updater/metadata_fetchers/nvd/nvd.go rename to ext/vulnmdsrc/nvd/nvd.go index fadf719c..31636a77 100644 --- a/updater/metadata_fetchers/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -1,3 +1,19 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nvd implements a vulnerability metadata appender using the NIST NVD +// database. package nvd import ( @@ -15,30 +31,28 @@ import ( "sync" "time" + "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair/database" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnmdsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" - "github.com/coreos/pkg/capnslog" ) const ( dataFeedURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz" dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" - metadataKey string = "NVD" + appenderName string = "NVD" ) -var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers") -) +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnmdsrc/nvd") -type NVDMetadataFetcher struct { +type appender struct { localPath string dataFeedHashes map[string]string - lock sync.Mutex - - metadata map[string]NVDMetadata + metadata map[string]NVDMetadata + sync.Mutex } type NVDMetadata struct { @@ -51,32 +65,32 @@ type NVDmetadataCVSSv2 struct { } func init() { - updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{}) + vulnmdsrc.RegisterAppender(appenderName, &appender{}) } -func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) BuildCache(datastore database.Datastore) error { + a.Lock() + defer a.Unlock() var err error - fetcher.metadata = make(map[string]NVDMetadata) + a.metadata = make(map[string]NVDMetadata) // Init if necessary. - if fetcher.localPath == "" { + if a.localPath == "" { // Create a temporary folder to store the NVD data and create hashes struct. - if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { + if a.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { return cerrors.ErrFilesystem } - fetcher.dataFeedHashes = make(map[string]string) + a.dataFeedHashes = make(map[string]string) } // Get data feeds. - dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath) + dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath) if err != nil { return err } - fetcher.dataFeedHashes = dataFeedHashes + a.dataFeedHashes = dataFeedHashes // Parse data feeds. for dataFeedName, dataFeedReader := range dataFeedReaders { @@ -90,7 +104,7 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { for _, nvdEntry := range nvd.Entries { // Create metadata entry. if metadata := nvdEntry.Metadata(); metadata != nil { - fetcher.metadata[nvdEntry.Name] = *metadata + a.metadata[nvdEntry.Name] = *metadata } } @@ -100,42 +114,29 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { return nil } -func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error { + a.Lock() + defer a.Unlock() - if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok { - vulnerability.Lock.Lock() - - // Create Metadata map if necessary and assign the NVD metadata. - if vulnerability.Metadata == nil { - vulnerability.Metadata = make(map[string]interface{}) - } - vulnerability.Metadata[metadataKey] = nvdMetadata - - // Set the Severity using the CVSSv2 Score if none is set yet. - if vulnerability.Severity == "" || vulnerability.Severity == types.Unknown { - vulnerability.Severity = scoreToPriority(nvdMetadata.CVSSv2.Score) - } - - vulnerability.Lock.Unlock() + if nvdMetadata, ok := a.metadata[vulnName]; ok { + appendFunc(appenderName, nvdMetadata, scoreToPriority(nvdMetadata.CVSSv2.Score)) } return nil } -func (fetcher *NVDMetadataFetcher) Unload() { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) PurgeCache() { + a.Lock() + defer a.Unlock() - fetcher.metadata = nil + a.metadata = nil } -func (fetcher *NVDMetadataFetcher) Clean() { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) Clean() { + a.Lock() + defer a.Unlock() - os.RemoveAll(fetcher.localPath) + os.RemoveAll(a.localPath) } func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) { diff --git a/updater/metadata_fetchers/nvd/xml.go b/ext/vulnmdsrc/nvd/xml.go similarity index 78% rename from updater/metadata_fetchers/nvd/xml.go rename to ext/vulnmdsrc/nvd/xml.go index db13c897..d7649ce6 100644 --- a/updater/metadata_fetchers/nvd/xml.go +++ b/ext/vulnmdsrc/nvd/xml.go @@ -1,3 +1,17 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package nvd import ( diff --git a/updater/fetchers/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go similarity index 80% rename from updater/fetchers/alpine/alpine.go rename to ext/vulnsrc/alpine/alpine.go index bda68326..a2bbf7ec 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package alpine implements a vulnerability Fetcher using the alpine-secdb -// git repository. +// Package alpine implements a vulnerability source updater using the +// alpine-secdb git repository. package alpine import ( - "errors" "fmt" "io" "io/ioutil" @@ -31,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -45,29 +44,23 @@ const ( ) var ( - // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. - ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs") - - // ErrGitFailure is returned when a fetcher fails to interact with git. - ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git") - - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine") ) func init() { - updater.RegisterFetcher("alpine", &fetcher{}) + vulnsrc.RegisterUpdater("alpine", &updater{}) } -type fetcher struct { +type updater struct { repositoryLocalPath string } -func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Alpine vulnerabilities") // Pull the master branch. var commit string - commit, err = f.pullRepository() + commit, err = u.pullRepository() if err != nil { return } @@ -90,12 +83,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon } var namespaces []string - namespaces, err = detectNamespaces(f.repositoryLocalPath) + namespaces, err = detectNamespaces(u.repositoryLocalPath) // Append any changed vulnerabilities to the response. for _, namespace := range namespaces { var vulns []database.Vulnerability var note string - vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) + vulns, note, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace) if err != nil { return } @@ -108,6 +101,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon return } +func (u *updater) Clean() { + if u.repositoryLocalPath != "" { + os.RemoveAll(u.repositoryLocalPath) + } +} + func detectNamespaces(path string) ([]string, error) { // Open the root directory. dir, err := os.Open(path) @@ -163,41 +162,35 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database return } -func (f *fetcher) pullRepository() (commit string, err error) { +func (u *updater) pullRepository() (commit string, err error) { // If the repository doesn't exist, clone it. - if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) { - if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { - return "", ErrFilesystem + if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { + if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { + return "", vulnsrc.ErrFilesystem } - if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { - f.Clean() + if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { + u.Clean() log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) return "", cerrors.ErrCouldNotDownload } } else { // The repository exists and it needs to be refreshed via a pull. - _, err := utils.Exec(f.repositoryLocalPath, "git", "pull") + _, err := utils.Exec(u.repositoryLocalPath, "git", "pull") if err != nil { - return "", ErrGitFailure + return "", vulnsrc.ErrGitFailure } } - out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD") + out, err := utils.Exec(u.repositoryLocalPath, "git", "rev-parse", "HEAD") if err != nil { - return "", ErrGitFailure + return "", vulnsrc.ErrGitFailure } commit = strings.TrimSpace(string(out)) return } -func (f *fetcher) Clean() { - if f.repositoryLocalPath != "" { - os.RemoveAll(f.repositoryLocalPath) - } -} - type secdb33File struct { Distro string `yaml:"distroversion"` Packages []struct { diff --git a/updater/fetchers/alpine/alpine_test.go b/ext/vulnsrc/alpine/alpine_test.go similarity index 100% rename from updater/fetchers/alpine/alpine_test.go rename to ext/vulnsrc/alpine/alpine_test.go diff --git a/updater/fetchers/alpine/testdata/v33_main.yaml b/ext/vulnsrc/alpine/testdata/v33_main.yaml similarity index 100% rename from updater/fetchers/alpine/testdata/v33_main.yaml rename to ext/vulnsrc/alpine/testdata/v33_main.yaml diff --git a/updater/fetchers/alpine/testdata/v34_main.yaml b/ext/vulnsrc/alpine/testdata/v34_main.yaml similarity index 100% rename from updater/fetchers/alpine/testdata/v34_main.yaml rename to ext/vulnsrc/alpine/testdata/v34_main.yaml diff --git a/updater/fetchers/debian/debian.go b/ext/vulnsrc/debian/debian.go similarity index 91% rename from updater/fetchers/debian/debian.go rename to ext/vulnsrc/debian/debian.go index 4838e622..84d4398f 100644 --- a/updater/fetchers/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package debian implements a vulnerability source updater using the Debian +// Security Tracker. package debian import ( @@ -28,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -39,7 +41,7 @@ const ( updaterFlag = "debianUpdater" ) -var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian") +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/debian") type jsonData map[string]map[string]jsonVuln @@ -54,16 +56,13 @@ type jsonRel struct { Urgency string `json:"urgency"` } -// DebianFetcher implements updater.Fetcher for the Debian Security Tracker -// (https://security-tracker.debian.org). -type DebianFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("debian", &DebianFetcher{}) + vulnsrc.RegisterUpdater("debian", &updater{}) } -// FetchUpdate fetches vulnerability updates from the Debian Security Tracker. -func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Debian vulnerabilities") // Download JSON. @@ -88,7 +87,9 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up return resp, nil } -func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.FetcherResponse, err error) { +func (u *updater) Clean() {} + +func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.UpdateResponse, err error) { hash := latestKnownHash // Defer the addition of flag information to the response. @@ -254,6 +255,3 @@ func urgencyToSeverity(urgency string) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (fetcher *DebianFetcher) Clean() {} diff --git a/updater/fetchers/debian/debian_test.go b/ext/vulnsrc/debian/debian_test.go similarity index 100% rename from updater/fetchers/debian/debian_test.go rename to ext/vulnsrc/debian/debian_test.go diff --git a/updater/fetchers/debian/testdata/fetcher_debian_test.json b/ext/vulnsrc/debian/testdata/fetcher_debian_test.json similarity index 100% rename from updater/fetchers/debian/testdata/fetcher_debian_test.json rename to ext/vulnsrc/debian/testdata/fetcher_debian_test.json diff --git a/ext/vulnsrc/driver.go b/ext/vulnsrc/driver.go new file mode 100644 index 00000000..132eabee --- /dev/null +++ b/ext/vulnsrc/driver.go @@ -0,0 +1,73 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vulnsrc exposes functions to dynamically register vulnerability +// sources used to update a Clair database. +package vulnsrc + +import ( + "errors" + + "github.com/coreos/clair/database" +) + +var ( + // Updaters is the list of registered Updaters. + Updaters = make(map[string]Updater) + + // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. + ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs") + + // ErrGitFailure is returned when a fetcher fails to interact with git. + ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git") +) + +// Updater represents anything that can fetch vulnerabilities and insert them +// into a Clair datastore. +type Updater interface { + // Update gets vulnerability updates. + Update(database.Datastore) (UpdateResponse, error) + + // Clean deletes any allocated resources. + // It is invoked when Clair stops. + Clean() +} + +// UpdateResponse represents the sum of results of an update. +type UpdateResponse struct { + FlagName string + FlagValue string + Notes []string + Vulnerabilities []database.Vulnerability +} + +// RegisterUpdater makes an Updater available by the provided name. +// +// If RegisterUpdater is called twice with the same name, the name is blank, or +// if the provided Updater is nil, this function panics. +func RegisterUpdater(name string, u Updater) { + if name == "" { + panic("vulnsrc: could not register an Updater with an empty name") + } + + if u == nil { + panic("vulnsrc: could not register a nil Updater") + } + + if _, dup := Updaters[name]; dup { + panic("vulnsrc: RegisterUpdater called twice for " + name) + } + + Updaters[name] = u +} diff --git a/updater/fetchers/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go similarity index 93% rename from updater/fetchers/oracle/oracle.go rename to ext/vulnsrc/oracle/oracle.go index fceae36f..32afeaa0 100644 --- a/updater/fetchers/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package oracle implements a vulnerability source updater using the +// Oracle Linux OVAL Database. package oracle import ( @@ -26,7 +28,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" "github.com/coreos/pkg/capnslog" @@ -47,7 +49,7 @@ var ( elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`) - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/oracle") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/oracle") ) type oval struct { @@ -77,16 +79,13 @@ type criterion struct { Comment string `xml:"comment,attr"` } -// OracleFetcher implements updater.Fetcher and gets vulnerability updates from -// the Oracle Linux OVAL definitions. -type OracleFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("Oracle", &OracleFetcher{}) + vulnsrc.RegisterUpdater("oracle", &updater{}) } -// FetchUpdate gets vulnerability updates from the Oracle Linux OVAL definitions. -func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Oracle Linux vulnerabilities") // Get the first ELSA we have to manage. @@ -153,6 +152,8 @@ func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater. return resp, nil } +func (u *updater) Clean() {} + func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) { // Decode the XML. var ov oval @@ -356,6 +357,3 @@ func priority(def definition) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (f *OracleFetcher) Clean() {} diff --git a/updater/fetchers/oracle/oracle_test.go b/ext/vulnsrc/oracle/oracle_test.go similarity index 100% rename from updater/fetchers/oracle/oracle_test.go rename to ext/vulnsrc/oracle/oracle_test.go diff --git a/updater/fetchers/oracle/testdata/fetcher_oracle_test.1.xml b/ext/vulnsrc/oracle/testdata/fetcher_oracle_test.1.xml similarity index 100% rename from updater/fetchers/oracle/testdata/fetcher_oracle_test.1.xml rename to ext/vulnsrc/oracle/testdata/fetcher_oracle_test.1.xml diff --git a/updater/fetchers/oracle/testdata/fetcher_oracle_test.2.xml b/ext/vulnsrc/oracle/testdata/fetcher_oracle_test.2.xml similarity index 100% rename from updater/fetchers/oracle/testdata/fetcher_oracle_test.2.xml rename to ext/vulnsrc/oracle/testdata/fetcher_oracle_test.2.xml diff --git a/updater/fetchers/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go similarity index 94% rename from updater/fetchers/rhel/rhel.go rename to ext/vulnsrc/rhel/rhel.go index 1a0c423d..6714dcd8 100644 --- a/updater/fetchers/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package rhel implements a vulnerability source updater using the +// Red Hat Linux OVAL Database. package rhel import ( @@ -28,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -82,17 +84,14 @@ type criterion struct { Comment string `xml:"comment,attr"` } -// RHELFetcher implements updater.Fetcher and gets vulnerability updates from -// the Red Hat OVAL definitions. -type RHELFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("Red Hat", &RHELFetcher{}) + vulnsrc.RegisterUpdater("rhel", &updater{}) } -// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions. -func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { - log.Info("fetching Red Hat vulnerabilities") +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { + log.Info("fetching RHEL vulnerabilities") // Get the first RHSA we have to manage. flagValue, err := datastore.GetKeyValue(updaterFlag) @@ -156,6 +155,8 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe return resp, nil } +func (u *updater) Clean() {} + func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) { // Decode the XML. var ov oval @@ -362,6 +363,3 @@ func priority(def definition) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (f *RHELFetcher) Clean() {} diff --git a/updater/fetchers/rhel/rhel_test.go b/ext/vulnsrc/rhel/rhel_test.go similarity index 100% rename from updater/fetchers/rhel/rhel_test.go rename to ext/vulnsrc/rhel/rhel_test.go diff --git a/updater/fetchers/rhel/testdata/fetcher_rhel_test.1.xml b/ext/vulnsrc/rhel/testdata/fetcher_rhel_test.1.xml similarity index 100% rename from updater/fetchers/rhel/testdata/fetcher_rhel_test.1.xml rename to ext/vulnsrc/rhel/testdata/fetcher_rhel_test.1.xml diff --git a/updater/fetchers/rhel/testdata/fetcher_rhel_test.2.xml b/ext/vulnsrc/rhel/testdata/fetcher_rhel_test.2.xml similarity index 100% rename from updater/fetchers/rhel/testdata/fetcher_rhel_test.2.xml rename to ext/vulnsrc/rhel/testdata/fetcher_rhel_test.2.xml diff --git a/updater/fetchers/ubuntu/testdata/fetcher_ubuntu_test.txt b/ext/vulnsrc/ubuntu/testdata/fetcher_ubuntu_test.txt similarity index 100% rename from updater/fetchers/ubuntu/testdata/fetcher_ubuntu_test.txt rename to ext/vulnsrc/ubuntu/testdata/fetcher_ubuntu_test.txt diff --git a/updater/fetchers/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go similarity index 86% rename from updater/fetchers/ubuntu/ubuntu.go rename to ext/vulnsrc/ubuntu/ubuntu.go index 68291557..ea0035da 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package ubuntu implements a vulnerability source updater using the +// Ubuntu CVE Tracker. package ubuntu import ( "bufio" "bytes" - "errors" "fmt" "io" "io/ioutil" @@ -31,7 +32,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -75,33 +76,27 @@ var ( affectsCaptureRegexp = regexp.MustCompile(`(?P.*)_(?P.*): (?P[^\s]*)( \(+(?P[^()]*)\)+)?`) affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames() - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu") - - // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. - ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/ubuntu") ) -// UbuntuFetcher implements updater.Fetcher and gets vulnerability updates from -// the Ubuntu CVE Tracker. -type UbuntuFetcher struct { +type updater struct { repositoryLocalPath string } func init() { - updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{}) + vulnsrc.RegisterUpdater("ubuntu", &updater{}) } -// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker. -func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Ubuntu vulnerabilities") // Pull the bzr repository. - if err = fetcher.pullRepository(); err != nil { + if err = u.pullRepository(); err != nil { return resp, err } // Get revision number. - revisionNumber, err := getRevisionNumber(fetcher.repositoryLocalPath) + revisionNumber, err := getRevisionNumber(u.repositoryLocalPath) if err != nil { return resp, err } @@ -113,7 +108,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up } // Get the list of vulnerabilities that we have to update. - modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, fetcher.repositoryLocalPath) + modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath) if err != nil { return resp, err } @@ -121,7 +116,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up notes := make(map[string]struct{}) for cvePath := range modifiedCVE { // Open the CVE file. - file, err := os.Open(fetcher.repositoryLocalPath + "/" + cvePath) + file, err := os.Open(u.repositoryLocalPath + "/" + cvePath) if err != nil { // This can happen when a file is modified and then moved in another // commit. @@ -166,16 +161,20 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up return } -func (fetcher *UbuntuFetcher) pullRepository() (err error) { +func (u *updater) Clean() { + os.RemoveAll(u.repositoryLocalPath) +} + +func (u *updater) pullRepository() (err error) { // Determine whether we should branch or pull. - if _, pathExists := os.Stat(fetcher.repositoryLocalPath); fetcher.repositoryLocalPath == "" || os.IsNotExist(pathExists) { + if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { // Create a temporary folder to store the repository. - if fetcher.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { - return ErrFilesystem + if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { + return vulnsrc.ErrFilesystem } // Branch repository. - if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { + if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out) return cerrors.ErrCouldNotDownload } @@ -184,8 +183,8 @@ func (fetcher *UbuntuFetcher) pullRepository() (err error) { } // Pull repository. - if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil { - os.RemoveAll(fetcher.repositoryLocalPath) + if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil { + os.RemoveAll(u.repositoryLocalPath) log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out) return cerrors.ErrCouldNotDownload @@ -217,14 +216,14 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat d, err := os.Open(repositoryLocalPath + "/" + folder) if err != nil { log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err) - return nil, ErrFilesystem + return nil, vulnsrc.ErrFilesystem } // Get the FileInfo of all the files in the directory. names, err := d.Readdirnames(-1) if err != nil { log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err) - return nil, ErrFilesystem + return nil, vulnsrc.ErrFilesystem } // Add the vulnerabilities to the list. @@ -414,8 +413,3 @@ func ubuntuPriorityToSeverity(priority string) types.Priority { log.Warning("Could not determine a vulnerability priority from: %s", priority) return types.Unknown } - -// Clean deletes any allocated resources. -func (fetcher *UbuntuFetcher) Clean() { - os.RemoveAll(fetcher.repositoryLocalPath) -} diff --git a/updater/fetchers/ubuntu/ubuntu_test.go b/ext/vulnsrc/ubuntu/ubuntu_test.go similarity index 100% rename from updater/fetchers/ubuntu/ubuntu_test.go rename to ext/vulnsrc/ubuntu/ubuntu_test.go diff --git a/notifier/notifier.go b/notifier/notifier.go index 81e480cd..d1fa6b2f 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package notifier fetches notifications from the database and informs the specified remote handler -// about their existences, inviting the third party to actively query the API about it. +// Package notifier fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. package notifier import ( @@ -26,6 +27,7 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" ) @@ -40,8 +42,6 @@ const ( var ( log = capnslog.NewPackageLogger("github.com/coreos/clair", "notifier") - notifiers = make(map[string]Notifier) - promNotifierLatencyMilliseconds = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "clair_notifier_latency_milliseconds", Help: "Time it takes to send a notification after it's been created.", @@ -53,57 +53,29 @@ var ( }, []string{"backend"}) ) -// Notifier represents anything that can transmit notifications. -type Notifier interface { - // Configure attempts to initialize the notifier with the provided configuration. - // It returns whether the notifier is enabled or not. - Configure(*config.NotifierConfig) (bool, error) - // Send informs the existence of the specified notification. - Send(notification database.VulnerabilityNotification) error -} - func init() { prometheus.MustRegister(promNotifierLatencyMilliseconds) prometheus.MustRegister(promNotifierBackendErrorsTotal) } -// RegisterNotifier makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterNotifier(name string, n Notifier) { - if name == "" { - panic("notifier: could not register a Notifier with an empty name") - } - - if n == nil { - panic("notifier: could not register a nil Notifier") - } - - if _, dup := notifiers[name]; dup { - panic("notifier: RegisterNotifier called twice for " + name) - } - - notifiers[name] = n -} - // Run starts the Notifier service. func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) { defer stopper.End() // Configure registered notifiers. - for notifierName, notifier := range notifiers { - if configured, err := notifier.Configure(config); configured { - log.Infof("notifier '%s' configured\n", notifierName) + for senderName, sender := range notification.Senders { + if configured, err := sender.Configure(config); configured { + log.Infof("sender '%s' configured\n", senderName) } else { - delete(notifiers, notifierName) + delete(notification.Senders, senderName) if err != nil { - log.Errorf("could not configure notifier '%s': %s", notifierName, err) + log.Errorf("could not configure notifier '%s': %s", senderName, err) } } } // Do not run the updater if there is no notifier enabled. - if len(notifiers) == 0 { + if len(notification.Senders) == 0 { log.Infof("notifier service is disabled") return } @@ -175,31 +147,31 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } } -func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { +func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { // Send notification. - for notifierName, notifier := range notifiers { + for senderName, sender := range notification.Senders { var attempts int var backOff time.Duration for { // Max attempts exceeded. if attempts >= maxAttempts { - log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts) + log.Infof("giving up on sending notification '%s' via sender '%s': max attempts exceeded (%d)\n", n.Name, senderName, maxAttempts) return false, false } // Backoff. if backOff > 0 { - log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts) + log.Infof("waiting %v before retrying to send notification '%s' via sender '%s' (Attempt %d / %d)\n", backOff, n.Name, senderName, attempts+1, maxAttempts) if !st.Sleep(backOff) { return false, true } } // Send using the current notifier. - if err := notifier.Send(notification); err != nil { + if err := sender.Send(n); err != nil { // Send failed; increase attempts/backoff and retry. - promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc() - log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err) + promNotifierBackendErrorsTotal.WithLabelValues(senderName).Inc() + log.Errorf("could not send notification '%s' via notifier '%s': %v", n.Name, senderName, err) backOff = timeutil.ExpBackoff(backOff, maxBackOff) attempts++ continue @@ -210,6 +182,6 @@ func handleTask(notification database.VulnerabilityNotification, st *utils.Stopp } } - log.Infof("successfully sent notification '%s'\n", notification.Name) + log.Infof("successfully sent notification '%s'\n", n.Name) return true, false } diff --git a/updater/fetchers.go b/updater/fetchers.go deleted file mode 100644 index 609ecd40..00000000 --- a/updater/fetchers.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package updater - -import "github.com/coreos/clair/database" - -var fetchers = make(map[string]Fetcher) - -// Fetcher represents anything that can fetch vulnerabilities. -type Fetcher interface { - // FetchUpdate gets vulnerability updates. - FetchUpdate(database.Datastore) (FetcherResponse, error) - - // Clean deletes any allocated resources. - // It is invoked when Clair stops. - Clean() -} - -// FetcherResponse represents the sum of results of an update. -type FetcherResponse struct { - FlagName string - FlagValue string - Notes []string - Vulnerabilities []database.Vulnerability -} - -// RegisterFetcher makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterFetcher(name string, f Fetcher) { - if name == "" { - panic("updater: could not register a Fetcher with an empty name") - } - - if f == nil { - panic("updater: could not register a nil Fetcher") - } - - if _, dup := fetchers[name]; dup { - panic("updater: RegisterFetcher called twice for " + name) - } - - fetchers[name] = f -} diff --git a/updater/metadata_fetchers.go b/updater/metadata_fetchers.go deleted file mode 100644 index 72e764af..00000000 --- a/updater/metadata_fetchers.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package updater - -import ( - "sync" - - "github.com/coreos/clair/database" -) - -var metadataFetchers = make(map[string]MetadataFetcher) - -type VulnerabilityWithLock struct { - *database.Vulnerability - Lock sync.Mutex -} - -// MetadataFetcher -type MetadataFetcher interface { - // Load runs right before the Updater calls AddMetadata for each vulnerabilities. - Load(database.Datastore) error - - // AddMetadata adds metadata to the given database.Vulnerability. - // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. - AddMetadata(*VulnerabilityWithLock) error - - // Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities. - Unload() - - // Clean deletes any allocated resources. - // It is invoked when Clair stops. - Clean() -} - -// RegisterFetcher makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterMetadataFetcher(name string, f MetadataFetcher) { - if name == "" { - panic("updater: could not register a MetadataFetcher with an empty name") - } - - if f == nil { - panic("updater: could not register a nil MetadataFetcher") - } - - if _, dup := fetchers[name]; dup { - panic("updater: RegisterMetadataFetcher called twice for " + name) - } - - metadataFetchers[name] = f -} diff --git a/updater/metadata_fetchers/nvd/nested_read_closer.go b/updater/metadata_fetchers/nvd/nested_read_closer.go deleted file mode 100644 index 3a99b174..00000000 --- a/updater/metadata_fetchers/nvd/nested_read_closer.go +++ /dev/null @@ -1,19 +0,0 @@ -package nvd - -import "io" - -// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed -// io.ReadCloser. -// It allows chaining io.ReadCloser together and still keep the ability to close them all in a -// simple manner. -type NestedReadCloser struct { - io.Reader - NestedReadClosers []io.ReadCloser -} - -// Close closes the gzip.Reader and the underlying io.ReadCloser. -func (nrc *NestedReadCloser) Close() { - for _, nestedReadCloser := range nrc.NestedReadClosers { - nestedReadCloser.Close() - } -} diff --git a/updater/updater.go b/updater/updater.go index bfb8680b..01feb0c2 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package updater updates the vulnerability database periodically using -// the registered vulnerability fetchers. +// Package updater updates the vulnerability database periodically using the +// registered vulnerability source updaters and vulnerability metadata +// appenders. package updater import ( @@ -22,12 +23,16 @@ import ( "sync" "time" - "github.com/coreos/clair/config" - "github.com/coreos/clair/database" - "github.com/coreos/clair/utils" "github.com/coreos/pkg/capnslog" "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/vulnmdsrc" + "github.com/coreos/clair/ext/vulnsrc" + "github.com/coreos/clair/utils" + "github.com/coreos/clair/utils/types" ) const ( @@ -147,11 +152,11 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S } // Clean resources. - for _, metadataFetcher := range metadataFetchers { - metadataFetcher.Clean() + for _, appenders := range vulnmdsrc.Appenders { + appenders.Clean() } - for _, fetcher := range fetchers { - fetcher.Clean() + for _, updaters := range vulnsrc.Updaters { + updaters.Clean() } log.Info("updater service stopped") @@ -209,10 +214,10 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Fetch updates in parallel. log.Info("fetching vulnerability updates") - var responseC = make(chan *FetcherResponse, 0) - for n, f := range fetchers { - go func(name string, fetcher Fetcher) { - response, err := fetcher.FetchUpdate(datastore) + var responseC = make(chan *vulnsrc.UpdateResponse, 0) + for n, u := range vulnsrc.Updaters { + go func(name string, u vulnsrc.Updater) { + response, err := u.Update(datastore) if err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when fetching update '%s': %s.", name, err) @@ -222,11 +227,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st } responseC <- &response - }(n, f) + }(n, u) } // Collect results of updates. - for i := 0; i < len(fetchers); i++ { + for i := 0; i < len(vulnsrc.Updaters); i++ { resp := <-responseC if resp != nil { vulnerabilities = append(vulnerabilities, doVulnerabilitiesNamespacing(resp.Vulnerabilities)...) @@ -243,42 +248,43 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel. func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability { - if len(metadataFetchers) == 0 { + if len(vulnmdsrc.Appenders) == 0 { return vulnerabilities } log.Info("adding metadata to vulnerabilities") - // Wrap vulnerabilities in VulnerabilityWithLock. - // It ensures that only one metadata fetcher at a time can modify the Metadata map. - vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities)) + // Add a mutex to each vulnerability to ensure that only one appender at a + // time can modify the vulnerability's Metadata map. + lockableVulnerabilities := make([]*lockableVulnerability, 0, len(vulnerabilities)) for i := 0; i < len(vulnerabilities); i++ { - vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{ + lockableVulnerabilities = append(lockableVulnerabilities, &lockableVulnerability{ Vulnerability: &vulnerabilities[i], }) } var wg sync.WaitGroup - wg.Add(len(metadataFetchers)) + wg.Add(len(vulnmdsrc.Appenders)) - for n, f := range metadataFetchers { - go func(name string, metadataFetcher MetadataFetcher) { + for n, a := range vulnmdsrc.Appenders { + go func(name string, appender vulnmdsrc.Appender) { defer wg.Done() - // Load the metadata fetcher. - if err := metadataFetcher.Load(datastore); err != nil { + // Build up a metadata cache. + if err := appender.BuildCache(datastore); err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err) return } - // Add metadata to each vulnerability. - for _, vulnerability := range vulnerabilitiesWithLocks { - metadataFetcher.AddMetadata(vulnerability) + // Append vulnerability metadata to each vulnerability. + for _, vulnerability := range lockableVulnerabilities { + appender.Append(vulnerability.Name, vulnerability.appendFunc) } - metadataFetcher.Unload() - }(n, f) + // Purge the metadata cache. + appender.PurgeCache() + }(n, a) } wg.Wait() @@ -305,6 +311,29 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) { return time.Unix(lastUpdateTS, 0).UTC(), false, nil } +type lockableVulnerability struct { + *database.Vulnerability + sync.Mutex +} + +func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity types.Priority) { + lv.Lock() + defer lv.Unlock() + + // If necessary, initialize the metadata map for the vulnerability. + if lv.Metadata == nil { + lv.Metadata = make(map[string]interface{}) + } + + // Append the metadata. + lv.Metadata[metadataKey] = metadata + + // If necessary, provide a severity for the vulnerability. + if lv.Severity == "" || lv.Severity == types.Unknown { + lv.Severity = severity + } +} + // doVulnerabilitiesNamespacing takes Vulnerabilities that don't have a Namespace and split them // into multiple vulnerabilities that have a Namespace and only contains the FixedIn // FeatureVersions corresponding to their Namespace.