refactor: move updaters and notifier into ext

cmd/clair/main.go

@@ -25,15 +25,14 @@ import (
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/config"
 
-	// Register components
+	// Register extensions.
-	_ "github.com/coreos/clair/notifier/notifiers"
+	_ "github.com/coreos/clair/ext/notification/webhook"
-
+	_ "github.com/coreos/clair/ext/vulnmdsrc/nvd"
-	_ "github.com/coreos/clair/updater/fetchers/alpine"
+	_ "github.com/coreos/clair/ext/vulnsrc/alpine"
-	_ "github.com/coreos/clair/updater/fetchers/debian"
+	_ "github.com/coreos/clair/ext/vulnsrc/debian"
-	_ "github.com/coreos/clair/updater/fetchers/oracle"
+	_ "github.com/coreos/clair/ext/vulnsrc/oracle"
-	_ "github.com/coreos/clair/updater/fetchers/rhel"
+	_ "github.com/coreos/clair/ext/vulnsrc/rhel"
-	_ "github.com/coreos/clair/updater/fetchers/ubuntu"
+	_ "github.com/coreos/clair/ext/vulnsrc/ubuntu"
-	_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"
 
 	_ "github.com/coreos/clair/worker/detectors/data/aci"
 	_ "github.com/coreos/clair/worker/detectors/data/docker"

ext/notification/driver.go

@@ -0,0 +1,65 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package notification fetches notifications from the database and informs the
+// specified remote handler about their existences, inviting the third party to
+// actively query the API about it.
+
+// Package notification exposes functions to dynamically register methods to
+// deliver notifications from the Clair database.
+package notification
+
+import (
+	"github.com/coreos/pkg/capnslog"
+
+	"github.com/coreos/clair/config"
+	"github.com/coreos/clair/database"
+)
+
+var (
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/notification")
+
+	// Senders is the list of registered Senders.
+	Senders = make(map[string]Sender)
+)
+
+// Sender represents anything that can transmit notifications.
+type Sender interface {
+	// Configure attempts to initialize the notifier with the provided configuration.
+	// It returns whether the notifier is enabled or not.
+	Configure(*config.NotifierConfig) (bool, error)
+
+	// Send informs the existence of the specified notification.
+	Send(notification database.VulnerabilityNotification) error
+}
+
+// RegisterSender makes a Sender available by the provided name.
+//
+// If RegisterSender is called twice with the same name, the name is blank, or
+// if the provided Sender is nil, this function panics.
+func RegisterSender(name string, s Sender) {
+	if name == "" {
+		panic("notification: could not register a Sender with an empty name")
+	}
+
+	if s == nil {
+		panic("notification: could not register a nil Sender")
+	}
+
+	if _, dup := Senders[name]; dup {
+		panic("notification: RegisterSender called twice for " + name)
+	}
+
+	Senders[name] = s
+}

ext/notification/webhook/webhook.go

@@ -1,4 +1,4 @@
-// Copyright 2015 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package notifiers implements several kinds of notifier.Notifier
+// Package webhook implements a notification sender for HTTP JSON webhooks.
-package notifiers
+package webhook
 
 import (
 	"bytes"
@@ -31,19 +31,18 @@ import (
 
 	"github.com/coreos/clair/config"
 	"github.com/coreos/clair/database"
-	"github.com/coreos/clair/notifier"
+	"github.com/coreos/clair/ext/notification"
 )
 
 const timeout = 5 * time.Second
 
-// A WebhookNotifier dispatches notifications to a webhook endpoint.
+type sender struct {
-type WebhookNotifier struct {
 	endpoint string
 	client   *http.Client
 }
 
-// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier.
+// Config represents the configuration of a Webhook Sender.
-type WebhookNotifierConfiguration struct {
+type Config struct {
 	Endpoint   string
 	ServerName string
 	CertFile   string
@@ -53,12 +52,12 @@ type WebhookNotifierConfiguration struct {
 }
 
 func init() {
-	notifier.RegisterNotifier("webhook", &WebhookNotifier{})
+	notification.RegisterSender("webhook", &sender{})
 }
 
-func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) {
+func (s *sender) Configure(config *config.NotifierConfig) (bool, error) {
 	// Get configuration
-	var httpConfig WebhookNotifierConfiguration
+	var httpConfig Config
 	if config == nil {
 		return false, nil
 	}
@@ -81,11 +80,11 @@ func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error)
 	if _, err := url.ParseRequestURI(httpConfig.Endpoint); err != nil {
 		return false, fmt.Errorf("could not parse endpoint URL: %s\n", err)
 	}
-	h.endpoint = httpConfig.Endpoint
+	s.endpoint = httpConfig.Endpoint
 
 	// Setup HTTP client.
 	transport := &http.Transport{}
-	h.client = &http.Client{
+	s.client = &http.Client{
 		Transport: transport,
 		Timeout:   timeout,
 	}
@@ -114,7 +113,7 @@ type notificationEnvelope struct {
 	}
 }
 
-func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error {
+func (s *sender) Send(notification database.VulnerabilityNotification) error {
 	// Marshal notification.
 	jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}})
 	if err != nil {
@@ -122,7 +121,7 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification)
 	}
 
 	// Send notification via HTTP POST.
-	resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
+	resp, err := s.client.Post(s.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
 	if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) {
 		if resp != nil {
 			return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode)
@@ -134,11 +133,11 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification)
 	return nil
 }
 
-// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration.
+// loadTLSClientConfig initializes a *tls.Config using the given Config.
 //
 // If no certificates are given, (nil, nil) is returned.
 // The CA certificate is optional and falls back to the system default.
-func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) {
+func loadTLSClientConfig(cfg *Config) (*tls.Config, error) {
 	if cfg.CertFile == "" || cfg.KeyFile == "" {
 		return nil, nil
 	}

ext/vulnmdsrc/driver.go

@@ -0,0 +1,68 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package vulnmdsrc exposes functions to dynamically register vulnerability
+// metadata sources used to update a Clair database.
+package vulnmdsrc
+
+import (
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+)
+
+// Appenders is the list of registered Appenders.
+var Appenders = make(map[string]Appender)
+
+// AppendFunc is the type of a callback provided to an Appender.
+type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority)
+
+// Appender represents anything that can fetch vulnerability metadata and
+// append it to a Vulnerability.
+type Appender interface {
+	// BuildCache loads metadata into memory such that it can be quickly accessed
+	// for future calls to Append.
+	BuildCache(database.Datastore) error
+
+	// AddMetadata adds metadata to the given database.Vulnerability.
+	// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
+	// Append
+	Append(vulnName string, callback AppendFunc) error
+
+	// PurgeCache deallocates metadata from memory after all calls to Append are
+	// finished.
+	PurgeCache()
+
+	// Clean deletes any allocated resources.
+	// It is invoked when Clair stops.
+	Clean()
+}
+
+// RegisterAppender makes an Appender available by the provided name.
+// If Register is called twice with the same name or if driver is nil,
+// it panics.
+func RegisterAppender(name string, a Appender) {
+	if name == "" {
+		panic("updater: could not register an Appender with an empty name")
+	}
+
+	if a == nil {
+		panic("vulnmdsrc: could not register a nil Appender")
+	}
+
+	if _, dup := Appenders[name]; dup {
+		panic("vulnmdsrc: RegisterAppender called twice for " + name)
+	}
+
+	Appenders[name] = a
+}

ext/vulnmdsrc/nvd/nested_read_closer.go

@@ -0,0 +1,33 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nvd
+
+import "io"
+
+// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
+// io.ReadCloser.
+// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
+// simple manner.
+type NestedReadCloser struct {
+	io.Reader
+	NestedReadClosers []io.ReadCloser
+}
+
+// Close closes the gzip.Reader and the underlying io.ReadCloser.
+func (nrc *NestedReadCloser) Close() {
+	for _, nestedReadCloser := range nrc.NestedReadClosers {
+		nestedReadCloser.Close()
+	}
+}

ext/vulnmdsrc/nvd/nvd.go

@@ -1,3 +1,19 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package nvd implements a vulnerability metadata appender using the NIST NVD
+// database.
 package nvd
 
 import (
@@ -15,30 +31,28 @@ import (
 	"sync"
 	"time"
 
+	"github.com/coreos/pkg/capnslog"
+
 	"github.com/coreos/clair/database"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnmdsrc"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
-	"github.com/coreos/pkg/capnslog"
 )
 
 const (
 	dataFeedURL     string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz"
 	dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta"
 
-	metadataKey string = "NVD"
+	appenderName string = "NVD"
 )
 
-var (
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnmdsrc/nvd")
-	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers")
-)
 
-type NVDMetadataFetcher struct {
+type appender struct {
 	localPath      string
 	dataFeedHashes map[string]string
-	lock           sync.Mutex
-
 	metadata       map[string]NVDMetadata
+	sync.Mutex
 }
 
 type NVDMetadata struct {
@@ -51,32 +65,32 @@ type NVDmetadataCVSSv2 struct {
 }
 
 func init() {
-	updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{})
+	vulnmdsrc.RegisterAppender(appenderName, &appender{})
 }
 
-func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
+func (a *appender) BuildCache(datastore database.Datastore) error {
-	fetcher.lock.Lock()
+	a.Lock()
-	defer fetcher.lock.Unlock()
+	defer a.Unlock()
 
 	var err error
-	fetcher.metadata = make(map[string]NVDMetadata)
+	a.metadata = make(map[string]NVDMetadata)
 
 	// Init if necessary.
-	if fetcher.localPath == "" {
+	if a.localPath == "" {
 		// Create a temporary folder to store the NVD data and create hashes struct.
-		if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil {
+		if a.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil {
 			return cerrors.ErrFilesystem
 		}
 
-		fetcher.dataFeedHashes = make(map[string]string)
+		a.dataFeedHashes = make(map[string]string)
 	}
 
 	// Get data feeds.
-	dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath)
+	dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath)
 	if err != nil {
 		return err
 	}
-	fetcher.dataFeedHashes = dataFeedHashes
+	a.dataFeedHashes = dataFeedHashes
 
 	// Parse data feeds.
 	for dataFeedName, dataFeedReader := range dataFeedReaders {
@@ -90,7 +104,7 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
 		for _, nvdEntry := range nvd.Entries {
 			// Create metadata entry.
 			if metadata := nvdEntry.Metadata(); metadata != nil {
-				fetcher.metadata[nvdEntry.Name] = *metadata
+				a.metadata[nvdEntry.Name] = *metadata
 			}
 		}
 
@@ -100,42 +114,29 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
 	return nil
 }
 
-func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error {
+func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error {
-	fetcher.lock.Lock()
+	a.Lock()
-	defer fetcher.lock.Unlock()
+	defer a.Unlock()
 
-	if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok {
+	if nvdMetadata, ok := a.metadata[vulnName]; ok {
-		vulnerability.Lock.Lock()
+		appendFunc(appenderName, nvdMetadata, scoreToPriority(nvdMetadata.CVSSv2.Score))
-
-		// Create Metadata map if necessary and assign the NVD metadata.
-		if vulnerability.Metadata == nil {
-			vulnerability.Metadata = make(map[string]interface{})
-		}
-		vulnerability.Metadata[metadataKey] = nvdMetadata
-
-		// Set the Severity using the CVSSv2 Score if none is set yet.
-		if vulnerability.Severity == "" || vulnerability.Severity == types.Unknown {
-			vulnerability.Severity = scoreToPriority(nvdMetadata.CVSSv2.Score)
-		}
-
-		vulnerability.Lock.Unlock()
 	}
 
 	return nil
 }
 
-func (fetcher *NVDMetadataFetcher) Unload() {
+func (a *appender) PurgeCache() {
-	fetcher.lock.Lock()
+	a.Lock()
-	defer fetcher.lock.Unlock()
+	defer a.Unlock()
 
-	fetcher.metadata = nil
+	a.metadata = nil
 }
 
-func (fetcher *NVDMetadataFetcher) Clean() {
+func (a *appender) Clean() {
-	fetcher.lock.Lock()
+	a.Lock()
-	defer fetcher.lock.Unlock()
+	defer a.Unlock()
 
-	os.RemoveAll(fetcher.localPath)
+	os.RemoveAll(a.localPath)
 }
 
 func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) {

ext/vulnmdsrc/nvd/xml.go

@@ -1,3 +1,17 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package nvd
 
 import (

ext/vulnsrc/alpine/alpine.go

@@ -1,4 +1,4 @@
-// Copyright 2016 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package alpine implements a vulnerability Fetcher using the alpine-secdb
+// Package alpine implements a vulnerability source updater using the
-// git repository.
+// alpine-secdb git repository.
 package alpine
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -31,7 +30,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/utils"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
@@ -45,29 +44,23 @@ const (
 )
 
 var (
-	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine")
-	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")
-
-	// ErrGitFailure is returned when a fetcher fails to interact with git.
-	ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")
-
-	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
 )
 
 func init() {
-	updater.RegisterFetcher("alpine", &fetcher{})
+	vulnsrc.RegisterUpdater("alpine", &updater{})
 }
 
-type fetcher struct {
+type updater struct {
 	repositoryLocalPath string
 }
 
-func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
+func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching Alpine vulnerabilities")
 
 	// Pull the master branch.
 	var commit string
-	commit, err = f.pullRepository()
+	commit, err = u.pullRepository()
 	if err != nil {
 		return
 	}
@@ -90,12 +83,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon
 	}
 
 	var namespaces []string
-	namespaces, err = detectNamespaces(f.repositoryLocalPath)
+	namespaces, err = detectNamespaces(u.repositoryLocalPath)
 	// Append any changed vulnerabilities to the response.
 	for _, namespace := range namespaces {
 		var vulns []database.Vulnerability
 		var note string
-		vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace)
+		vulns, note, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace)
 		if err != nil {
 			return
 		}
@@ -108,6 +101,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon
 	return
 }
 
+func (u *updater) Clean() {
+	if u.repositoryLocalPath != "" {
+		os.RemoveAll(u.repositoryLocalPath)
+	}
+}
+
 func detectNamespaces(path string) ([]string, error) {
 	// Open the root directory.
 	dir, err := os.Open(path)
@@ -163,41 +162,35 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
 	return
 }
 
-func (f *fetcher) pullRepository() (commit string, err error) {
+func (u *updater) pullRepository() (commit string, err error) {
 	// If the repository doesn't exist, clone it.
-	if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
+	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
-		if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
+		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
-			return "", ErrFilesystem
+			return "", vulnsrc.ErrFilesystem
 		}
 
-		if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
+		if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
-			f.Clean()
+			u.Clean()
 			log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
 			return "", cerrors.ErrCouldNotDownload
 		}
 	} else {
 		// The repository exists and it needs to be refreshed via a pull.
-		_, err := utils.Exec(f.repositoryLocalPath, "git", "pull")
+		_, err := utils.Exec(u.repositoryLocalPath, "git", "pull")
 		if err != nil {
-			return "", ErrGitFailure
+			return "", vulnsrc.ErrGitFailure
 		}
 	}
 
-	out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
+	out, err := utils.Exec(u.repositoryLocalPath, "git", "rev-parse", "HEAD")
 	if err != nil {
-		return "", ErrGitFailure
+		return "", vulnsrc.ErrGitFailure
 	}
 
 	commit = strings.TrimSpace(string(out))
 	return
 }
 
-func (f *fetcher) Clean() {
-	if f.repositoryLocalPath != "" {
-		os.RemoveAll(f.repositoryLocalPath)
-	}
-}
-
 type secdb33File struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {

ext/vulnsrc/debian/debian.go

@@ -1,4 +1,4 @@
-// Copyright 2016 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package debian implements a vulnerability source updater using the Debian
+// Security Tracker.
 package debian
 
 import (
@@ -28,7 +30,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnsrc"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
 )
@@ -39,7 +41,7 @@ const (
 	updaterFlag  = "debianUpdater"
 )
 
-var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian")
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/debian")
 
 type jsonData map[string]map[string]jsonVuln
 
@@ -54,16 +56,13 @@ type jsonRel struct {
 	Urgency      string `json:"urgency"`
 }
 
-// DebianFetcher implements updater.Fetcher for the Debian Security Tracker
+type updater struct{}
-// (https://security-tracker.debian.org).
-type DebianFetcher struct{}
 
 func init() {
-	updater.RegisterFetcher("debian", &DebianFetcher{})
+	vulnsrc.RegisterUpdater("debian", &updater{})
 }
 
-// FetchUpdate fetches vulnerability updates from the Debian Security Tracker.
+func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
-func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
 	log.Info("fetching Debian vulnerabilities")
 
 	// Download JSON.
@@ -88,7 +87,9 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up
 	return resp, nil
 }
 
-func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.FetcherResponse, err error) {
+func (u *updater) Clean() {}
+
+func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.UpdateResponse, err error) {
 	hash := latestKnownHash
 
 	// Defer the addition of flag information to the response.
@@ -254,6 +255,3 @@ func urgencyToSeverity(urgency string) types.Priority {
 		return types.Unknown
 	}
 }
-
-// Clean deletes any allocated resources.
-func (fetcher *DebianFetcher) Clean() {}

ext/vulnsrc/driver.go

@@ -0,0 +1,73 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package vulnsrc exposes functions to dynamically register vulnerability
+// sources used to update a Clair database.
+package vulnsrc
+
+import (
+	"errors"
+
+	"github.com/coreos/clair/database"
+)
+
+var (
+	// Updaters is the list of registered Updaters.
+	Updaters = make(map[string]Updater)
+
+	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
+	ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")
+
+	// ErrGitFailure is returned when a fetcher fails to interact with git.
+	ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")
+)
+
+// Updater represents anything that can fetch vulnerabilities and insert them
+// into a Clair datastore.
+type Updater interface {
+	// Update gets vulnerability updates.
+	Update(database.Datastore) (UpdateResponse, error)
+
+	// Clean deletes any allocated resources.
+	// It is invoked when Clair stops.
+	Clean()
+}
+
+// UpdateResponse represents the sum of results of an update.
+type UpdateResponse struct {
+	FlagName        string
+	FlagValue       string
+	Notes           []string
+	Vulnerabilities []database.Vulnerability
+}
+
+// RegisterUpdater makes an Updater available by the provided name.
+//
+// If RegisterUpdater is called twice with the same name, the name is blank, or
+// if the provided Updater is nil, this function panics.
+func RegisterUpdater(name string, u Updater) {
+	if name == "" {
+		panic("vulnsrc: could not register an Updater with an empty name")
+	}
+
+	if u == nil {
+		panic("vulnsrc: could not register a nil Updater")
+	}
+
+	if _, dup := Updaters[name]; dup {
+		panic("vulnsrc: RegisterUpdater called twice for " + name)
+	}
+
+	Updaters[name] = u
+}

ext/vulnsrc/oracle/oracle.go

@@ -1,4 +1,4 @@
-// Copyright 2016 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package oracle implements a vulnerability source updater using the
+// Oracle Linux OVAL Database.
 package oracle
 
 import (
@@ -26,7 +28,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/rpm"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnsrc"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
 	"github.com/coreos/pkg/capnslog"
@@ -47,7 +49,7 @@ var (
 
 	elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`)
 
-	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/oracle")
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/oracle")
 )
 
 type oval struct {
@@ -77,16 +79,13 @@ type criterion struct {
 	Comment string `xml:"comment,attr"`
 }
 
-// OracleFetcher implements updater.Fetcher and gets vulnerability updates from
+type updater struct{}
-// the Oracle Linux OVAL definitions.
-type OracleFetcher struct{}
 
 func init() {
-	updater.RegisterFetcher("Oracle", &OracleFetcher{})
+	vulnsrc.RegisterUpdater("oracle", &updater{})
 }
 
-// FetchUpdate gets vulnerability updates from the Oracle Linux OVAL definitions.
+func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
-func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
 	log.Info("fetching Oracle Linux vulnerabilities")
 
 	// Get the first ELSA we have to manage.
@@ -153,6 +152,8 @@ func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.
 	return resp, nil
 }
 
+func (u *updater) Clean() {}
+
 func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
 	// Decode the XML.
 	var ov oval
@@ -356,6 +357,3 @@ func priority(def definition) types.Priority {
 		return types.Unknown
 	}
 }
-
-// Clean deletes any allocated resources.
-func (f *OracleFetcher) Clean() {}

ext/vulnsrc/rhel/rhel.go

@@ -1,4 +1,4 @@
-// Copyright 2016 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package rhel implements a vulnerability source updater using the
+// Red Hat Linux OVAL Database.
 package rhel
 
 import (
@@ -28,7 +30,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/rpm"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnsrc"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
 )
@@ -82,17 +84,14 @@ type criterion struct {
 	Comment string `xml:"comment,attr"`
 }
 
-// RHELFetcher implements updater.Fetcher and gets vulnerability updates from
+type updater struct{}
-// the Red Hat OVAL definitions.
-type RHELFetcher struct{}
 
 func init() {
-	updater.RegisterFetcher("Red Hat", &RHELFetcher{})
+	vulnsrc.RegisterUpdater("rhel", &updater{})
 }
 
-// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions.
+func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
-func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
+	log.Info("fetching RHEL vulnerabilities")
-	log.Info("fetching Red Hat vulnerabilities")
 
 	// Get the first RHSA we have to manage.
 	flagValue, err := datastore.GetKeyValue(updaterFlag)
@@ -156,6 +155,8 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe
 	return resp, nil
 }
 
+func (u *updater) Clean() {}
+
 func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
 	// Decode the XML.
 	var ov oval
@@ -362,6 +363,3 @@ func priority(def definition) types.Priority {
 		return types.Unknown
 	}
 }
-
-// Clean deletes any allocated resources.
-func (f *RHELFetcher) Clean() {}

ext/vulnsrc/ubuntu/ubuntu.go

@@ -1,4 +1,4 @@
-// Copyright 2015 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package ubuntu implements a vulnerability source updater using the
+// Ubuntu CVE Tracker.
 package ubuntu
 
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -31,7 +32,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
-	"github.com/coreos/clair/updater"
+	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/utils"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
@@ -75,33 +76,27 @@ var (
 	affectsCaptureRegexp      = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
 	affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()
 
-	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu")
+	log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/ubuntu")
-
-	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
-	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")
 )
 
-// UbuntuFetcher implements updater.Fetcher and gets vulnerability updates from
+type updater struct {
-// the Ubuntu CVE Tracker.
-type UbuntuFetcher struct {
 	repositoryLocalPath string
 }
 
 func init() {
-	updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{})
+	vulnsrc.RegisterUpdater("ubuntu", &updater{})
 }
 
-// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
+func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
-func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
 	log.Info("fetching Ubuntu vulnerabilities")
 
 	// Pull the bzr repository.
-	if err = fetcher.pullRepository(); err != nil {
+	if err = u.pullRepository(); err != nil {
 		return resp, err
 	}
 
 	// Get revision number.
-	revisionNumber, err := getRevisionNumber(fetcher.repositoryLocalPath)
+	revisionNumber, err := getRevisionNumber(u.repositoryLocalPath)
 	if err != nil {
 		return resp, err
 	}
@@ -113,7 +108,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
 	}
 
 	// Get the list of vulnerabilities that we have to update.
-	modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, fetcher.repositoryLocalPath)
+	modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath)
 	if err != nil {
 		return resp, err
 	}
@@ -121,7 +116,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
 	notes := make(map[string]struct{})
 	for cvePath := range modifiedCVE {
 		// Open the CVE file.
-		file, err := os.Open(fetcher.repositoryLocalPath + "/" + cvePath)
+		file, err := os.Open(u.repositoryLocalPath + "/" + cvePath)
 		if err != nil {
 			// This can happen when a file is modified and then moved in another
 			// commit.
@@ -166,16 +161,20 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
 	return
 }
 
-func (fetcher *UbuntuFetcher) pullRepository() (err error) {
+func (u *updater) Clean() {
+	os.RemoveAll(u.repositoryLocalPath)
+}
+
+func (u *updater) pullRepository() (err error) {
 	// Determine whether we should branch or pull.
-	if _, pathExists := os.Stat(fetcher.repositoryLocalPath); fetcher.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
+	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
 		// Create a temporary folder to store the repository.
-		if fetcher.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
+		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
-			return ErrFilesystem
+			return vulnsrc.ErrFilesystem
 		}
 
 		// Branch repository.
-		if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil {
+		if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil {
 			log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out)
 			return cerrors.ErrCouldNotDownload
 		}
@@ -184,8 +183,8 @@ func (fetcher *UbuntuFetcher) pullRepository() (err error) {
 	}
 
 	// Pull repository.
-	if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil {
+	if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil {
-		os.RemoveAll(fetcher.repositoryLocalPath)
+		os.RemoveAll(u.repositoryLocalPath)
 
 		log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out)
 		return cerrors.ErrCouldNotDownload
@@ -217,14 +216,14 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat
 			d, err := os.Open(repositoryLocalPath + "/" + folder)
 			if err != nil {
 				log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err)
-				return nil, ErrFilesystem
+				return nil, vulnsrc.ErrFilesystem
 			}
 
 			// Get the FileInfo of all the files in the directory.
 			names, err := d.Readdirnames(-1)
 			if err != nil {
 				log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err)
-				return nil, ErrFilesystem
+				return nil, vulnsrc.ErrFilesystem
 			}
 
 			// Add the vulnerabilities to the list.
@@ -414,8 +413,3 @@ func ubuntuPriorityToSeverity(priority string) types.Priority {
 	log.Warning("Could not determine a vulnerability priority from: %s", priority)
 	return types.Unknown
 }
-
-// Clean deletes any allocated resources.
-func (fetcher *UbuntuFetcher) Clean() {
-	os.RemoveAll(fetcher.repositoryLocalPath)
-}

notifier/notifier.go

@@ -1,4 +1,4 @@
-// Copyright 2015 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package notifier fetches notifications from the database and informs the specified remote handler
+// Package notifier fetches notifications from the database and informs the
-// about their existences, inviting the third party to actively query the API about it.
+// specified remote handler about their existences, inviting the third party to
+// actively query the API about it.
 package notifier
 
 import (
@@ -26,6 +27,7 @@ import (
 
 	"github.com/coreos/clair/config"
 	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/notification"
 	"github.com/coreos/clair/utils"
 	cerrors "github.com/coreos/clair/utils/errors"
 )
@@ -40,8 +42,6 @@ const (
 var (
 	log = capnslog.NewPackageLogger("github.com/coreos/clair", "notifier")
 
-	notifiers = make(map[string]Notifier)
-
 	promNotifierLatencyMilliseconds = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Name: "clair_notifier_latency_milliseconds",
 		Help: "Time it takes to send a notification after it's been created.",
@@ -53,57 +53,29 @@ var (
 	}, []string{"backend"})
 )
 
-// Notifier represents anything that can transmit notifications.
-type Notifier interface {
-	// Configure attempts to initialize the notifier with the provided configuration.
-	// It returns whether the notifier is enabled or not.
-	Configure(*config.NotifierConfig) (bool, error)
-	// Send informs the existence of the specified notification.
-	Send(notification database.VulnerabilityNotification) error
-}
-
 func init() {
 	prometheus.MustRegister(promNotifierLatencyMilliseconds)
 	prometheus.MustRegister(promNotifierBackendErrorsTotal)
 }
 
-// RegisterNotifier makes a Fetcher available by the provided name.
-// If Register is called twice with the same name or if driver is nil,
-// it panics.
-func RegisterNotifier(name string, n Notifier) {
-	if name == "" {
-		panic("notifier: could not register a Notifier with an empty name")
-	}
-
-	if n == nil {
-		panic("notifier: could not register a nil Notifier")
-	}
-
-	if _, dup := notifiers[name]; dup {
-		panic("notifier: RegisterNotifier called twice for " + name)
-	}
-
-	notifiers[name] = n
-}
-
 // Run starts the Notifier service.
 func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) {
 	defer stopper.End()
 
 	// Configure registered notifiers.
-	for notifierName, notifier := range notifiers {
+	for senderName, sender := range notification.Senders {
-		if configured, err := notifier.Configure(config); configured {
+		if configured, err := sender.Configure(config); configured {
-			log.Infof("notifier '%s' configured\n", notifierName)
+			log.Infof("sender '%s' configured\n", senderName)
 		} else {
-			delete(notifiers, notifierName)
+			delete(notification.Senders, senderName)
 			if err != nil {
-				log.Errorf("could not configure notifier '%s': %s", notifierName, err)
+				log.Errorf("could not configure notifier '%s': %s", senderName, err)
 			}
 		}
 	}
 
 	// Do not run the updater if there is no notifier enabled.
-	if len(notifiers) == 0 {
+	if len(notification.Senders) == 0 {
 		log.Infof("notifier service is disabled")
 		return
 	}
@@ -175,31 +147,31 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA
 	}
 }
 
-func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
+func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
 	// Send notification.
-	for notifierName, notifier := range notifiers {
+	for senderName, sender := range notification.Senders {
 		var attempts int
 		var backOff time.Duration
 		for {
 			// Max attempts exceeded.
 			if attempts >= maxAttempts {
-				log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts)
+				log.Infof("giving up on sending notification '%s' via sender '%s': max attempts exceeded (%d)\n", n.Name, senderName, maxAttempts)
 				return false, false
 			}
 
 			// Backoff.
 			if backOff > 0 {
-				log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts)
+				log.Infof("waiting %v before retrying to send notification '%s' via sender '%s' (Attempt %d / %d)\n", backOff, n.Name, senderName, attempts+1, maxAttempts)
 				if !st.Sleep(backOff) {
 					return false, true
 				}
 			}
 
 			// Send using the current notifier.
-			if err := notifier.Send(notification); err != nil {
+			if err := sender.Send(n); err != nil {
 				// Send failed; increase attempts/backoff and retry.
-				promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc()
+				promNotifierBackendErrorsTotal.WithLabelValues(senderName).Inc()
-				log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err)
+				log.Errorf("could not send notification '%s' via notifier '%s': %v", n.Name, senderName, err)
 				backOff = timeutil.ExpBackoff(backOff, maxBackOff)
 				attempts++
 				continue
@@ -210,6 +182,6 @@ func handleTask(notification database.VulnerabilityNotification, st *utils.Stopp
 		}
 	}
 
-	log.Infof("successfully sent notification '%s'\n", notification.Name)
+	log.Infof("successfully sent notification '%s'\n", n.Name)
 	return true, false
 }

updater/fetchers.go

@@ -1,56 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package updater
-
-import "github.com/coreos/clair/database"
-
-var fetchers = make(map[string]Fetcher)
-
-// Fetcher represents anything that can fetch vulnerabilities.
-type Fetcher interface {
-	// FetchUpdate gets vulnerability updates.
-	FetchUpdate(database.Datastore) (FetcherResponse, error)
-
-	// Clean deletes any allocated resources.
-	// It is invoked when Clair stops.
-	Clean()
-}
-
-// FetcherResponse represents the sum of results of an update.
-type FetcherResponse struct {
-	FlagName        string
-	FlagValue       string
-	Notes           []string
-	Vulnerabilities []database.Vulnerability
-}
-
-// RegisterFetcher makes a Fetcher available by the provided name.
-// If Register is called twice with the same name or if driver is nil,
-// it panics.
-func RegisterFetcher(name string, f Fetcher) {
-	if name == "" {
-		panic("updater: could not register a Fetcher with an empty name")
-	}
-
-	if f == nil {
-		panic("updater: could not register a nil Fetcher")
-	}
-
-	if _, dup := fetchers[name]; dup {
-		panic("updater: RegisterFetcher called twice for " + name)
-	}
-
-	fetchers[name] = f
-}

updater/metadata_fetchers.go

@@ -1,64 +0,0 @@
-// Copyright 2015 clair authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package updater
-
-import (
-	"sync"
-
-	"github.com/coreos/clair/database"
-)
-
-var metadataFetchers = make(map[string]MetadataFetcher)
-
-type VulnerabilityWithLock struct {
-	*database.Vulnerability
-	Lock sync.Mutex
-}
-
-// MetadataFetcher
-type MetadataFetcher interface {
-	// Load runs right before the Updater calls AddMetadata for each vulnerabilities.
-	Load(database.Datastore) error
-
-	// AddMetadata adds metadata to the given database.Vulnerability.
-	// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
-	AddMetadata(*VulnerabilityWithLock) error
-
-	// Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities.
-	Unload()
-
-	// Clean deletes any allocated resources.
-	// It is invoked when Clair stops.
-	Clean()
-}
-
-// RegisterFetcher makes a Fetcher available by the provided name.
-// If Register is called twice with the same name or if driver is nil,
-// it panics.
-func RegisterMetadataFetcher(name string, f MetadataFetcher) {
-	if name == "" {
-		panic("updater: could not register a MetadataFetcher with an empty name")
-	}
-
-	if f == nil {
-		panic("updater: could not register a nil MetadataFetcher")
-	}
-
-	if _, dup := fetchers[name]; dup {
-		panic("updater: RegisterMetadataFetcher called twice for " + name)
-	}
-
-	metadataFetchers[name] = f
-}

updater/metadata_fetchers/nvd/nested_read_closer.go

@@ -1,19 +0,0 @@
-package nvd
-
-import "io"
-
-// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
-// io.ReadCloser.
-// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
-// simple manner.
-type NestedReadCloser struct {
-	io.Reader
-	NestedReadClosers []io.ReadCloser
-}
-
-// Close closes the gzip.Reader and the underlying io.ReadCloser.
-func (nrc *NestedReadCloser) Close() {
-	for _, nestedReadCloser := range nrc.NestedReadClosers {
-		nestedReadCloser.Close()
-	}
-}

updater/updater.go

@@ -1,4 +1,4 @@
-// Copyright 2015 clair authors
+// Copyright 2017 clair authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,8 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package updater updates the vulnerability database periodically using
+// Package updater updates the vulnerability database periodically using the
-// the registered vulnerability fetchers.
+// registered vulnerability source updaters and vulnerability metadata
+// appenders.
 package updater
 
 import (
@@ -22,12 +23,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/clair/config"
-	"github.com/coreos/clair/database"
-	"github.com/coreos/clair/utils"
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pborman/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/coreos/clair/config"
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/vulnmdsrc"
+	"github.com/coreos/clair/ext/vulnsrc"
+	"github.com/coreos/clair/utils"
+	"github.com/coreos/clair/utils/types"
 )
 
 const (
@@ -147,11 +152,11 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
 	}
 
 	// Clean resources.
-	for _, metadataFetcher := range metadataFetchers {
+	for _, appenders := range vulnmdsrc.Appenders {
-		metadataFetcher.Clean()
+		appenders.Clean()
 	}
-	for _, fetcher := range fetchers {
+	for _, updaters := range vulnsrc.Updaters {
-		fetcher.Clean()
+		updaters.Clean()
 	}
 
 	log.Info("updater service stopped")
@@ -209,10 +214,10 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
 
 	// Fetch updates in parallel.
 	log.Info("fetching vulnerability updates")
-	var responseC = make(chan *FetcherResponse, 0)
+	var responseC = make(chan *vulnsrc.UpdateResponse, 0)
-	for n, f := range fetchers {
+	for n, u := range vulnsrc.Updaters {
-		go func(name string, fetcher Fetcher) {
+		go func(name string, u vulnsrc.Updater) {
-			response, err := fetcher.FetchUpdate(datastore)
+			response, err := u.Update(datastore)
 			if err != nil {
 				promUpdaterErrorsTotal.Inc()
 				log.Errorf("an error occured when fetching update '%s': %s.", name, err)
@@ -222,11 +227,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
 			}
 
 			responseC <- &response
-		}(n, f)
+		}(n, u)
 	}
 
 	// Collect results of updates.
-	for i := 0; i < len(fetchers); i++ {
+	for i := 0; i < len(vulnsrc.Updaters); i++ {
 		resp := <-responseC
 		if resp != nil {
 			vulnerabilities = append(vulnerabilities, doVulnerabilitiesNamespacing(resp.Vulnerabilities)...)
@@ -243,42 +248,43 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
 
 // Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel.
 func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability {
-	if len(metadataFetchers) == 0 {
+	if len(vulnmdsrc.Appenders) == 0 {
 		return vulnerabilities
 	}
 
 	log.Info("adding metadata to vulnerabilities")
 
-	// Wrap vulnerabilities in VulnerabilityWithLock.
+	// Add a mutex to each vulnerability to ensure that only one appender at a
-	// It ensures that only one metadata fetcher at a time can modify the Metadata map.
+	// time can modify the vulnerability's Metadata map.
-	vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities))
+	lockableVulnerabilities := make([]*lockableVulnerability, 0, len(vulnerabilities))
 	for i := 0; i < len(vulnerabilities); i++ {
-		vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{
+		lockableVulnerabilities = append(lockableVulnerabilities, &lockableVulnerability{
 			Vulnerability: &vulnerabilities[i],
 		})
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(len(metadataFetchers))
+	wg.Add(len(vulnmdsrc.Appenders))
 
-	for n, f := range metadataFetchers {
+	for n, a := range vulnmdsrc.Appenders {
-		go func(name string, metadataFetcher MetadataFetcher) {
+		go func(name string, appender vulnmdsrc.Appender) {
 			defer wg.Done()
 
-			// Load the metadata fetcher.
+			// Build up a metadata cache.
-			if err := metadataFetcher.Load(datastore); err != nil {
+			if err := appender.BuildCache(datastore); err != nil {
 				promUpdaterErrorsTotal.Inc()
 				log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err)
 				return
 			}
 
-			// Add metadata to each vulnerability.
+			// Append vulnerability metadata  to each vulnerability.
-			for _, vulnerability := range vulnerabilitiesWithLocks {
+			for _, vulnerability := range lockableVulnerabilities {
-				metadataFetcher.AddMetadata(vulnerability)
+				appender.Append(vulnerability.Name, vulnerability.appendFunc)
 			}
 
-			metadataFetcher.Unload()
+			// Purge the metadata cache.
-		}(n, f)
+			appender.PurgeCache()
+		}(n, a)
 	}
 
 	wg.Wait()
@@ -305,6 +311,29 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) {
 	return time.Unix(lastUpdateTS, 0).UTC(), false, nil
 }
 
+type lockableVulnerability struct {
+	*database.Vulnerability
+	sync.Mutex
+}
+
+func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity types.Priority) {
+	lv.Lock()
+	defer lv.Unlock()
+
+	// If necessary, initialize the metadata map for the vulnerability.
+	if lv.Metadata == nil {
+		lv.Metadata = make(map[string]interface{})
+	}
+
+	// Append the metadata.
+	lv.Metadata[metadataKey] = metadata
+
+	// If necessary, provide a severity for the vulnerability.
+	if lv.Severity == "" || lv.Severity == types.Unknown {
+		lv.Severity = severity
+	}
+}
+
 // doVulnerabilitiesNamespacing takes Vulnerabilities that don't have a Namespace and split them
 // into multiple vulnerabilities that have a Namespace and only contains the FixedIn
 // FeatureVersions corresponding to their Namespace.