arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactor: move updaters and notifier into ext

Browse Source
This commit is contained in:
Jimmy Zelinskie 2017-01-03 21:44:32 -05:00
parent f66103c773
commit 4a990372ff
31 changed files with 487 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/clair/main.go
View File

@ -25,15 +25,14 @@ import (
"github.com/coreos/clair"
"github.com/coreos/clair/config"
// Register components
_ "github.com/coreos/clair/notifier/notifiers"
_ "github.com/coreos/clair/updater/fetchers/alpine"
_ "github.com/coreos/clair/updater/fetchers/debian"
_ "github.com/coreos/clair/updater/fetchers/oracle"
_ "github.com/coreos/clair/updater/fetchers/rhel"
_ "github.com/coreos/clair/updater/fetchers/ubuntu"
_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"
// Register extensions.
_ "github.com/coreos/clair/ext/notification/webhook"
_ "github.com/coreos/clair/ext/vulnmdsrc/nvd"
_ "github.com/coreos/clair/ext/vulnsrc/alpine"
_ "github.com/coreos/clair/ext/vulnsrc/debian"
_ "github.com/coreos/clair/ext/vulnsrc/oracle"
_ "github.com/coreos/clair/ext/vulnsrc/rhel"
_ "github.com/coreos/clair/ext/vulnsrc/ubuntu"
_ "github.com/coreos/clair/worker/detectors/data/aci"
_ "github.com/coreos/clair/worker/detectors/data/docker"

65
ext/notification/driver.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package notification fetches notifications from the database and informs the
// specified remote handler about their existences, inviting the third party to
// actively query the API about it.
// Package notification exposes functions to dynamically register methods to
// deliver notifications from the Clair database.
package notification
import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/notification")
// Senders is the list of registered Senders.
Senders = make(map[string]Sender)
)
// Sender represents anything that can transmit notifications.
type Sender interface {
// Configure attempts to initialize the notifier with the provided configuration.
// It returns whether the notifier is enabled or not.
Configure(*config.NotifierConfig) (bool, error)
// Send informs the existence of the specified notification.
Send(notification database.VulnerabilityNotification) error
}
// RegisterSender makes a Sender available by the provided name.
//
// If RegisterSender is called twice with the same name, the name is blank, or
// if the provided Sender is nil, this function panics.
func RegisterSender(name string, s Sender) {
if name == "" {
panic("notification: could not register a Sender with an empty name")
}
if s == nil {
panic("notification: could not register a nil Sender")
}
if _, dup := Senders[name]; dup {
panic("notification: RegisterSender called twice for " + name)
}
Senders[name] = s
}

33
notifier/notifiers/webhook.go → ext/notification/webhook/webhook.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package notifiers implements several kinds of notifier.Notifier
package notifiers
// Package webhook implements a notification sender for HTTP JSON webhooks.
package webhook
import (
"bytes"
@ -31,19 +31,18 @@ import (
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/notifier"
"github.com/coreos/clair/ext/notification"
)
const timeout = 5 * time.Second
// A WebhookNotifier dispatches notifications to a webhook endpoint.
type WebhookNotifier struct {
type sender struct {
endpoint string
client *http.Client
}
// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier.
type WebhookNotifierConfiguration struct {
// Config represents the configuration of a Webhook Sender.
type Config struct {
Endpoint string
ServerName string
CertFile string
@ -53,12 +52,12 @@ type WebhookNotifierConfiguration struct {
}
func init() {
notifier.RegisterNotifier("webhook", &WebhookNotifier{})
notification.RegisterSender("webhook", &sender{})
}
func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) {
func (s *sender) Configure(config *config.NotifierConfig) (bool, error) {
// Get configuration
var httpConfig WebhookNotifierConfiguration
var httpConfig Config
if config == nil {
return false, nil
}
@ -81,11 +80,11 @@ func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error)
if _, err := url.ParseRequestURI(httpConfig.Endpoint); err != nil {
return false, fmt.Errorf("could not parse endpoint URL: %s\n", err)
}
h.endpoint = httpConfig.Endpoint
s.endpoint = httpConfig.Endpoint
// Setup HTTP client.
transport := &http.Transport{}
h.client = &http.Client{
s.client = &http.Client{
Transport: transport,
Timeout: timeout,
}
@ -114,7 +113,7 @@ type notificationEnvelope struct {
}
}
func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error {
func (s *sender) Send(notification database.VulnerabilityNotification) error {
// Marshal notification.
jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}})
if err != nil {
@ -122,7 +121,7 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification)
}
// Send notification via HTTP POST.
resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
resp, err := s.client.Post(s.endpoint, "application/json", bytes.NewBuffer(jsonNotification))
if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) {
if resp != nil {
return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode)
@ -134,11 +133,11 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification)
return nil
}
// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration.
// loadTLSClientConfig initializes a *tls.Config using the given Config.
//
// If no certificates are given, (nil, nil) is returned.
// The CA certificate is optional and falls back to the system default.
func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) {
func loadTLSClientConfig(cfg *Config) (*tls.Config, error) {
if cfg.CertFile == "" || cfg.KeyFile == "" {
return nil, nil
}

68
ext/vulnmdsrc/driver.go Normal file
View File

@ -0,0 +1,68 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package vulnmdsrc exposes functions to dynamically register vulnerability
// metadata sources used to update a Clair database.
package vulnmdsrc
import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils/types"
)
// Appenders is the list of registered Appenders.
var Appenders = make(map[string]Appender)
// AppendFunc is the type of a callback provided to an Appender.
type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority)
// Appender represents anything that can fetch vulnerability metadata and
// append it to a Vulnerability.
type Appender interface {
// BuildCache loads metadata into memory such that it can be quickly accessed
// for future calls to Append.
BuildCache(database.Datastore) error
// AddMetadata adds metadata to the given database.Vulnerability.
// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
// Append
Append(vulnName string, callback AppendFunc) error
// PurgeCache deallocates metadata from memory after all calls to Append are
// finished.
PurgeCache()
// Clean deletes any allocated resources.
// It is invoked when Clair stops.
Clean()
}
// RegisterAppender makes an Appender available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func RegisterAppender(name string, a Appender) {
if name == "" {
panic("updater: could not register an Appender with an empty name")
}
if a == nil {
panic("vulnmdsrc: could not register a nil Appender")
}
if _, dup := Appenders[name]; dup {
panic("vulnmdsrc: RegisterAppender called twice for " + name)
}
Appenders[name] = a
}

33
ext/vulnmdsrc/nvd/nested_read_closer.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nvd
import "io"
// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
// io.ReadCloser.
// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
// simple manner.
type NestedReadCloser struct {
io.Reader
NestedReadClosers []io.ReadCloser
}
// Close closes the gzip.Reader and the underlying io.ReadCloser.
func (nrc *NestedReadCloser) Close() {
for _, nestedReadCloser := range nrc.NestedReadClosers {
nestedReadCloser.Close()
}
}

93
updater/metadata_fetchers/nvd/nvd.go → ext/vulnmdsrc/nvd/nvd.go
View File

@ -1,3 +1,19 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package nvd implements a vulnerability metadata appender using the NIST NVD
// database.
package nvd
import (
@ -15,30 +31,28 @@ import (
"sync"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnmdsrc"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
)
const (
dataFeedURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz"
dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta"
metadataKey string = "NVD"
appenderName string = "NVD"
)
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers")
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnmdsrc/nvd")
type NVDMetadataFetcher struct {
type appender struct {
localPath string
dataFeedHashes map[string]string
lock sync.Mutex
metadata map[string]NVDMetadata
sync.Mutex
}
type NVDMetadata struct {
@ -51,32 +65,32 @@ type NVDmetadataCVSSv2 struct {
}
func init() {
updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{})
vulnmdsrc.RegisterAppender(appenderName, &appender{})
}
func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
fetcher.lock.Lock()
defer fetcher.lock.Unlock()
func (a *appender) BuildCache(datastore database.Datastore) error {
a.Lock()
defer a.Unlock()
var err error
fetcher.metadata = make(map[string]NVDMetadata)
a.metadata = make(map[string]NVDMetadata)
// Init if necessary.
if fetcher.localPath == "" {
if a.localPath == "" {
// Create a temporary folder to store the NVD data and create hashes struct.
if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil {
if a.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil {
return cerrors.ErrFilesystem
}
fetcher.dataFeedHashes = make(map[string]string)
a.dataFeedHashes = make(map[string]string)
}
// Get data feeds.
dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath)
dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath)
if err != nil {
return err
}
fetcher.dataFeedHashes = dataFeedHashes
a.dataFeedHashes = dataFeedHashes
// Parse data feeds.
for dataFeedName, dataFeedReader := range dataFeedReaders {
@ -90,7 +104,7 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
for _, nvdEntry := range nvd.Entries {
// Create metadata entry.
if metadata := nvdEntry.Metadata(); metadata != nil {
fetcher.metadata[nvdEntry.Name] = *metadata
a.metadata[nvdEntry.Name] = *metadata
}
}
@ -100,42 +114,29 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
return nil
}
func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error {
fetcher.lock.Lock()
defer fetcher.lock.Unlock()
func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error {
a.Lock()
defer a.Unlock()
if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok {
vulnerability.Lock.Lock()
// Create Metadata map if necessary and assign the NVD metadata.
if vulnerability.Metadata == nil {
vulnerability.Metadata = make(map[string]interface{})
}
vulnerability.Metadata[metadataKey] = nvdMetadata
// Set the Severity using the CVSSv2 Score if none is set yet.
if vulnerability.Severity == "" || vulnerability.Severity == types.Unknown {
vulnerability.Severity = scoreToPriority(nvdMetadata.CVSSv2.Score)
}
vulnerability.Lock.Unlock()
if nvdMetadata, ok := a.metadata[vulnName]; ok {
appendFunc(appenderName, nvdMetadata, scoreToPriority(nvdMetadata.CVSSv2.Score))
}
return nil
}
func (fetcher *NVDMetadataFetcher) Unload() {
fetcher.lock.Lock()
defer fetcher.lock.Unlock()
func (a *appender) PurgeCache() {
a.Lock()
defer a.Unlock()
fetcher.metadata = nil
a.metadata = nil
}
func (fetcher *NVDMetadataFetcher) Clean() {
fetcher.lock.Lock()
defer fetcher.lock.Unlock()
func (a *appender) Clean() {
a.Lock()
defer a.Unlock()
os.RemoveAll(fetcher.localPath)
os.RemoveAll(a.localPath)
}
func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) {

14
updater/metadata_fetchers/nvd/xml.go → ext/vulnmdsrc/nvd/xml.go
View File

@ -1,3 +1,17 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nvd
import (

61
updater/fetchers/alpine/alpine.go → ext/vulnsrc/alpine/alpine.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,12 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package alpine implements a vulnerability Fetcher using the alpine-secdb
// git repository.
// Package alpine implements a vulnerability source updater using the
// alpine-secdb git repository.
package alpine
import (
"errors"
"fmt"
"io"
"io/ioutil"
@ -31,7 +30,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -45,29 +44,23 @@ const (
)
var (
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")
// ErrGitFailure is returned when a fetcher fails to interact with git.
ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine")
)
func init() {
updater.RegisterFetcher("alpine", &fetcher{})
vulnsrc.RegisterUpdater("alpine", &updater{})
}
type fetcher struct {
type updater struct {
repositoryLocalPath string
}
func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.Info("fetching Alpine vulnerabilities")
// Pull the master branch.
var commit string
commit, err = f.pullRepository()
commit, err = u.pullRepository()
if err != nil {
return
}
@ -90,12 +83,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon
}
var namespaces []string
namespaces, err = detectNamespaces(f.repositoryLocalPath)
namespaces, err = detectNamespaces(u.repositoryLocalPath)
// Append any changed vulnerabilities to the response.
for _, namespace := range namespaces {
var vulns []database.Vulnerability
var note string
vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace)
vulns, note, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace)
if err != nil {
return
}
@ -108,6 +101,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon
return
}
func (u *updater) Clean() {
if u.repositoryLocalPath != "" {
os.RemoveAll(u.repositoryLocalPath)
}
}
func detectNamespaces(path string) ([]string, error) {
// Open the root directory.
dir, err := os.Open(path)
@ -163,41 +162,35 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
return
}
func (f *fetcher) pullRepository() (commit string, err error) {
func (u *updater) pullRepository() (commit string, err error) {
// If the repository doesn't exist, clone it.
if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
return "", ErrFilesystem
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
return "", vulnsrc.ErrFilesystem
}
if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
f.Clean()
if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
u.Clean()
log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
return "", cerrors.ErrCouldNotDownload
}
} else {
// The repository exists and it needs to be refreshed via a pull.
_, err := utils.Exec(f.repositoryLocalPath, "git", "pull")
_, err := utils.Exec(u.repositoryLocalPath, "git", "pull")
if err != nil {
return "", ErrGitFailure
return "", vulnsrc.ErrGitFailure
}
}
out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
out, err := utils.Exec(u.repositoryLocalPath, "git", "rev-parse", "HEAD")
if err != nil {
return "", ErrGitFailure
return "", vulnsrc.ErrGitFailure
}
commit = strings.TrimSpace(string(out))
return
}
func (f *fetcher) Clean() {
if f.repositoryLocalPath != "" {
os.RemoveAll(f.repositoryLocalPath)
}
}
type secdb33File struct {
Distro string `yaml:"distroversion"`
Packages []struct {

0
updater/fetchers/alpine/alpine_test.go → ext/vulnsrc/alpine/alpine_test.go
View File

0
updater/fetchers/alpine/testdata/v33_main.yaml → ext/vulnsrc/alpine/testdata/v33_main.yaml vendored
View File

0
updater/fetchers/alpine/testdata/v34_main.yaml → ext/vulnsrc/alpine/testdata/v34_main.yaml vendored
View File

24
updater/fetchers/debian/debian.go → ext/vulnsrc/debian/debian.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package debian implements a vulnerability source updater using the Debian
// Security Tracker.
package debian
import (
@ -28,7 +30,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnsrc"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
@ -39,7 +41,7 @@ const (
updaterFlag = "debianUpdater"
)
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian")
var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/debian")
type jsonData map[string]map[string]jsonVuln
@ -54,16 +56,13 @@ type jsonRel struct {
Urgency string `json:"urgency"`
}
// DebianFetcher implements updater.Fetcher for the Debian Security Tracker
// (https://security-tracker.debian.org).
type DebianFetcher struct{}
type updater struct{}
func init() {
updater.RegisterFetcher("debian", &DebianFetcher{})
vulnsrc.RegisterUpdater("debian", &updater{})
}
// FetchUpdate fetches vulnerability updates from the Debian Security Tracker.
func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.Info("fetching Debian vulnerabilities")
// Download JSON.
@ -88,7 +87,9 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up
return resp, nil
}
func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.FetcherResponse, err error) {
func (u *updater) Clean() {}
func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.UpdateResponse, err error) {
hash := latestKnownHash
// Defer the addition of flag information to the response.
@ -254,6 +255,3 @@ func urgencyToSeverity(urgency string) types.Priority {
return types.Unknown
}
}
// Clean deletes any allocated resources.
func (fetcher *DebianFetcher) Clean() {}

0
updater/fetchers/debian/debian_test.go → ext/vulnsrc/debian/debian_test.go
View File

0
updater/fetchers/debian/testdata/fetcher_debian_test.json → ext/vulnsrc/debian/testdata/fetcher_debian_test.json vendored
View File

73
ext/vulnsrc/driver.go Normal file
View File

@ -0,0 +1,73 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package vulnsrc exposes functions to dynamically register vulnerability
// sources used to update a Clair database.
package vulnsrc
import (
"errors"
"github.com/coreos/clair/database"
)
var (
// Updaters is the list of registered Updaters.
Updaters = make(map[string]Updater)
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs")
// ErrGitFailure is returned when a fetcher fails to interact with git.
ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git")
)
// Updater represents anything that can fetch vulnerabilities and insert them
// into a Clair datastore.
type Updater interface {
// Update gets vulnerability updates.
Update(database.Datastore) (UpdateResponse, error)
// Clean deletes any allocated resources.
// It is invoked when Clair stops.
Clean()
}
// UpdateResponse represents the sum of results of an update.
type UpdateResponse struct {
FlagName string
FlagValue string
Notes []string
Vulnerabilities []database.Vulnerability
}
// RegisterUpdater makes an Updater available by the provided name.
//
// If RegisterUpdater is called twice with the same name, the name is blank, or
// if the provided Updater is nil, this function panics.
func RegisterUpdater(name string, u Updater) {
if name == "" {
panic("vulnsrc: could not register an Updater with an empty name")
}
if u == nil {
panic("vulnsrc: could not register a nil Updater")
}
if _, dup := Updaters[name]; dup {
panic("vulnsrc: RegisterUpdater called twice for " + name)
}
Updaters[name] = u
}

22
updater/fetchers/oracle/oracle.go → ext/vulnsrc/oracle/oracle.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package oracle implements a vulnerability source updater using the
// Oracle Linux OVAL Database.
package oracle
import (
@ -26,7 +28,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/rpm"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnsrc"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
@ -47,7 +49,7 @@ var (
elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`)
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/oracle")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/oracle")
)
type oval struct {
@ -77,16 +79,13 @@ type criterion struct {
Comment string `xml:"comment,attr"`
}
// OracleFetcher implements updater.Fetcher and gets vulnerability updates from
// the Oracle Linux OVAL definitions.
type OracleFetcher struct{}
type updater struct{}
func init() {
updater.RegisterFetcher("Oracle", &OracleFetcher{})
vulnsrc.RegisterUpdater("oracle", &updater{})
}
// FetchUpdate gets vulnerability updates from the Oracle Linux OVAL definitions.
func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.Info("fetching Oracle Linux vulnerabilities")
// Get the first ELSA we have to manage.
@ -153,6 +152,8 @@ func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.
return resp, nil
}
func (u *updater) Clean() {}
func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
// Decode the XML.
var ov oval
@ -356,6 +357,3 @@ func priority(def definition) types.Priority {
return types.Unknown
}
}
// Clean deletes any allocated resources.
func (f *OracleFetcher) Clean() {}

0
updater/fetchers/oracle/oracle_test.go → ext/vulnsrc/oracle/oracle_test.go
View File

0
updater/fetchers/oracle/testdata/fetcher_oracle_test.1.xml → ext/vulnsrc/oracle/testdata/fetcher_oracle_test.1.xml vendored
View File

0
updater/fetchers/oracle/testdata/fetcher_oracle_test.2.xml → ext/vulnsrc/oracle/testdata/fetcher_oracle_test.2.xml vendored
View File

22
updater/fetchers/rhel/rhel.go → ext/vulnsrc/rhel/rhel.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package rhel implements a vulnerability source updater using the
// Red Hat Linux OVAL Database.
package rhel
import (
@ -28,7 +30,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/rpm"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnsrc"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
@ -82,17 +84,14 @@ type criterion struct {
Comment string `xml:"comment,attr"`
}
// RHELFetcher implements updater.Fetcher and gets vulnerability updates from
// the Red Hat OVAL definitions.
type RHELFetcher struct{}
type updater struct{}
func init() {
updater.RegisterFetcher("Red Hat", &RHELFetcher{})
vulnsrc.RegisterUpdater("rhel", &updater{})
}
// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions.
func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
log.Info("fetching Red Hat vulnerabilities")
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.Info("fetching RHEL vulnerabilities")
// Get the first RHSA we have to manage.
flagValue, err := datastore.GetKeyValue(updaterFlag)
@ -156,6 +155,8 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe
return resp, nil
}
func (u *updater) Clean() {}
func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
// Decode the XML.
var ov oval
@ -362,6 +363,3 @@ func priority(def definition) types.Priority {
return types.Unknown
}
}
// Clean deletes any allocated resources.
func (f *RHELFetcher) Clean() {}

0
updater/fetchers/rhel/rhel_test.go → ext/vulnsrc/rhel/rhel_test.go
View File

0
updater/fetchers/rhel/testdata/fetcher_rhel_test.1.xml → ext/vulnsrc/rhel/testdata/fetcher_rhel_test.1.xml vendored
View File

0
updater/fetchers/rhel/testdata/fetcher_rhel_test.2.xml → ext/vulnsrc/rhel/testdata/fetcher_rhel_test.2.xml vendored
View File

0
updater/fetchers/ubuntu/testdata/fetcher_ubuntu_test.txt → ext/vulnsrc/ubuntu/testdata/fetcher_ubuntu_test.txt vendored
View File

56
updater/fetchers/ubuntu/ubuntu.go → ext/vulnsrc/ubuntu/ubuntu.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,12 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package ubuntu implements a vulnerability source updater using the
// Ubuntu CVE Tracker.
package ubuntu
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
@ -31,7 +32,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
"github.com/coreos/clair/ext/versionfmt/dpkg"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -75,33 +76,27 @@ var (
affectsCaptureRegexp = regexp.MustCompile(`(?P<release>.*)_(?P<package>.*): (?P<status>[^\s]*)( \(+(?P<note>[^()]*)\)+)?`)
affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames()
log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu")
// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")
log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/ubuntu")
)
// UbuntuFetcher implements updater.Fetcher and gets vulnerability updates from
// the Ubuntu CVE Tracker.
type UbuntuFetcher struct {
type updater struct {
repositoryLocalPath string
}
func init() {
updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{})
vulnsrc.RegisterUpdater("ubuntu", &updater{})
}
// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
log.Info("fetching Ubuntu vulnerabilities")
// Pull the bzr repository.
if err = fetcher.pullRepository(); err != nil {
if err = u.pullRepository(); err != nil {
return resp, err
}
// Get revision number.
revisionNumber, err := getRevisionNumber(fetcher.repositoryLocalPath)
revisionNumber, err := getRevisionNumber(u.repositoryLocalPath)
if err != nil {
return resp, err
}
@ -113,7 +108,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
}
// Get the list of vulnerabilities that we have to update.
modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, fetcher.repositoryLocalPath)
modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath)
if err != nil {
return resp, err
}
@ -121,7 +116,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
notes := make(map[string]struct{})
for cvePath := range modifiedCVE {
// Open the CVE file.
file, err := os.Open(fetcher.repositoryLocalPath + "/" + cvePath)
file, err := os.Open(u.repositoryLocalPath + "/" + cvePath)
if err != nil {
// This can happen when a file is modified and then moved in another
// commit.
@ -166,16 +161,20 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
return
}
func (fetcher *UbuntuFetcher) pullRepository() (err error) {
func (u *updater) Clean() {
os.RemoveAll(u.repositoryLocalPath)
}
func (u *updater) pullRepository() (err error) {
// Determine whether we should branch or pull.
if _, pathExists := os.Stat(fetcher.repositoryLocalPath); fetcher.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
// Create a temporary folder to store the repository.
if fetcher.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return ErrFilesystem
if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil {
return vulnsrc.ErrFilesystem
}
// Branch repository.
if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil {
if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil {
log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out)
return cerrors.ErrCouldNotDownload
}
@ -184,8 +183,8 @@ func (fetcher *UbuntuFetcher) pullRepository() (err error) {
}
// Pull repository.
if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil {
os.RemoveAll(fetcher.repositoryLocalPath)
if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil {
os.RemoveAll(u.repositoryLocalPath)
log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out)
return cerrors.ErrCouldNotDownload
@ -217,14 +216,14 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat
d, err := os.Open(repositoryLocalPath + "/" + folder)
if err != nil {
log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err)
return nil, ErrFilesystem
return nil, vulnsrc.ErrFilesystem
}
// Get the FileInfo of all the files in the directory.
names, err := d.Readdirnames(-1)
if err != nil {
log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err)
return nil, ErrFilesystem
return nil, vulnsrc.ErrFilesystem
}
// Add the vulnerabilities to the list.
@ -414,8 +413,3 @@ func ubuntuPriorityToSeverity(priority string) types.Priority {
log.Warning("Could not determine a vulnerability priority from: %s", priority)
return types.Unknown
}
// Clean deletes any allocated resources.
func (fetcher *UbuntuFetcher) Clean() {
os.RemoveAll(fetcher.repositoryLocalPath)
}

0
updater/fetchers/ubuntu/ubuntu_test.go → ext/vulnsrc/ubuntu/ubuntu_test.go
View File

66
notifier/notifier.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,8 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package notifier fetches notifications from the database and informs the specified remote handler
// about their existences, inviting the third party to actively query the API about it.
// Package notifier fetches notifications from the database and informs the
// specified remote handler about their existences, inviting the third party to
// actively query the API about it.
package notifier
import (
@ -26,6 +27,7 @@ import (
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/notification"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
)
@ -40,8 +42,6 @@ const (
var (
log = capnslog.NewPackageLogger("github.com/coreos/clair", "notifier")
notifiers = make(map[string]Notifier)
promNotifierLatencyMilliseconds = prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "clair_notifier_latency_milliseconds",
Help: "Time it takes to send a notification after it's been created.",
@ -53,57 +53,29 @@ var (
}, []string{"backend"})
)
// Notifier represents anything that can transmit notifications.
type Notifier interface {
// Configure attempts to initialize the notifier with the provided configuration.
// It returns whether the notifier is enabled or not.
Configure(*config.NotifierConfig) (bool, error)
// Send informs the existence of the specified notification.
Send(notification database.VulnerabilityNotification) error
}
func init() {
prometheus.MustRegister(promNotifierLatencyMilliseconds)
prometheus.MustRegister(promNotifierBackendErrorsTotal)
}
// RegisterNotifier makes a Fetcher available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func RegisterNotifier(name string, n Notifier) {
if name == "" {
panic("notifier: could not register a Notifier with an empty name")
}
if n == nil {
panic("notifier: could not register a nil Notifier")
}
if _, dup := notifiers[name]; dup {
panic("notifier: RegisterNotifier called twice for " + name)
}
notifiers[name] = n
}
// Run starts the Notifier service.
func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) {
defer stopper.End()
// Configure registered notifiers.
for notifierName, notifier := range notifiers {
if configured, err := notifier.Configure(config); configured {
log.Infof("notifier '%s' configured\n", notifierName)
for senderName, sender := range notification.Senders {
if configured, err := sender.Configure(config); configured {
log.Infof("sender '%s' configured\n", senderName)
} else {
delete(notifiers, notifierName)
delete(notification.Senders, senderName)
if err != nil {
log.Errorf("could not configure notifier '%s': %s", notifierName, err)
log.Errorf("could not configure notifier '%s': %s", senderName, err)
}
}
}
// Do not run the updater if there is no notifier enabled.
if len(notifiers) == 0 {
if len(notification.Senders) == 0 {
log.Infof("notifier service is disabled")
return
}
@ -175,31 +147,31 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA
}
}
func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Send notification.
for notifierName, notifier := range notifiers {
for senderName, sender := range notification.Senders {
var attempts int
var backOff time.Duration
for {
// Max attempts exceeded.
if attempts >= maxAttempts {
log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts)
log.Infof("giving up on sending notification '%s' via sender '%s': max attempts exceeded (%d)\n", n.Name, senderName, maxAttempts)
return false, false
}
// Backoff.
if backOff > 0 {
log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts)
log.Infof("waiting %v before retrying to send notification '%s' via sender '%s' (Attempt %d / %d)\n", backOff, n.Name, senderName, attempts+1, maxAttempts)
if !st.Sleep(backOff) {
return false, true
}
}
// Send using the current notifier.
if err := notifier.Send(notification); err != nil {
if err := sender.Send(n); err != nil {
// Send failed; increase attempts/backoff and retry.
promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc()
log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err)
promNotifierBackendErrorsTotal.WithLabelValues(senderName).Inc()
log.Errorf("could not send notification '%s' via notifier '%s': %v", n.Name, senderName, err)
backOff = timeutil.ExpBackoff(backOff, maxBackOff)
attempts++
continue
@ -210,6 +182,6 @@ func handleTask(notification database.VulnerabilityNotification, st *utils.Stopp
}
}
log.Infof("successfully sent notification '%s'\n", notification.Name)
log.Infof("successfully sent notification '%s'\n", n.Name)
return true, false
}

56
updater/fetchers.go
View File

@ -1,56 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package updater
import "github.com/coreos/clair/database"
var fetchers = make(map[string]Fetcher)
// Fetcher represents anything that can fetch vulnerabilities.
type Fetcher interface {
// FetchUpdate gets vulnerability updates.
FetchUpdate(database.Datastore) (FetcherResponse, error)
// Clean deletes any allocated resources.
// It is invoked when Clair stops.
Clean()
}
// FetcherResponse represents the sum of results of an update.
type FetcherResponse struct {
FlagName string
FlagValue string
Notes []string
Vulnerabilities []database.Vulnerability
}
// RegisterFetcher makes a Fetcher available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func RegisterFetcher(name string, f Fetcher) {
if name == "" {
panic("updater: could not register a Fetcher with an empty name")
}
if f == nil {
panic("updater: could not register a nil Fetcher")
}
if _, dup := fetchers[name]; dup {
panic("updater: RegisterFetcher called twice for " + name)
}
fetchers[name] = f
}

64
updater/metadata_fetchers.go
View File

@ -1,64 +0,0 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package updater
import (
"sync"
"github.com/coreos/clair/database"
)
var metadataFetchers = make(map[string]MetadataFetcher)
type VulnerabilityWithLock struct {
*database.Vulnerability
Lock sync.Mutex
}
// MetadataFetcher
type MetadataFetcher interface {
// Load runs right before the Updater calls AddMetadata for each vulnerabilities.
Load(database.Datastore) error
// AddMetadata adds metadata to the given database.Vulnerability.
// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
AddMetadata(*VulnerabilityWithLock) error
// Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities.
Unload()
// Clean deletes any allocated resources.
// It is invoked when Clair stops.
Clean()
}
// RegisterFetcher makes a Fetcher available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func RegisterMetadataFetcher(name string, f MetadataFetcher) {
if name == "" {
panic("updater: could not register a MetadataFetcher with an empty name")
}
if f == nil {
panic("updater: could not register a nil MetadataFetcher")
}
if _, dup := fetchers[name]; dup {
panic("updater: RegisterMetadataFetcher called twice for " + name)
}
metadataFetchers[name] = f
}

19
updater/metadata_fetchers/nvd/nested_read_closer.go
View File

@ -1,19 +0,0 @@
package nvd
import "io"
// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
// io.ReadCloser.
// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
// simple manner.
type NestedReadCloser struct {
io.Reader
NestedReadClosers []io.ReadCloser
}
// Close closes the gzip.Reader and the underlying io.ReadCloser.
func (nrc *NestedReadCloser) Close() {
for _, nestedReadCloser := range nrc.NestedReadClosers {
nestedReadCloser.Close()
}
}

91
updater/updater.go
View File

@ -1,4 +1,4 @@
// Copyright 2015 clair authors
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,8 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package updater updates the vulnerability database periodically using
// the registered vulnerability fetchers.
// Package updater updates the vulnerability database periodically using the
// registered vulnerability source updaters and vulnerability metadata
// appenders.
package updater
import (
@ -22,12 +23,16 @@ import (
"sync"
"time"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
"github.com/coreos/pkg/capnslog"
"github.com/pborman/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/vulnmdsrc"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/utils/types"
)
const (
@ -147,11 +152,11 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
}
// Clean resources.
for _, metadataFetcher := range metadataFetchers {
metadataFetcher.Clean()
for _, appenders := range vulnmdsrc.Appenders {
appenders.Clean()
}
for _, fetcher := range fetchers {
fetcher.Clean()
for _, updaters := range vulnsrc.Updaters {
updaters.Clean()
}
log.Info("updater service stopped")
@ -209,10 +214,10 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
// Fetch updates in parallel.
log.Info("fetching vulnerability updates")
var responseC = make(chan *FetcherResponse, 0)
for n, f := range fetchers {
go func(name string, fetcher Fetcher) {
response, err := fetcher.FetchUpdate(datastore)
var responseC = make(chan *vulnsrc.UpdateResponse, 0)
for n, u := range vulnsrc.Updaters {
go func(name string, u vulnsrc.Updater) {
response, err := u.Update(datastore)
if err != nil {
promUpdaterErrorsTotal.Inc()
log.Errorf("an error occured when fetching update '%s': %s.", name, err)
@ -222,11 +227,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
}
responseC <- &response
}(n, f)
}(n, u)
}
// Collect results of updates.
for i := 0; i < len(fetchers); i++ {
for i := 0; i < len(vulnsrc.Updaters); i++ {
resp := <-responseC
if resp != nil {
vulnerabilities = append(vulnerabilities, doVulnerabilitiesNamespacing(resp.Vulnerabilities)...)
@ -243,42 +248,43 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
// Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel.
func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability {
if len(metadataFetchers) == 0 {
if len(vulnmdsrc.Appenders) == 0 {
return vulnerabilities
}
log.Info("adding metadata to vulnerabilities")
// Wrap vulnerabilities in VulnerabilityWithLock.
// It ensures that only one metadata fetcher at a time can modify the Metadata map.
vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities))
// Add a mutex to each vulnerability to ensure that only one appender at a
// time can modify the vulnerability's Metadata map.
lockableVulnerabilities := make([]*lockableVulnerability, 0, len(vulnerabilities))
for i := 0; i < len(vulnerabilities); i++ {
vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{
lockableVulnerabilities = append(lockableVulnerabilities, &lockableVulnerability{
Vulnerability: &vulnerabilities[i],
})
}
var wg sync.WaitGroup
wg.Add(len(metadataFetchers))
wg.Add(len(vulnmdsrc.Appenders))
for n, f := range metadataFetchers {
go func(name string, metadataFetcher MetadataFetcher) {
for n, a := range vulnmdsrc.Appenders {
go func(name string, appender vulnmdsrc.Appender) {
defer wg.Done()
// Load the metadata fetcher.
if err := metadataFetcher.Load(datastore); err != nil {
// Build up a metadata cache.
if err := appender.BuildCache(datastore); err != nil {
promUpdaterErrorsTotal.Inc()
log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err)
return
}
// Add metadata to each vulnerability.
for _, vulnerability := range vulnerabilitiesWithLocks {
metadataFetcher.AddMetadata(vulnerability)
// Append vulnerability metadata to each vulnerability.
for _, vulnerability := range lockableVulnerabilities {
appender.Append(vulnerability.Name, vulnerability.appendFunc)
}
metadataFetcher.Unload()
}(n, f)
// Purge the metadata cache.
appender.PurgeCache()
}(n, a)
}
wg.Wait()
@ -305,6 +311,29 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) {
return time.Unix(lastUpdateTS, 0).UTC(), false, nil
}
type lockableVulnerability struct {
*database.Vulnerability
sync.Mutex
}
func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity types.Priority) {
lv.Lock()
defer lv.Unlock()
// If necessary, initialize the metadata map for the vulnerability.
if lv.Metadata == nil {
lv.Metadata = make(map[string]interface{})
}
// Append the metadata.
lv.Metadata[metadataKey] = metadata
// If necessary, provide a severity for the vulnerability.
if lv.Severity == "" || lv.Severity == types.Unknown {
lv.Severity = severity
}
}
// doVulnerabilitiesNamespacing takes Vulnerabilities that don't have a Namespace and split them
// into multiple vulnerabilities that have a Namespace and only contains the FixedIn
// FeatureVersions corresponding to their Namespace.