From 4a990372fff35f606184e276976f0279e4ea5a56 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 3 Jan 2017 21:44:32 -0500 Subject: [PATCH 01/23] refactor: move updaters and notifier into ext --- cmd/clair/main.go | 17 ++-- ext/notification/driver.go | 65 +++++++++++++ .../notification/webhook}/webhook.go | 33 ++++--- ext/vulnmdsrc/driver.go | 68 +++++++++++++ ext/vulnmdsrc/nvd/nested_read_closer.go | 33 +++++++ .../vulnmdsrc}/nvd/nvd.go | 95 ++++++++++--------- .../vulnmdsrc}/nvd/xml.go | 14 +++ .../fetchers => ext/vulnsrc}/alpine/alpine.go | 61 ++++++------ .../vulnsrc}/alpine/alpine_test.go | 0 .../vulnsrc}/alpine/testdata/v33_main.yaml | 0 .../vulnsrc}/alpine/testdata/v34_main.yaml | 0 .../fetchers => ext/vulnsrc}/debian/debian.go | 24 +++-- .../vulnsrc}/debian/debian_test.go | 0 .../debian/testdata/fetcher_debian_test.json | 0 ext/vulnsrc/driver.go | 73 ++++++++++++++ .../fetchers => ext/vulnsrc}/oracle/oracle.go | 22 ++--- .../vulnsrc}/oracle/oracle_test.go | 0 .../oracle/testdata/fetcher_oracle_test.1.xml | 0 .../oracle/testdata/fetcher_oracle_test.2.xml | 0 .../fetchers => ext/vulnsrc}/rhel/rhel.go | 22 ++--- .../vulnsrc}/rhel/rhel_test.go | 0 .../rhel/testdata/fetcher_rhel_test.1.xml | 0 .../rhel/testdata/fetcher_rhel_test.2.xml | 0 .../ubuntu/testdata/fetcher_ubuntu_test.txt | 0 .../fetchers => ext/vulnsrc}/ubuntu/ubuntu.go | 56 +++++------ .../vulnsrc}/ubuntu/ubuntu_test.go | 0 notifier/notifier.go | 66 ++++--------- updater/fetchers.go | 56 ----------- updater/metadata_fetchers.go | 64 ------------- .../nvd/nested_read_closer.go | 19 ---- updater/updater.go | 91 ++++++++++++------ 31 files changed, 487 insertions(+), 392 deletions(-) create mode 100644 ext/notification/driver.go rename {notifier/notifiers => ext/notification/webhook}/webhook.go (79%) create mode 100644 ext/vulnmdsrc/driver.go create mode 100644 ext/vulnmdsrc/nvd/nested_read_closer.go rename {updater/metadata_fetchers => ext/vulnmdsrc}/nvd/nvd.go (72%) rename {updater/metadata_fetchers => ext/vulnmdsrc}/nvd/xml.go (78%) rename {updater/fetchers => ext/vulnsrc}/alpine/alpine.go (80%) rename {updater/fetchers => ext/vulnsrc}/alpine/alpine_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/alpine/testdata/v33_main.yaml (100%) rename {updater/fetchers => ext/vulnsrc}/alpine/testdata/v34_main.yaml (100%) rename {updater/fetchers => ext/vulnsrc}/debian/debian.go (91%) rename {updater/fetchers => ext/vulnsrc}/debian/debian_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/debian/testdata/fetcher_debian_test.json (100%) create mode 100644 ext/vulnsrc/driver.go rename {updater/fetchers => ext/vulnsrc}/oracle/oracle.go (93%) rename {updater/fetchers => ext/vulnsrc}/oracle/oracle_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/oracle/testdata/fetcher_oracle_test.1.xml (100%) rename {updater/fetchers => ext/vulnsrc}/oracle/testdata/fetcher_oracle_test.2.xml (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/rhel.go (94%) rename {updater/fetchers => ext/vulnsrc}/rhel/rhel_test.go (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/testdata/fetcher_rhel_test.1.xml (100%) rename {updater/fetchers => ext/vulnsrc}/rhel/testdata/fetcher_rhel_test.2.xml (100%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/testdata/fetcher_ubuntu_test.txt (100%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/ubuntu.go (86%) rename {updater/fetchers => ext/vulnsrc}/ubuntu/ubuntu_test.go (100%) delete mode 100644 updater/fetchers.go delete mode 100644 updater/metadata_fetchers.go delete mode 100644 updater/metadata_fetchers/nvd/nested_read_closer.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 1e3735d8..8d3e35f5 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -25,15 +25,14 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/config" - // Register components - _ "github.com/coreos/clair/notifier/notifiers" - - _ "github.com/coreos/clair/updater/fetchers/alpine" - _ "github.com/coreos/clair/updater/fetchers/debian" - _ "github.com/coreos/clair/updater/fetchers/oracle" - _ "github.com/coreos/clair/updater/fetchers/rhel" - _ "github.com/coreos/clair/updater/fetchers/ubuntu" - _ "github.com/coreos/clair/updater/metadata_fetchers/nvd" + // Register extensions. + _ "github.com/coreos/clair/ext/notification/webhook" + _ "github.com/coreos/clair/ext/vulnmdsrc/nvd" + _ "github.com/coreos/clair/ext/vulnsrc/alpine" + _ "github.com/coreos/clair/ext/vulnsrc/debian" + _ "github.com/coreos/clair/ext/vulnsrc/oracle" + _ "github.com/coreos/clair/ext/vulnsrc/rhel" + _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" _ "github.com/coreos/clair/worker/detectors/data/aci" _ "github.com/coreos/clair/worker/detectors/data/docker" diff --git a/ext/notification/driver.go b/ext/notification/driver.go new file mode 100644 index 00000000..2f3b6429 --- /dev/null +++ b/ext/notification/driver.go @@ -0,0 +1,65 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notification fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. + +// Package notification exposes functions to dynamically register methods to +// deliver notifications from the Clair database. +package notification + +import ( + "github.com/coreos/pkg/capnslog" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/database" +) + +var ( + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/notification") + + // Senders is the list of registered Senders. + Senders = make(map[string]Sender) +) + +// Sender represents anything that can transmit notifications. +type Sender interface { + // Configure attempts to initialize the notifier with the provided configuration. + // It returns whether the notifier is enabled or not. + Configure(*config.NotifierConfig) (bool, error) + + // Send informs the existence of the specified notification. + Send(notification database.VulnerabilityNotification) error +} + +// RegisterSender makes a Sender available by the provided name. +// +// If RegisterSender is called twice with the same name, the name is blank, or +// if the provided Sender is nil, this function panics. +func RegisterSender(name string, s Sender) { + if name == "" { + panic("notification: could not register a Sender with an empty name") + } + + if s == nil { + panic("notification: could not register a nil Sender") + } + + if _, dup := Senders[name]; dup { + panic("notification: RegisterSender called twice for " + name) + } + + Senders[name] = s +} diff --git a/notifier/notifiers/webhook.go b/ext/notification/webhook/webhook.go similarity index 79% rename from notifier/notifiers/webhook.go rename to ext/notification/webhook/webhook.go index 28b76604..6a991cf4 100644 --- a/notifier/notifiers/webhook.go +++ b/ext/notification/webhook/webhook.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package notifiers implements several kinds of notifier.Notifier -package notifiers +// Package webhook implements a notification sender for HTTP JSON webhooks. +package webhook import ( "bytes" @@ -31,19 +31,18 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" - "github.com/coreos/clair/notifier" + "github.com/coreos/clair/ext/notification" ) const timeout = 5 * time.Second -// A WebhookNotifier dispatches notifications to a webhook endpoint. -type WebhookNotifier struct { +type sender struct { endpoint string client *http.Client } -// A WebhookNotifierConfiguration represents the configuration of a WebhookNotifier. -type WebhookNotifierConfiguration struct { +// Config represents the configuration of a Webhook Sender. +type Config struct { Endpoint string ServerName string CertFile string @@ -53,12 +52,12 @@ type WebhookNotifierConfiguration struct { } func init() { - notifier.RegisterNotifier("webhook", &WebhookNotifier{}) + notification.RegisterSender("webhook", &sender{}) } -func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) { +func (s *sender) Configure(config *config.NotifierConfig) (bool, error) { // Get configuration - var httpConfig WebhookNotifierConfiguration + var httpConfig Config if config == nil { return false, nil } @@ -81,11 +80,11 @@ func (h *WebhookNotifier) Configure(config *config.NotifierConfig) (bool, error) if _, err := url.ParseRequestURI(httpConfig.Endpoint); err != nil { return false, fmt.Errorf("could not parse endpoint URL: %s\n", err) } - h.endpoint = httpConfig.Endpoint + s.endpoint = httpConfig.Endpoint // Setup HTTP client. transport := &http.Transport{} - h.client = &http.Client{ + s.client = &http.Client{ Transport: transport, Timeout: timeout, } @@ -114,7 +113,7 @@ type notificationEnvelope struct { } } -func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error { +func (s *sender) Send(notification database.VulnerabilityNotification) error { // Marshal notification. jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}}) if err != nil { @@ -122,7 +121,7 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) } // Send notification via HTTP POST. - resp, err := h.client.Post(h.endpoint, "application/json", bytes.NewBuffer(jsonNotification)) + resp, err := s.client.Post(s.endpoint, "application/json", bytes.NewBuffer(jsonNotification)) if err != nil || resp == nil || (resp.StatusCode != 200 && resp.StatusCode != 201) { if resp != nil { return fmt.Errorf("got status %d, expected 200/201", resp.StatusCode) @@ -134,11 +133,11 @@ func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) return nil } -// loadTLSClientConfig initializes a *tls.Config using the given WebhookNotifierConfiguration. +// loadTLSClientConfig initializes a *tls.Config using the given Config. // // If no certificates are given, (nil, nil) is returned. // The CA certificate is optional and falls back to the system default. -func loadTLSClientConfig(cfg *WebhookNotifierConfiguration) (*tls.Config, error) { +func loadTLSClientConfig(cfg *Config) (*tls.Config, error) { if cfg.CertFile == "" || cfg.KeyFile == "" { return nil, nil } diff --git a/ext/vulnmdsrc/driver.go b/ext/vulnmdsrc/driver.go new file mode 100644 index 00000000..658125de --- /dev/null +++ b/ext/vulnmdsrc/driver.go @@ -0,0 +1,68 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vulnmdsrc exposes functions to dynamically register vulnerability +// metadata sources used to update a Clair database. +package vulnmdsrc + +import ( + "github.com/coreos/clair/database" + "github.com/coreos/clair/utils/types" +) + +// Appenders is the list of registered Appenders. +var Appenders = make(map[string]Appender) + +// AppendFunc is the type of a callback provided to an Appender. +type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority) + +// Appender represents anything that can fetch vulnerability metadata and +// append it to a Vulnerability. +type Appender interface { + // BuildCache loads metadata into memory such that it can be quickly accessed + // for future calls to Append. + BuildCache(database.Datastore) error + + // AddMetadata adds metadata to the given database.Vulnerability. + // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. + // Append + Append(vulnName string, callback AppendFunc) error + + // PurgeCache deallocates metadata from memory after all calls to Append are + // finished. + PurgeCache() + + // Clean deletes any allocated resources. + // It is invoked when Clair stops. + Clean() +} + +// RegisterAppender makes an Appender available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func RegisterAppender(name string, a Appender) { + if name == "" { + panic("updater: could not register an Appender with an empty name") + } + + if a == nil { + panic("vulnmdsrc: could not register a nil Appender") + } + + if _, dup := Appenders[name]; dup { + panic("vulnmdsrc: RegisterAppender called twice for " + name) + } + + Appenders[name] = a +} diff --git a/ext/vulnmdsrc/nvd/nested_read_closer.go b/ext/vulnmdsrc/nvd/nested_read_closer.go new file mode 100644 index 00000000..23322655 --- /dev/null +++ b/ext/vulnmdsrc/nvd/nested_read_closer.go @@ -0,0 +1,33 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvd + +import "io" + +// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed +// io.ReadCloser. +// It allows chaining io.ReadCloser together and still keep the ability to close them all in a +// simple manner. +type NestedReadCloser struct { + io.Reader + NestedReadClosers []io.ReadCloser +} + +// Close closes the gzip.Reader and the underlying io.ReadCloser. +func (nrc *NestedReadCloser) Close() { + for _, nestedReadCloser := range nrc.NestedReadClosers { + nestedReadCloser.Close() + } +} diff --git a/updater/metadata_fetchers/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go similarity index 72% rename from updater/metadata_fetchers/nvd/nvd.go rename to ext/vulnmdsrc/nvd/nvd.go index fadf719c..31636a77 100644 --- a/updater/metadata_fetchers/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -1,3 +1,19 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nvd implements a vulnerability metadata appender using the NIST NVD +// database. package nvd import ( @@ -15,30 +31,28 @@ import ( "sync" "time" + "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair/database" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnmdsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" - "github.com/coreos/pkg/capnslog" ) const ( dataFeedURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz" dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" - metadataKey string = "NVD" + appenderName string = "NVD" ) -var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers") -) +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnmdsrc/nvd") -type NVDMetadataFetcher struct { +type appender struct { localPath string dataFeedHashes map[string]string - lock sync.Mutex - - metadata map[string]NVDMetadata + metadata map[string]NVDMetadata + sync.Mutex } type NVDMetadata struct { @@ -51,32 +65,32 @@ type NVDmetadataCVSSv2 struct { } func init() { - updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{}) + vulnmdsrc.RegisterAppender(appenderName, &appender{}) } -func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) BuildCache(datastore database.Datastore) error { + a.Lock() + defer a.Unlock() var err error - fetcher.metadata = make(map[string]NVDMetadata) + a.metadata = make(map[string]NVDMetadata) // Init if necessary. - if fetcher.localPath == "" { + if a.localPath == "" { // Create a temporary folder to store the NVD data and create hashes struct. - if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { + if a.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { return cerrors.ErrFilesystem } - fetcher.dataFeedHashes = make(map[string]string) + a.dataFeedHashes = make(map[string]string) } // Get data feeds. - dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath) + dataFeedReaders, dataFeedHashes, err := getDataFeeds(a.dataFeedHashes, a.localPath) if err != nil { return err } - fetcher.dataFeedHashes = dataFeedHashes + a.dataFeedHashes = dataFeedHashes // Parse data feeds. for dataFeedName, dataFeedReader := range dataFeedReaders { @@ -90,7 +104,7 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { for _, nvdEntry := range nvd.Entries { // Create metadata entry. if metadata := nvdEntry.Metadata(); metadata != nil { - fetcher.metadata[nvdEntry.Name] = *metadata + a.metadata[nvdEntry.Name] = *metadata } } @@ -100,42 +114,29 @@ func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { return nil } -func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error { + a.Lock() + defer a.Unlock() - if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok { - vulnerability.Lock.Lock() - - // Create Metadata map if necessary and assign the NVD metadata. - if vulnerability.Metadata == nil { - vulnerability.Metadata = make(map[string]interface{}) - } - vulnerability.Metadata[metadataKey] = nvdMetadata - - // Set the Severity using the CVSSv2 Score if none is set yet. - if vulnerability.Severity == "" || vulnerability.Severity == types.Unknown { - vulnerability.Severity = scoreToPriority(nvdMetadata.CVSSv2.Score) - } - - vulnerability.Lock.Unlock() + if nvdMetadata, ok := a.metadata[vulnName]; ok { + appendFunc(appenderName, nvdMetadata, scoreToPriority(nvdMetadata.CVSSv2.Score)) } return nil } -func (fetcher *NVDMetadataFetcher) Unload() { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) PurgeCache() { + a.Lock() + defer a.Unlock() - fetcher.metadata = nil + a.metadata = nil } -func (fetcher *NVDMetadataFetcher) Clean() { - fetcher.lock.Lock() - defer fetcher.lock.Unlock() +func (a *appender) Clean() { + a.Lock() + defer a.Unlock() - os.RemoveAll(fetcher.localPath) + os.RemoveAll(a.localPath) } func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) { diff --git a/updater/metadata_fetchers/nvd/xml.go b/ext/vulnmdsrc/nvd/xml.go similarity index 78% rename from updater/metadata_fetchers/nvd/xml.go rename to ext/vulnmdsrc/nvd/xml.go index db13c897..d7649ce6 100644 --- a/updater/metadata_fetchers/nvd/xml.go +++ b/ext/vulnmdsrc/nvd/xml.go @@ -1,3 +1,17 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package nvd import ( diff --git a/updater/fetchers/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go similarity index 80% rename from updater/fetchers/alpine/alpine.go rename to ext/vulnsrc/alpine/alpine.go index bda68326..a2bbf7ec 100644 --- a/updater/fetchers/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package alpine implements a vulnerability Fetcher using the alpine-secdb -// git repository. +// Package alpine implements a vulnerability source updater using the +// alpine-secdb git repository. package alpine import ( - "errors" "fmt" "io" "io/ioutil" @@ -31,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -45,29 +44,23 @@ const ( ) var ( - // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. - ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs") - - // ErrGitFailure is returned when a fetcher fails to interact with git. - ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git") - - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine") ) func init() { - updater.RegisterFetcher("alpine", &fetcher{}) + vulnsrc.RegisterUpdater("alpine", &updater{}) } -type fetcher struct { +type updater struct { repositoryLocalPath string } -func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Alpine vulnerabilities") // Pull the master branch. var commit string - commit, err = f.pullRepository() + commit, err = u.pullRepository() if err != nil { return } @@ -90,12 +83,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon } var namespaces []string - namespaces, err = detectNamespaces(f.repositoryLocalPath) + namespaces, err = detectNamespaces(u.repositoryLocalPath) // Append any changed vulnerabilities to the response. for _, namespace := range namespaces { var vulns []database.Vulnerability var note string - vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace) + vulns, note, err = parseVulnsFromNamespace(u.repositoryLocalPath, namespace) if err != nil { return } @@ -108,6 +101,12 @@ func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherRespon return } +func (u *updater) Clean() { + if u.repositoryLocalPath != "" { + os.RemoveAll(u.repositoryLocalPath) + } +} + func detectNamespaces(path string) ([]string, error) { // Open the root directory. dir, err := os.Open(path) @@ -163,41 +162,35 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database return } -func (f *fetcher) pullRepository() (commit string, err error) { +func (u *updater) pullRepository() (commit string, err error) { // If the repository doesn't exist, clone it. - if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) { - if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { - return "", ErrFilesystem + if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { + if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil { + return "", vulnsrc.ErrFilesystem } - if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { - f.Clean() + if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { + u.Clean() log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) return "", cerrors.ErrCouldNotDownload } } else { // The repository exists and it needs to be refreshed via a pull. - _, err := utils.Exec(f.repositoryLocalPath, "git", "pull") + _, err := utils.Exec(u.repositoryLocalPath, "git", "pull") if err != nil { - return "", ErrGitFailure + return "", vulnsrc.ErrGitFailure } } - out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD") + out, err := utils.Exec(u.repositoryLocalPath, "git", "rev-parse", "HEAD") if err != nil { - return "", ErrGitFailure + return "", vulnsrc.ErrGitFailure } commit = strings.TrimSpace(string(out)) return } -func (f *fetcher) Clean() { - if f.repositoryLocalPath != "" { - os.RemoveAll(f.repositoryLocalPath) - } -} - type secdb33File struct { Distro string `yaml:"distroversion"` Packages []struct { diff --git a/updater/fetchers/alpine/alpine_test.go b/ext/vulnsrc/alpine/alpine_test.go similarity index 100% rename from updater/fetchers/alpine/alpine_test.go rename to ext/vulnsrc/alpine/alpine_test.go diff --git a/updater/fetchers/alpine/testdata/v33_main.yaml b/ext/vulnsrc/alpine/testdata/v33_main.yaml similarity index 100% rename from updater/fetchers/alpine/testdata/v33_main.yaml rename to ext/vulnsrc/alpine/testdata/v33_main.yaml diff --git a/updater/fetchers/alpine/testdata/v34_main.yaml b/ext/vulnsrc/alpine/testdata/v34_main.yaml similarity index 100% rename from updater/fetchers/alpine/testdata/v34_main.yaml rename to ext/vulnsrc/alpine/testdata/v34_main.yaml diff --git a/updater/fetchers/debian/debian.go b/ext/vulnsrc/debian/debian.go similarity index 91% rename from updater/fetchers/debian/debian.go rename to ext/vulnsrc/debian/debian.go index 4838e622..84d4398f 100644 --- a/updater/fetchers/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package debian implements a vulnerability source updater using the Debian +// Security Tracker. package debian import ( @@ -28,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -39,7 +41,7 @@ const ( updaterFlag = "debianUpdater" ) -var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian") +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/debian") type jsonData map[string]map[string]jsonVuln @@ -54,16 +56,13 @@ type jsonRel struct { Urgency string `json:"urgency"` } -// DebianFetcher implements updater.Fetcher for the Debian Security Tracker -// (https://security-tracker.debian.org). -type DebianFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("debian", &DebianFetcher{}) + vulnsrc.RegisterUpdater("debian", &updater{}) } -// FetchUpdate fetches vulnerability updates from the Debian Security Tracker. -func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Debian vulnerabilities") // Download JSON. @@ -88,7 +87,9 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up return resp, nil } -func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.FetcherResponse, err error) { +func (u *updater) Clean() {} + +func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.UpdateResponse, err error) { hash := latestKnownHash // Defer the addition of flag information to the response. @@ -254,6 +255,3 @@ func urgencyToSeverity(urgency string) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (fetcher *DebianFetcher) Clean() {} diff --git a/updater/fetchers/debian/debian_test.go b/ext/vulnsrc/debian/debian_test.go similarity index 100% rename from updater/fetchers/debian/debian_test.go rename to ext/vulnsrc/debian/debian_test.go diff --git a/updater/fetchers/debian/testdata/fetcher_debian_test.json b/ext/vulnsrc/debian/testdata/fetcher_debian_test.json similarity index 100% rename from updater/fetchers/debian/testdata/fetcher_debian_test.json rename to ext/vulnsrc/debian/testdata/fetcher_debian_test.json diff --git a/ext/vulnsrc/driver.go b/ext/vulnsrc/driver.go new file mode 100644 index 00000000..132eabee --- /dev/null +++ b/ext/vulnsrc/driver.go @@ -0,0 +1,73 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vulnsrc exposes functions to dynamically register vulnerability +// sources used to update a Clair database. +package vulnsrc + +import ( + "errors" + + "github.com/coreos/clair/database" +) + +var ( + // Updaters is the list of registered Updaters. + Updaters = make(map[string]Updater) + + // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. + ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs") + + // ErrGitFailure is returned when a fetcher fails to interact with git. + ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git") +) + +// Updater represents anything that can fetch vulnerabilities and insert them +// into a Clair datastore. +type Updater interface { + // Update gets vulnerability updates. + Update(database.Datastore) (UpdateResponse, error) + + // Clean deletes any allocated resources. + // It is invoked when Clair stops. + Clean() +} + +// UpdateResponse represents the sum of results of an update. +type UpdateResponse struct { + FlagName string + FlagValue string + Notes []string + Vulnerabilities []database.Vulnerability +} + +// RegisterUpdater makes an Updater available by the provided name. +// +// If RegisterUpdater is called twice with the same name, the name is blank, or +// if the provided Updater is nil, this function panics. +func RegisterUpdater(name string, u Updater) { + if name == "" { + panic("vulnsrc: could not register an Updater with an empty name") + } + + if u == nil { + panic("vulnsrc: could not register a nil Updater") + } + + if _, dup := Updaters[name]; dup { + panic("vulnsrc: RegisterUpdater called twice for " + name) + } + + Updaters[name] = u +} diff --git a/updater/fetchers/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go similarity index 93% rename from updater/fetchers/oracle/oracle.go rename to ext/vulnsrc/oracle/oracle.go index fceae36f..32afeaa0 100644 --- a/updater/fetchers/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package oracle implements a vulnerability source updater using the +// Oracle Linux OVAL Database. package oracle import ( @@ -26,7 +28,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" "github.com/coreos/pkg/capnslog" @@ -47,7 +49,7 @@ var ( elsaRegexp = regexp.MustCompile(`com.oracle.elsa-(\d+).xml`) - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/oracle") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/oracle") ) type oval struct { @@ -77,16 +79,13 @@ type criterion struct { Comment string `xml:"comment,attr"` } -// OracleFetcher implements updater.Fetcher and gets vulnerability updates from -// the Oracle Linux OVAL definitions. -type OracleFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("Oracle", &OracleFetcher{}) + vulnsrc.RegisterUpdater("oracle", &updater{}) } -// FetchUpdate gets vulnerability updates from the Oracle Linux OVAL definitions. -func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Oracle Linux vulnerabilities") // Get the first ELSA we have to manage. @@ -153,6 +152,8 @@ func (f *OracleFetcher) FetchUpdate(datastore database.Datastore) (resp updater. return resp, nil } +func (u *updater) Clean() {} + func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) { // Decode the XML. var ov oval @@ -356,6 +357,3 @@ func priority(def definition) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (f *OracleFetcher) Clean() {} diff --git a/updater/fetchers/oracle/oracle_test.go b/ext/vulnsrc/oracle/oracle_test.go similarity index 100% rename from updater/fetchers/oracle/oracle_test.go rename to ext/vulnsrc/oracle/oracle_test.go diff --git a/updater/fetchers/oracle/testdata/fetcher_oracle_test.1.xml b/ext/vulnsrc/oracle/testdata/fetcher_oracle_test.1.xml similarity index 100% rename from updater/fetchers/oracle/testdata/fetcher_oracle_test.1.xml rename to ext/vulnsrc/oracle/testdata/fetcher_oracle_test.1.xml diff --git a/updater/fetchers/oracle/testdata/fetcher_oracle_test.2.xml b/ext/vulnsrc/oracle/testdata/fetcher_oracle_test.2.xml similarity index 100% rename from updater/fetchers/oracle/testdata/fetcher_oracle_test.2.xml rename to ext/vulnsrc/oracle/testdata/fetcher_oracle_test.2.xml diff --git a/updater/fetchers/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go similarity index 94% rename from updater/fetchers/rhel/rhel.go rename to ext/vulnsrc/rhel/rhel.go index 1a0c423d..6714dcd8 100644 --- a/updater/fetchers/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package rhel implements a vulnerability source updater using the +// Red Hat Linux OVAL Database. package rhel import ( @@ -28,7 +30,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -82,17 +84,14 @@ type criterion struct { Comment string `xml:"comment,attr"` } -// RHELFetcher implements updater.Fetcher and gets vulnerability updates from -// the Red Hat OVAL definitions. -type RHELFetcher struct{} +type updater struct{} func init() { - updater.RegisterFetcher("Red Hat", &RHELFetcher{}) + vulnsrc.RegisterUpdater("rhel", &updater{}) } -// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions. -func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { - log.Info("fetching Red Hat vulnerabilities") +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { + log.Info("fetching RHEL vulnerabilities") // Get the first RHSA we have to manage. flagValue, err := datastore.GetKeyValue(updaterFlag) @@ -156,6 +155,8 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe return resp, nil } +func (u *updater) Clean() {} + func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) { // Decode the XML. var ov oval @@ -362,6 +363,3 @@ func priority(def definition) types.Priority { return types.Unknown } } - -// Clean deletes any allocated resources. -func (f *RHELFetcher) Clean() {} diff --git a/updater/fetchers/rhel/rhel_test.go b/ext/vulnsrc/rhel/rhel_test.go similarity index 100% rename from updater/fetchers/rhel/rhel_test.go rename to ext/vulnsrc/rhel/rhel_test.go diff --git a/updater/fetchers/rhel/testdata/fetcher_rhel_test.1.xml b/ext/vulnsrc/rhel/testdata/fetcher_rhel_test.1.xml similarity index 100% rename from updater/fetchers/rhel/testdata/fetcher_rhel_test.1.xml rename to ext/vulnsrc/rhel/testdata/fetcher_rhel_test.1.xml diff --git a/updater/fetchers/rhel/testdata/fetcher_rhel_test.2.xml b/ext/vulnsrc/rhel/testdata/fetcher_rhel_test.2.xml similarity index 100% rename from updater/fetchers/rhel/testdata/fetcher_rhel_test.2.xml rename to ext/vulnsrc/rhel/testdata/fetcher_rhel_test.2.xml diff --git a/updater/fetchers/ubuntu/testdata/fetcher_ubuntu_test.txt b/ext/vulnsrc/ubuntu/testdata/fetcher_ubuntu_test.txt similarity index 100% rename from updater/fetchers/ubuntu/testdata/fetcher_ubuntu_test.txt rename to ext/vulnsrc/ubuntu/testdata/fetcher_ubuntu_test.txt diff --git a/updater/fetchers/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go similarity index 86% rename from updater/fetchers/ubuntu/ubuntu.go rename to ext/vulnsrc/ubuntu/ubuntu.go index 68291557..ea0035da 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package ubuntu implements a vulnerability source updater using the +// Ubuntu CVE Tracker. package ubuntu import ( "bufio" "bytes" - "errors" "fmt" "io" "io/ioutil" @@ -31,7 +32,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/updater" + "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -75,33 +76,27 @@ var ( affectsCaptureRegexp = regexp.MustCompile(`(?P.*)_(?P.*): (?P[^\s]*)( \(+(?P[^()]*)\)+)?`) affectsCaptureRegexpNames = affectsCaptureRegexp.SubexpNames() - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/ubuntu") - - // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. - ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/ubuntu") ) -// UbuntuFetcher implements updater.Fetcher and gets vulnerability updates from -// the Ubuntu CVE Tracker. -type UbuntuFetcher struct { +type updater struct { repositoryLocalPath string } func init() { - updater.RegisterFetcher("Ubuntu", &UbuntuFetcher{}) + vulnsrc.RegisterUpdater("ubuntu", &updater{}) } -// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker. -func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Ubuntu vulnerabilities") // Pull the bzr repository. - if err = fetcher.pullRepository(); err != nil { + if err = u.pullRepository(); err != nil { return resp, err } // Get revision number. - revisionNumber, err := getRevisionNumber(fetcher.repositoryLocalPath) + revisionNumber, err := getRevisionNumber(u.repositoryLocalPath) if err != nil { return resp, err } @@ -113,7 +108,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up } // Get the list of vulnerabilities that we have to update. - modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, fetcher.repositoryLocalPath) + modifiedCVE, err := collectModifiedVulnerabilities(revisionNumber, dbRevisionNumber, u.repositoryLocalPath) if err != nil { return resp, err } @@ -121,7 +116,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up notes := make(map[string]struct{}) for cvePath := range modifiedCVE { // Open the CVE file. - file, err := os.Open(fetcher.repositoryLocalPath + "/" + cvePath) + file, err := os.Open(u.repositoryLocalPath + "/" + cvePath) if err != nil { // This can happen when a file is modified and then moved in another // commit. @@ -166,16 +161,20 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up return } -func (fetcher *UbuntuFetcher) pullRepository() (err error) { +func (u *updater) Clean() { + os.RemoveAll(u.repositoryLocalPath) +} + +func (u *updater) pullRepository() (err error) { // Determine whether we should branch or pull. - if _, pathExists := os.Stat(fetcher.repositoryLocalPath); fetcher.repositoryLocalPath == "" || os.IsNotExist(pathExists) { + if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) { // Create a temporary folder to store the repository. - if fetcher.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { - return ErrFilesystem + if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "ubuntu-cve-tracker"); err != nil { + return vulnsrc.ErrFilesystem } // Branch repository. - if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { + if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out) return cerrors.ErrCouldNotDownload } @@ -184,8 +183,8 @@ func (fetcher *UbuntuFetcher) pullRepository() (err error) { } // Pull repository. - if out, err := utils.Exec(fetcher.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil { - os.RemoveAll(fetcher.repositoryLocalPath) + if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil { + os.RemoveAll(u.repositoryLocalPath) log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out) return cerrors.ErrCouldNotDownload @@ -217,14 +216,14 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat d, err := os.Open(repositoryLocalPath + "/" + folder) if err != nil { log.Errorf("could not open Ubuntu vulnerabilities repository's folder: %s", err) - return nil, ErrFilesystem + return nil, vulnsrc.ErrFilesystem } // Get the FileInfo of all the files in the directory. names, err := d.Readdirnames(-1) if err != nil { log.Errorf("could not read Ubuntu vulnerabilities repository's folder:: %s.", err) - return nil, ErrFilesystem + return nil, vulnsrc.ErrFilesystem } // Add the vulnerabilities to the list. @@ -414,8 +413,3 @@ func ubuntuPriorityToSeverity(priority string) types.Priority { log.Warning("Could not determine a vulnerability priority from: %s", priority) return types.Unknown } - -// Clean deletes any allocated resources. -func (fetcher *UbuntuFetcher) Clean() { - os.RemoveAll(fetcher.repositoryLocalPath) -} diff --git a/updater/fetchers/ubuntu/ubuntu_test.go b/ext/vulnsrc/ubuntu/ubuntu_test.go similarity index 100% rename from updater/fetchers/ubuntu/ubuntu_test.go rename to ext/vulnsrc/ubuntu/ubuntu_test.go diff --git a/notifier/notifier.go b/notifier/notifier.go index 81e480cd..d1fa6b2f 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package notifier fetches notifications from the database and informs the specified remote handler -// about their existences, inviting the third party to actively query the API about it. +// Package notifier fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. package notifier import ( @@ -26,6 +27,7 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" ) @@ -40,8 +42,6 @@ const ( var ( log = capnslog.NewPackageLogger("github.com/coreos/clair", "notifier") - notifiers = make(map[string]Notifier) - promNotifierLatencyMilliseconds = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "clair_notifier_latency_milliseconds", Help: "Time it takes to send a notification after it's been created.", @@ -53,57 +53,29 @@ var ( }, []string{"backend"}) ) -// Notifier represents anything that can transmit notifications. -type Notifier interface { - // Configure attempts to initialize the notifier with the provided configuration. - // It returns whether the notifier is enabled or not. - Configure(*config.NotifierConfig) (bool, error) - // Send informs the existence of the specified notification. - Send(notification database.VulnerabilityNotification) error -} - func init() { prometheus.MustRegister(promNotifierLatencyMilliseconds) prometheus.MustRegister(promNotifierBackendErrorsTotal) } -// RegisterNotifier makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterNotifier(name string, n Notifier) { - if name == "" { - panic("notifier: could not register a Notifier with an empty name") - } - - if n == nil { - panic("notifier: could not register a nil Notifier") - } - - if _, dup := notifiers[name]; dup { - panic("notifier: RegisterNotifier called twice for " + name) - } - - notifiers[name] = n -} - // Run starts the Notifier service. func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) { defer stopper.End() // Configure registered notifiers. - for notifierName, notifier := range notifiers { - if configured, err := notifier.Configure(config); configured { - log.Infof("notifier '%s' configured\n", notifierName) + for senderName, sender := range notification.Senders { + if configured, err := sender.Configure(config); configured { + log.Infof("sender '%s' configured\n", senderName) } else { - delete(notifiers, notifierName) + delete(notification.Senders, senderName) if err != nil { - log.Errorf("could not configure notifier '%s': %s", notifierName, err) + log.Errorf("could not configure notifier '%s': %s", senderName, err) } } } // Do not run the updater if there is no notifier enabled. - if len(notifiers) == 0 { + if len(notification.Senders) == 0 { log.Infof("notifier service is disabled") return } @@ -175,31 +147,31 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } } -func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { +func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { // Send notification. - for notifierName, notifier := range notifiers { + for senderName, sender := range notification.Senders { var attempts int var backOff time.Duration for { // Max attempts exceeded. if attempts >= maxAttempts { - log.Infof("giving up on sending notification '%s' via notifier '%s': max attempts exceeded (%d)\n", notification.Name, notifierName, maxAttempts) + log.Infof("giving up on sending notification '%s' via sender '%s': max attempts exceeded (%d)\n", n.Name, senderName, maxAttempts) return false, false } // Backoff. if backOff > 0 { - log.Infof("waiting %v before retrying to send notification '%s' via notifier '%s' (Attempt %d / %d)\n", backOff, notification.Name, notifierName, attempts+1, maxAttempts) + log.Infof("waiting %v before retrying to send notification '%s' via sender '%s' (Attempt %d / %d)\n", backOff, n.Name, senderName, attempts+1, maxAttempts) if !st.Sleep(backOff) { return false, true } } // Send using the current notifier. - if err := notifier.Send(notification); err != nil { + if err := sender.Send(n); err != nil { // Send failed; increase attempts/backoff and retry. - promNotifierBackendErrorsTotal.WithLabelValues(notifierName).Inc() - log.Errorf("could not send notification '%s' via notifier '%s': %v", notification.Name, notifierName, err) + promNotifierBackendErrorsTotal.WithLabelValues(senderName).Inc() + log.Errorf("could not send notification '%s' via notifier '%s': %v", n.Name, senderName, err) backOff = timeutil.ExpBackoff(backOff, maxBackOff) attempts++ continue @@ -210,6 +182,6 @@ func handleTask(notification database.VulnerabilityNotification, st *utils.Stopp } } - log.Infof("successfully sent notification '%s'\n", notification.Name) + log.Infof("successfully sent notification '%s'\n", n.Name) return true, false } diff --git a/updater/fetchers.go b/updater/fetchers.go deleted file mode 100644 index 609ecd40..00000000 --- a/updater/fetchers.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package updater - -import "github.com/coreos/clair/database" - -var fetchers = make(map[string]Fetcher) - -// Fetcher represents anything that can fetch vulnerabilities. -type Fetcher interface { - // FetchUpdate gets vulnerability updates. - FetchUpdate(database.Datastore) (FetcherResponse, error) - - // Clean deletes any allocated resources. - // It is invoked when Clair stops. - Clean() -} - -// FetcherResponse represents the sum of results of an update. -type FetcherResponse struct { - FlagName string - FlagValue string - Notes []string - Vulnerabilities []database.Vulnerability -} - -// RegisterFetcher makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterFetcher(name string, f Fetcher) { - if name == "" { - panic("updater: could not register a Fetcher with an empty name") - } - - if f == nil { - panic("updater: could not register a nil Fetcher") - } - - if _, dup := fetchers[name]; dup { - panic("updater: RegisterFetcher called twice for " + name) - } - - fetchers[name] = f -} diff --git a/updater/metadata_fetchers.go b/updater/metadata_fetchers.go deleted file mode 100644 index 72e764af..00000000 --- a/updater/metadata_fetchers.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package updater - -import ( - "sync" - - "github.com/coreos/clair/database" -) - -var metadataFetchers = make(map[string]MetadataFetcher) - -type VulnerabilityWithLock struct { - *database.Vulnerability - Lock sync.Mutex -} - -// MetadataFetcher -type MetadataFetcher interface { - // Load runs right before the Updater calls AddMetadata for each vulnerabilities. - Load(database.Datastore) error - - // AddMetadata adds metadata to the given database.Vulnerability. - // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. - AddMetadata(*VulnerabilityWithLock) error - - // Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities. - Unload() - - // Clean deletes any allocated resources. - // It is invoked when Clair stops. - Clean() -} - -// RegisterFetcher makes a Fetcher available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. -func RegisterMetadataFetcher(name string, f MetadataFetcher) { - if name == "" { - panic("updater: could not register a MetadataFetcher with an empty name") - } - - if f == nil { - panic("updater: could not register a nil MetadataFetcher") - } - - if _, dup := fetchers[name]; dup { - panic("updater: RegisterMetadataFetcher called twice for " + name) - } - - metadataFetchers[name] = f -} diff --git a/updater/metadata_fetchers/nvd/nested_read_closer.go b/updater/metadata_fetchers/nvd/nested_read_closer.go deleted file mode 100644 index 3a99b174..00000000 --- a/updater/metadata_fetchers/nvd/nested_read_closer.go +++ /dev/null @@ -1,19 +0,0 @@ -package nvd - -import "io" - -// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed -// io.ReadCloser. -// It allows chaining io.ReadCloser together and still keep the ability to close them all in a -// simple manner. -type NestedReadCloser struct { - io.Reader - NestedReadClosers []io.ReadCloser -} - -// Close closes the gzip.Reader and the underlying io.ReadCloser. -func (nrc *NestedReadCloser) Close() { - for _, nestedReadCloser := range nrc.NestedReadClosers { - nestedReadCloser.Close() - } -} diff --git a/updater/updater.go b/updater/updater.go index bfb8680b..01feb0c2 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package updater updates the vulnerability database periodically using -// the registered vulnerability fetchers. +// Package updater updates the vulnerability database periodically using the +// registered vulnerability source updaters and vulnerability metadata +// appenders. package updater import ( @@ -22,12 +23,16 @@ import ( "sync" "time" - "github.com/coreos/clair/config" - "github.com/coreos/clair/database" - "github.com/coreos/clair/utils" "github.com/coreos/pkg/capnslog" "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/vulnmdsrc" + "github.com/coreos/clair/ext/vulnsrc" + "github.com/coreos/clair/utils" + "github.com/coreos/clair/utils/types" ) const ( @@ -147,11 +152,11 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S } // Clean resources. - for _, metadataFetcher := range metadataFetchers { - metadataFetcher.Clean() + for _, appenders := range vulnmdsrc.Appenders { + appenders.Clean() } - for _, fetcher := range fetchers { - fetcher.Clean() + for _, updaters := range vulnsrc.Updaters { + updaters.Clean() } log.Info("updater service stopped") @@ -209,10 +214,10 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Fetch updates in parallel. log.Info("fetching vulnerability updates") - var responseC = make(chan *FetcherResponse, 0) - for n, f := range fetchers { - go func(name string, fetcher Fetcher) { - response, err := fetcher.FetchUpdate(datastore) + var responseC = make(chan *vulnsrc.UpdateResponse, 0) + for n, u := range vulnsrc.Updaters { + go func(name string, u vulnsrc.Updater) { + response, err := u.Update(datastore) if err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when fetching update '%s': %s.", name, err) @@ -222,11 +227,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st } responseC <- &response - }(n, f) + }(n, u) } // Collect results of updates. - for i := 0; i < len(fetchers); i++ { + for i := 0; i < len(vulnsrc.Updaters); i++ { resp := <-responseC if resp != nil { vulnerabilities = append(vulnerabilities, doVulnerabilitiesNamespacing(resp.Vulnerabilities)...) @@ -243,42 +248,43 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel. func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability { - if len(metadataFetchers) == 0 { + if len(vulnmdsrc.Appenders) == 0 { return vulnerabilities } log.Info("adding metadata to vulnerabilities") - // Wrap vulnerabilities in VulnerabilityWithLock. - // It ensures that only one metadata fetcher at a time can modify the Metadata map. - vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities)) + // Add a mutex to each vulnerability to ensure that only one appender at a + // time can modify the vulnerability's Metadata map. + lockableVulnerabilities := make([]*lockableVulnerability, 0, len(vulnerabilities)) for i := 0; i < len(vulnerabilities); i++ { - vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{ + lockableVulnerabilities = append(lockableVulnerabilities, &lockableVulnerability{ Vulnerability: &vulnerabilities[i], }) } var wg sync.WaitGroup - wg.Add(len(metadataFetchers)) + wg.Add(len(vulnmdsrc.Appenders)) - for n, f := range metadataFetchers { - go func(name string, metadataFetcher MetadataFetcher) { + for n, a := range vulnmdsrc.Appenders { + go func(name string, appender vulnmdsrc.Appender) { defer wg.Done() - // Load the metadata fetcher. - if err := metadataFetcher.Load(datastore); err != nil { + // Build up a metadata cache. + if err := appender.BuildCache(datastore); err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err) return } - // Add metadata to each vulnerability. - for _, vulnerability := range vulnerabilitiesWithLocks { - metadataFetcher.AddMetadata(vulnerability) + // Append vulnerability metadata to each vulnerability. + for _, vulnerability := range lockableVulnerabilities { + appender.Append(vulnerability.Name, vulnerability.appendFunc) } - metadataFetcher.Unload() - }(n, f) + // Purge the metadata cache. + appender.PurgeCache() + }(n, a) } wg.Wait() @@ -305,6 +311,29 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) { return time.Unix(lastUpdateTS, 0).UTC(), false, nil } +type lockableVulnerability struct { + *database.Vulnerability + sync.Mutex +} + +func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity types.Priority) { + lv.Lock() + defer lv.Unlock() + + // If necessary, initialize the metadata map for the vulnerability. + if lv.Metadata == nil { + lv.Metadata = make(map[string]interface{}) + } + + // Append the metadata. + lv.Metadata[metadataKey] = metadata + + // If necessary, provide a severity for the vulnerability. + if lv.Severity == "" || lv.Severity == types.Unknown { + lv.Severity = severity + } +} + // doVulnerabilitiesNamespacing takes Vulnerabilities that don't have a Namespace and split them // into multiple vulnerabilities that have a Namespace and only contains the FixedIn // FeatureVersions corresponding to their Namespace. From 6c90635848da7aa3d5c7ed011773de93cf119775 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 3 Jan 2017 22:09:17 -0500 Subject: [PATCH 02/23] README: update to reflect ext directory --- README.md | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 58734082..64273f24 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,15 @@ Please use [releases] instead of the `master` branch in order to get stable bina Clair is an open source project for the static analysis of vulnerabilities in [appc] and [docker] containers. Vulnerability data is continuously imported from a known set of sources and correlated with the indexed contents of container images in order to produce lists of vulnerabilities that threaten a container. -When vulnerability data changes upstream, the previous state and new state of the vulnerability along with the images they affect can be sent via webhook to a configured endpoint. -All major components can be [customized programmatically] at compile-time without forking the project. +When vulnerability data changes upstream, a notification can be delivered, and the API queried to provide the previous state and new state of the vulnerability along with the images affected by both. +All major components can be [extended programmatically] at compile-time without forking the project. Our goal is to enable a more transparent view of the security of container-based infrastructure. Thus, the project was named `Clair` after the French term which translates to *clear*, *bright*, *transparent*. [appc]: https://github.com/appc/spec [docker]: https://github.com/docker/docker/blob/master/image/spec/v1.md -[customized programmatically]: #customization +[extended programmatically]: #customization [releases]: https://github.com/coreos/clair/releases ## Common Use Cases @@ -127,7 +127,8 @@ While container images for every releases are available at [quay.io/repository/c ## Documentation -The latest stable documentation can be found [on the CoreOS website]. Documentation for the current branch can be found [inside the Documentation directory][docs-dir] at the root of the project's source code. +The latest stable documentation can be found [on the CoreOS website]. +Documentation for the current branch can be found [inside the Documentation directory][docs-dir] at the root of the project's source code. [on the CoreOS website]: https://coreos.com/clair/docs/latest/ [docs-dir]: /Documentation @@ -143,7 +144,8 @@ The latest stable documentation can be found [on the CoreOS website]. Documentat - *Detector* - a Go package that identifies the content, *namespaces* and *features* from a *layer* - *Namespace* - a context around *features* and *vulnerabilities* (e.g. an operating system) - *Feature* - anything that when present could be an indication of a *vulnerability* (e.g. the presence of a file or an installed software package) -- *Fetcher* - a Go package that tracks an upstream vulnerability database and imports them into Clair +- *Vulnerability Updater* - a Go package that tracks upstream vulnerability data and imports them into Clair +- *Vulnerability Metadata Appender* - a Go package that tracks upstream vulnerability metadata and appends them into vulnerabilities managed by Clair ### Vulnerability Analysis @@ -164,13 +166,13 @@ By indexing the features of an image into the database, images only need to be r | [Red Hat Security Data] | CentOS 5, 6, 7 namespaces | [rpm] | [CVRF] | | [Oracle Linux Security Data] | Oracle Linux 5, 6, 7 namespaces | [rpm] | [CVRF] | | [Alpine SecDB] | Alpine 3.3, Alpine 3.4 namespaces | [apk] | [MIT] | -| [NVD] | Generic Vulnerability Metadata | N/A | [Public Domain] | +| [NIST NVD] | Generic Vulnerability Metadata | N/A | [Public Domain] | [Debian Security Bug Tracker]: https://security-tracker.debian.org/tracker [Ubuntu CVE Tracker]: https://launchpad.net/ubuntu-cve-tracker [Red Hat Security Data]: https://www.redhat.com/security/data/metrics [Oracle Linux Security Data]: https://linux.oracle.com/security/ -[NVD]: https://nvd.nist.gov +[NIST NVD]: https://nvd.nist.gov [dpkg]: https://en.wikipedia.org/wiki/dpkg [rpm]: http://www.rpm.org [Debian]: https://www.debian.org/license @@ -185,21 +187,13 @@ By indexing the features of an image into the database, images only need to be r ### Customization The major components of Clair are all programmatically extensible in the same way Go's standard [database/sql] package is extensible. +Everything extendable is located in the `ext` directory. -Custom behavior can be accomplished by creating a package that contains a type that implements an interface declared in Clair and registering that interface in [init()]. To expose the new behavior, unqualified imports to the package must be added in your [main.go], which should then start Clair using `Boot(*config.Config)`. +Custom behavior can be accomplished by creating a package that contains a type that implements an interface declared in Clair and registering that interface in [init()]. +To expose the new behavior, unqualified imports to the package must be added in your own custom [main.go], which should then start Clair using `Boot(*config.Config)`. -The following interfaces can have custom implementations registered via [init()] at compile time: - -- `Datastore` - the backing storage -- `Notifier` - the means by which endpoints are notified of vulnerability changes -- `Fetcher` - the sources of vulnerability data that is automatically imported -- `MetadataFetcher` - the sources of vulnerability metadata that is automatically added to known vulnerabilities -- `DataDetector` - the means by which contents of an image are detected -- `FeatureDetector` - the means by which features are identified from a layer -- `NamespaceDetector` - the means by which a namespace is identified from a layer - -[init()]: https://golang.org/doc/effective_go.html#init [database/sql]: https://godoc.org/database/sql +[init()]: https://golang.org/doc/effective_go.html#init [main.go]: https://github.com/coreos/clair/blob/master/cmd/clair/main.go ## Related Links From 03bac0f1b6f504a416937e309a62fdfc308dc397 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 00:56:08 -0500 Subject: [PATCH 03/23] pkg: utils/tar.go -> pkg/tarutil --- api/v1/routes.go | 6 +- utils/tar.go => pkg/tarutil/tarutil.go | 141 ++++++++++-------- pkg/tarutil/tarutil_test.go | 80 ++++++++++ .../tarutil}/testdata/utils_test.tar | Bin .../tarutil}/testdata/utils_test.tar.bz2 | Bin .../tarutil}/testdata/utils_test.tar.gz | Bin .../tarutil}/testdata/utils_test.tar.xz | Bin utils/http/http.go | 4 +- utils/utils_test.go | 40 ----- 9 files changed, 160 insertions(+), 111 deletions(-) rename utils/tar.go => pkg/tarutil/tarutil.go (70%) create mode 100644 pkg/tarutil/tarutil_test.go rename {utils => pkg/tarutil}/testdata/utils_test.tar (100%) rename {utils => pkg/tarutil}/testdata/utils_test.tar.bz2 (100%) rename {utils => pkg/tarutil}/testdata/utils_test.tar.gz (100%) rename {utils => pkg/tarutil}/testdata/utils_test.tar.xz (100%) diff --git a/api/v1/routes.go b/api/v1/routes.go index ed82ebe2..344e2e28 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -27,7 +27,7 @@ import ( "github.com/coreos/clair/api/context" "github.com/coreos/clair/database" - "github.com/coreos/clair/utils" + "github.com/coreos/clair/pkg/tarutil" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" ) @@ -111,8 +111,8 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) if err != nil { - if err == utils.ErrCouldNotExtract || - err == utils.ErrExtractedFileTooBig || + if err == tarutil.ErrCouldNotExtract || + err == tarutil.ErrExtractedFileTooBig || err == worker.ErrUnsupported { writeResponse(w, r, statusUnprocessableEntity, LayerEnvelope{Error: &Error{err.Error()}}) return postLayerRoute, statusUnprocessableEntity diff --git a/utils/tar.go b/pkg/tarutil/tarutil.go similarity index 70% rename from utils/tar.go rename to pkg/tarutil/tarutil.go index f2cff669..357f96da 100644 --- a/utils/tar.go +++ b/pkg/tarutil/tarutil.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +// Package tarutil implements some tar utility functions. +package tarutil import ( "archive/tar" @@ -29,28 +30,87 @@ import ( var ( // ErrCouldNotExtract occurs when an extraction fails. - ErrCouldNotExtract = errors.New("utils: could not extract the archive") + ErrCouldNotExtract = errors.New("tarutil: could not extract the archive") // ErrExtractedFileTooBig occurs when a file to extract is too big. - ErrExtractedFileTooBig = errors.New("utils: could not extract one or more files from the archive: file too big") + ErrExtractedFileTooBig = errors.New("tarutil: could not extract one or more files from the archive: file too big") - readLen = 6 // max bytes to sniff + // MaxExtractableFileSize enforces the maximum size of a single file within a + // tarball that will be extracted. This protects against malicious files that + // may used in an attempt to perform a Denial of Service attack. + MaxExtractableFileSize int64 = 200 * 1024 * 1024 // 200 MiB + readLen = 6 // max bytes to sniff gzipHeader = []byte{0x1f, 0x8b} bzip2Header = []byte{0x42, 0x5a, 0x68} xzHeader = []byte{0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00} ) -// XzReader is an io.ReadCloser which decompresses xz compressed data. +// FilesMap is a map of files' paths to their contents. +type FilesMap map[string][]byte + +// ExtractFiles decompresses and extracts only the specified files from an +// io.Reader representing an archive. +func ExtractFiles(r io.Reader, filenames []string) (FilesMap, error) { + data := make(map[string][]byte) + + // Decompress the archive. + tr, err := NewTarReadCloser(r) + if err != nil { + return data, ErrCouldNotExtract + } + defer tr.Close() + + // For each element in the archive + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return data, ErrCouldNotExtract + } + + // Get element filename + filename := hdr.Name + filename = strings.TrimPrefix(filename, "./") + + // Determine if we should extract the element + toBeExtracted := false + for _, s := range filenames { + if strings.HasPrefix(filename, s) { + toBeExtracted = true + break + } + } + + if toBeExtracted { + // File size limit + if hdr.Size > MaxExtractableFileSize { + return data, ErrExtractedFileTooBig + } + + // Extract the element + if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink || hdr.Typeflag == tar.TypeReg { + d, _ := ioutil.ReadAll(tr) + data[filename] = d + } + } + } + + return data, nil +} + +// XzReader implements io.ReadCloser for data compressed via `xz`. type XzReader struct { io.ReadCloser cmd *exec.Cmd closech chan error } -// NewXzReader shells out to a command line xz executable (if -// available) to decompress the given io.Reader using the xz -// compression format and returns an *XzReader. +// NewXzReader returns an io.ReadCloser by executing a command line `xz` +// executable to decompress the provided io.Reader. +// // It is the caller's responsibility to call Close on the XzReader when done. func NewXzReader(r io.Reader) (*XzReader, error) { rpipe, wpipe := io.Pipe() @@ -74,6 +134,7 @@ func NewXzReader(r io.Reader) (*XzReader, error) { return &XzReader{rpipe, cmd, closech}, nil } +// Close cleans up the resources used by an XzReader. func (r *XzReader) Close() error { r.ReadCloser.Close() r.cmd.Process.Kill() @@ -88,72 +149,20 @@ type TarReadCloser struct { io.Closer } +// Close cleans up the resources used by a TarReadCloser. func (r *TarReadCloser) Close() error { return r.Closer.Close() } -// SelectivelyExtractArchive extracts the specified files and folders -// from targz data read from the given reader and store them in a map indexed by file paths -func SelectivelyExtractArchive(r io.Reader, prefix string, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - data := make(map[string][]byte) - - // Create a tar or tar/tar-gzip/tar-bzip2/tar-xz reader - tr, err := getTarReader(r) - if err != nil { - return data, ErrCouldNotExtract - } - defer tr.Close() - - // For each element in the archive - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return data, ErrCouldNotExtract - } - - // Get element filename - filename := hdr.Name - filename = strings.TrimPrefix(filename, "./") - if prefix != "" { - filename = strings.TrimPrefix(filename, prefix) - } - - // Determine if we should extract the element - toBeExtracted := false - for _, s := range toExtract { - if strings.HasPrefix(filename, s) { - toBeExtracted = true - break - } - } - - if toBeExtracted { - // File size limit - if maxFileSize > 0 && hdr.Size > maxFileSize { - return data, ErrExtractedFileTooBig - } - - // Extract the element - if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink || hdr.Typeflag == tar.TypeReg { - d, _ := ioutil.ReadAll(tr) - data[filename] = d - } - } - } - - return data, nil -} - -// getTarReader returns a TarReaderCloser associated with the specified io.Reader. +// NewTarReadCloser attempts to detect the compression algorithm for an +// io.Reader and returns a TarReadCloser wrapping the Reader to transparently +// decompress the contents. // // Gzip/Bzip2/XZ detection is done by using the magic numbers: // Gzip: the first two bytes should be 0x1f and 0x8b. Defined in the RFC1952. // Bzip2: the first three bytes should be 0x42, 0x5a and 0x68. No RFC. // XZ: the first three bytes should be 0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00. No RFC. -func getTarReader(r io.Reader) (*TarReadCloser, error) { +func NewTarReadCloser(r io.Reader) (*TarReadCloser, error) { br := bufio.NewReader(r) header, err := br.Peek(readLen) if err == nil { diff --git a/pkg/tarutil/tarutil_test.go b/pkg/tarutil/tarutil_test.go new file mode 100644 index 00000000..9db377ab --- /dev/null +++ b/pkg/tarutil/tarutil_test.go @@ -0,0 +1,80 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutil + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +var testTarballs = []string{ + "utils_test.tar", + "utils_test.tar.gz", + "utils_test.tar.bz2", + "utils_test.tar.xz", +} + +func testfilepath(filename string) string { + _, path, _, _ := runtime.Caller(0) + testDataDir := "/testdata" + return filepath.Join(filepath.Dir(path), testDataDir, filename) +} + +func TestExtract(t *testing.T) { + for _, filename := range testTarballs { + f, err := os.Open(testfilepath(filename)) + assert.Nil(t, err) + defer f.Close() + + data, err := ExtractFiles(f, []string{"test/"}) + assert.Nil(t, err) + + if c, n := data["test/test.txt"]; !n { + assert.Fail(t, "test/test.txt should have been extracted") + } else { + assert.NotEqual(t, 0, len(c) > 0, "test/test.txt file is empty") + } + if _, n := data["test.txt"]; n { + assert.Fail(t, "test.txt should not be extracted") + } + } +} + +func TestExtractUncompressedData(t *testing.T) { + for _, filename := range testTarballs { + f, err := os.Open(testfilepath(filename)) + assert.Nil(t, err) + defer f.Close() + + _, err = ExtractFiles(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), []string{}) + assert.Error(t, err, "Extracting uncompressed data should return an error") + } +} + +func TestMaxExtractableFileSize(t *testing.T) { + for _, filename := range testTarballs { + f, err := os.Open(testfilepath(filename)) + assert.Nil(t, err) + defer f.Close() + MaxExtractableFileSize = 50 + _, err = ExtractFiles(f, []string{"test"}) + assert.Equal(t, ErrExtractedFileTooBig, err) + } +} diff --git a/utils/testdata/utils_test.tar b/pkg/tarutil/testdata/utils_test.tar similarity index 100% rename from utils/testdata/utils_test.tar rename to pkg/tarutil/testdata/utils_test.tar diff --git a/utils/testdata/utils_test.tar.bz2 b/pkg/tarutil/testdata/utils_test.tar.bz2 similarity index 100% rename from utils/testdata/utils_test.tar.bz2 rename to pkg/tarutil/testdata/utils_test.tar.bz2 diff --git a/utils/testdata/utils_test.tar.gz b/pkg/tarutil/testdata/utils_test.tar.gz similarity index 100% rename from utils/testdata/utils_test.tar.gz rename to pkg/tarutil/testdata/utils_test.tar.gz diff --git a/utils/testdata/utils_test.tar.xz b/pkg/tarutil/testdata/utils_test.tar.xz similarity index 100% rename from utils/testdata/utils_test.tar.xz rename to pkg/tarutil/testdata/utils_test.tar.xz diff --git a/utils/http/http.go b/utils/http/http.go index 1aac81f3..b02430a4 100644 --- a/utils/http/http.go +++ b/utils/http/http.go @@ -21,7 +21,7 @@ import ( "net/http" "github.com/coreos/clair/database" - "github.com/coreos/clair/utils" + "github.com/coreos/clair/pkg/tarutil" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" ) @@ -56,7 +56,7 @@ func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) { httpStatus = http.StatusNotFound case database.ErrBackendException: httpStatus = http.StatusServiceUnavailable - case worker.ErrParentUnknown, worker.ErrUnsupported, utils.ErrCouldNotExtract, utils.ErrExtractedFileTooBig: + case worker.ErrParentUnknown, worker.ErrUnsupported, tarutil.ErrCouldNotExtract, tarutil.ErrExtractedFileTooBig: httpStatus = http.StatusBadRequest } } diff --git a/utils/utils_test.go b/utils/utils_test.go index 57bfc9e0..83cc6cce 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -15,10 +15,6 @@ package utils import ( - "bytes" - "os" - "path/filepath" - "runtime" "testing" "github.com/pborman/uuid" @@ -61,42 +57,6 @@ func TestString(t *testing.T) { assert.False(t, Contains("c", []string{"a", "b"})) } -// TestTar tests the tar.go file -func TestTar(t *testing.T) { - var err error - var data map[string][]byte - _, path, _, _ := runtime.Caller(0) - testDataDir := "/testdata" - for _, filename := range []string{"utils_test.tar.gz", "utils_test.tar.bz2", "utils_test.tar.xz", "utils_test.tar"} { - testArchivePath := filepath.Join(filepath.Dir(path), testDataDir, filename) - - // Extract non compressed data - data, err = SelectivelyExtractArchive(bytes.NewReader([]byte("that string does not represent a tar or tar-gzip file")), "", []string{}, 0) - assert.Error(t, err, "Extracting non compressed data should return an error") - - // Extract an archive - f, _ := os.Open(testArchivePath) - defer f.Close() - data, err = SelectivelyExtractArchive(f, "", []string{"test/"}, 0) - assert.Nil(t, err) - - if c, n := data["test/test.txt"]; !n { - assert.Fail(t, "test/test.txt should have been extracted") - } else { - assert.NotEqual(t, 0, len(c) > 0, "test/test.txt file is empty") - } - if _, n := data["test.txt"]; n { - assert.Fail(t, "test.txt should not be extracted") - } - - // File size limit - f, _ = os.Open(testArchivePath) - defer f.Close() - data, err = SelectivelyExtractArchive(f, "", []string{"test"}, 50) - assert.Equal(t, ErrExtractedFileTooBig, err) - } -} - func TestCleanURL(t *testing.T) { assert.Equal(t, "Test http://test.cn/test Test", CleanURL("Test http://test.cn/test?foo=bar&bar=foo Test")) } From 8dea7442369ed9e54c9d56ed8f9027e4e57f2726 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 01:37:18 -0500 Subject: [PATCH 04/23] delete unused types.Version --- utils/types/version.go | 296 ------------------------------------ utils/types/version_test.go | 243 ----------------------------- 2 files changed, 539 deletions(-) delete mode 100644 utils/types/version.go delete mode 100644 utils/types/version_test.go diff --git a/utils/types/version.go b/utils/types/version.go deleted file mode 100644 index 4c241f24..00000000 --- a/utils/types/version.go +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "database/sql/driver" - "encoding/json" - "errors" - "strconv" - "strings" - "unicode" -) - -// Version represents a package version -type Version struct { - epoch int - version string - revision string -} - -var ( - // MinVersion is a special package version which is always sorted first - MinVersion = Version{version: "#MINV#"} - // MaxVersion is a special package version which is always sorted last - MaxVersion = Version{version: "#MAXV#"} - - versionAllowedSymbols = []rune{'.', '-', '+', '~', ':', '_'} - revisionAllowedSymbols = []rune{'.', '+', '~', '_'} -) - -// NewVersion function parses a string into a Version struct which can be compared -// -// The implementation is based on http://man.he.net/man5/deb-version -// on https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version -// -// It uses the dpkg-1.17.25's algorithm (lib/parsehelp.c) -func NewVersion(str string) (Version, error) { - var version Version - - // Trim leading and trailing space - str = strings.TrimSpace(str) - - if len(str) == 0 { - return Version{}, errors.New("Version string is empty") - } - - // Max/Min versions - if str == MaxVersion.String() { - return MaxVersion, nil - } - if str == MinVersion.String() { - return MinVersion, nil - } - - // Find epoch - sepepoch := strings.Index(str, ":") - if sepepoch > -1 { - intepoch, err := strconv.Atoi(str[:sepepoch]) - if err == nil { - version.epoch = intepoch - } else { - return Version{}, errors.New("epoch in version is not a number") - } - if intepoch < 0 { - return Version{}, errors.New("epoch in version is negative") - } - } else { - version.epoch = 0 - } - - // Find version / revision - seprevision := strings.LastIndex(str, "-") - if seprevision > -1 { - version.version = str[sepepoch+1 : seprevision] - version.revision = str[seprevision+1:] - } else { - version.version = str[sepepoch+1:] - version.revision = "" - } - // Verify format - if len(version.version) == 0 { - return Version{}, errors.New("No version") - } - - if !unicode.IsDigit(rune(version.version[0])) { - return Version{}, errors.New("version does not start with digit") - } - - for i := 0; i < len(version.version); i = i + 1 { - r := rune(version.version[i]) - if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(versionAllowedSymbols, r) { - return Version{}, errors.New("invalid character in version") - } - } - - for i := 0; i < len(version.revision); i = i + 1 { - r := rune(version.revision[i]) - if !unicode.IsDigit(r) && !unicode.IsLetter(r) && !containsRune(revisionAllowedSymbols, r) { - return Version{}, errors.New("invalid character in revision") - } - } - - return version, nil -} - -// NewVersionUnsafe is just a wrapper around NewVersion that ignore potentiel -// parsing error. Useful for test purposes -func NewVersionUnsafe(str string) Version { - v, _ := NewVersion(str) - return v -} - -// Compare function compares two Debian-like package version -// -// The implementation is based on http://man.he.net/man5/deb-version -// on https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version -// -// It uses the dpkg-1.17.25's algorithm (lib/version.c) -func (a Version) Compare(b Version) int { - // Quick check - if a == b { - return 0 - } - - // Max/Min comparison - if a == MinVersion || b == MaxVersion { - return -1 - } - if b == MinVersion || a == MaxVersion { - return 1 - } - - // Compare epochs - if a.epoch > b.epoch { - return 1 - } - if a.epoch < b.epoch { - return -1 - } - - // Compare version - rc := verrevcmp(a.version, b.version) - if rc != 0 { - return signum(rc) - } - - // Compare revision - return signum(verrevcmp(a.revision, b.revision)) -} - -// String returns the string representation of a Version -func (v Version) String() (s string) { - if v.epoch != 0 { - s = strconv.Itoa(v.epoch) + ":" - } - s += v.version - if v.revision != "" { - s += "-" + v.revision - } - return -} - -func (v Version) MarshalJSON() ([]byte, error) { - return json.Marshal(v.String()) -} - -func (v *Version) UnmarshalJSON(b []byte) (err error) { - var str string - json.Unmarshal(b, &str) - vp := NewVersionUnsafe(str) - *v = vp - return -} - -func (v *Version) Scan(value interface{}) (err error) { - val, ok := value.([]byte) - if !ok { - return errors.New("could not scan a Version from a non-string input") - } - *v, err = NewVersion(string(val)) - return -} - -func (v *Version) Value() (driver.Value, error) { - return v.String(), nil -} - -func verrevcmp(t1, t2 string) int { - t1, rt1 := nextRune(t1) - t2, rt2 := nextRune(t2) - - for rt1 != nil || rt2 != nil { - firstDiff := 0 - - for (rt1 != nil && !unicode.IsDigit(*rt1)) || (rt2 != nil && !unicode.IsDigit(*rt2)) { - ac := 0 - bc := 0 - if rt1 != nil { - ac = order(*rt1) - } - if rt2 != nil { - bc = order(*rt2) - } - - if ac != bc { - return ac - bc - } - - t1, rt1 = nextRune(t1) - t2, rt2 = nextRune(t2) - } - for rt1 != nil && *rt1 == '0' { - t1, rt1 = nextRune(t1) - } - for rt2 != nil && *rt2 == '0' { - t2, rt2 = nextRune(t2) - } - for rt1 != nil && unicode.IsDigit(*rt1) && rt2 != nil && unicode.IsDigit(*rt2) { - if firstDiff == 0 { - firstDiff = int(*rt1) - int(*rt2) - } - t1, rt1 = nextRune(t1) - t2, rt2 = nextRune(t2) - } - if rt1 != nil && unicode.IsDigit(*rt1) { - return 1 - } - if rt2 != nil && unicode.IsDigit(*rt2) { - return -1 - } - if firstDiff != 0 { - return firstDiff - } - } - - return 0 -} - -// order compares runes using a modified ASCII table -// so that letters are sorted earlier than non-letters -// and so that tildes sorts before anything -func order(r rune) int { - if unicode.IsDigit(r) { - return 0 - } - - if unicode.IsLetter(r) { - return int(r) - } - - if r == '~' { - return -1 - } - - return int(r) + 256 -} - -func nextRune(str string) (string, *rune) { - if len(str) >= 1 { - r := rune(str[0]) - return str[1:], &r - } - return str, nil -} - -func containsRune(s []rune, e rune) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - -func signum(a int) int { - switch { - case a < 0: - return -1 - case a > 0: - return +1 - } - - return 0 -} diff --git a/utils/types/version_test.go b/utils/types/version_test.go deleted file mode 100644 index 4796ceb9..00000000 --- a/utils/types/version_test.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -const ( - LESS = -1 - EQUAL = 0 - GREATER = 1 -) - -func TestCompareSimpleVersion(t *testing.T) { - cases := []struct { - v1 Version - expected int - v2 Version - }{ - {Version{}, EQUAL, Version{}}, - {Version{epoch: 1}, LESS, Version{epoch: 2}}, - {Version{epoch: 0, version: "1", revision: "1"}, LESS, Version{epoch: 0, version: "2", revision: "1"}}, - {Version{epoch: 0, version: "a", revision: "0"}, LESS, Version{epoch: 0, version: "b", revision: "0"}}, - {Version{epoch: 0, version: "1", revision: "1"}, LESS, Version{epoch: 0, version: "1", revision: "2"}}, - {Version{epoch: 0, version: "0", revision: "0"}, EQUAL, Version{epoch: 0, version: "0", revision: "0"}}, - {Version{epoch: 0, version: "0", revision: "00"}, EQUAL, Version{epoch: 0, version: "00", revision: "0"}}, - {Version{epoch: 1, version: "2", revision: "3"}, EQUAL, Version{epoch: 1, version: "2", revision: "3"}}, - {Version{epoch: 0, version: "0", revision: "a"}, LESS, Version{epoch: 0, version: "0", revision: "b"}}, - {MinVersion, LESS, MaxVersion}, - {MinVersion, LESS, Version{}}, - {MinVersion, LESS, Version{version: "0"}}, - {MaxVersion, GREATER, Version{}}, - {MaxVersion, GREATER, Version{epoch: 9999999, version: "9999999", revision: "9999999"}}, - } - - for _, c := range cases { - cmp := c.v1.Compare(c.v2) - assert.Equal(t, c.expected, cmp, "%s vs. %s, = %d, expected %d", c.v1, c.v2, cmp, c.expected) - - cmp = c.v2.Compare(c.v1) - assert.Equal(t, -c.expected, cmp, "%s vs. %s, = %d, expected %d", c.v2, c.v1, cmp, -c.expected) - } -} - -func TestParse(t *testing.T) { - cases := []struct { - str string - ver Version - err bool - }{ - // Test 0 - {"0", Version{epoch: 0, version: "0", revision: ""}, false}, - {"0:0", Version{epoch: 0, version: "0", revision: ""}, false}, - {"0:0-", Version{epoch: 0, version: "0", revision: ""}, false}, - {"0:0-0", Version{epoch: 0, version: "0", revision: "0"}, false}, - {"0:0.0-0.0", Version{epoch: 0, version: "0.0", revision: "0.0"}, false}, - // Test epoched - {"1:0", Version{epoch: 1, version: "0", revision: ""}, false}, - {"5:1", Version{epoch: 5, version: "1", revision: ""}, false}, - // Test multiple hypens - {"0:0-0-0", Version{epoch: 0, version: "0-0", revision: "0"}, false}, - {"0:0-0-0-0", Version{epoch: 0, version: "0-0-0", revision: "0"}, false}, - // Test multiple colons - {"0:0:0-0", Version{epoch: 0, version: "0:0", revision: "0"}, false}, - {"0:0:0:0-0", Version{epoch: 0, version: "0:0:0", revision: "0"}, false}, - // Test multiple hyphens and colons - {"0:0:0-0-0", Version{epoch: 0, version: "0:0-0", revision: "0"}, false}, - {"0:0-0:0-0", Version{epoch: 0, version: "0-0:0", revision: "0"}, false}, - // Test valid characters in version - {"0:09azAZ.-+~:_-0", Version{epoch: 0, version: "09azAZ.-+~:_", revision: "0"}, false}, - // Test valid characters in debian revision - {"0:0-azAZ09.+~_", Version{epoch: 0, version: "0", revision: "azAZ09.+~_"}, false}, - // Test version with leading and trailing spaces - {" 0:0-1", Version{epoch: 0, version: "0", revision: "1"}, false}, - {"0:0-1 ", Version{epoch: 0, version: "0", revision: "1"}, false}, - {" 0:0-1 ", Version{epoch: 0, version: "0", revision: "1"}, false}, - // Test empty version - {"", Version{}, true}, - {" ", Version{}, true}, - {"0:", Version{}, true}, - // Test version with embedded spaces - {"0:0 0-1", Version{}, true}, - // Test version with negative epoch - {"-1:0-1", Version{}, true}, - // Test invalid characters in epoch - {"a:0-0", Version{}, true}, - {"A:0-0", Version{}, true}, - // Test version not starting with a digit - {"0:abc3-0", Version{}, true}, - } - for _, c := range cases { - v, err := NewVersion(c.str) - - if c.err { - assert.Error(t, err, "When parsing '%s'", c.str) - } else { - assert.Nil(t, err, "When parsing '%s'", c.str) - } - assert.Equal(t, c.ver, v, "When parsing '%s'", c.str) - } - - // Test invalid characters in version - versym := []rune{'!', '#', '@', '$', '%', '&', '/', '|', '\\', '<', '>', '(', ')', '[', ']', '{', '}', ';', ',', '=', '*', '^', '\''} - for _, r := range versym { - _, err := NewVersion(strings.Join([]string{"0:0", string(r), "-0"}, "")) - assert.Error(t, err, "Parsing with invalid character '%s' in version should have failed", string(r)) - } - - // Test invalid characters in revision - versym = []rune{'!', '#', '@', '$', '%', '&', '/', '|', '\\', '<', '>', '(', ')', '[', ']', '{', '}', ':', ';', ',', '=', '*', '^', '\''} - for _, r := range versym { - _, err := NewVersion(strings.Join([]string{"0:0-", string(r)}, "")) - assert.Error(t, err, "Parsing with invalid character '%s' in revision should have failed", string(r)) - } -} - -func TestParseAndCompare(t *testing.T) { - const LESS = -1 - const EQUAL = 0 - const GREATER = 1 - - cases := []struct { - v1 string - expected int - v2 string - }{ - {"7.6p2-4", GREATER, "7.6-0"}, - {"1.0.3-3", GREATER, "1.0-1"}, - {"1.3", GREATER, "1.2.2-2"}, - {"1.3", GREATER, "1.2.2"}, - // Some properties of text strings - {"0-pre", EQUAL, "0-pre"}, - {"0-pre", LESS, "0-pree"}, - {"1.1.6r2-2", GREATER, "1.1.6r-1"}, - {"2.6b2-1", GREATER, "2.6b-2"}, - {"98.1p5-1", LESS, "98.1-pre2-b6-2"}, - {"0.4a6-2", GREATER, "0.4-1"}, - {"1:3.0.5-2", LESS, "1:3.0.5.1"}, - // epochs - {"1:0.4", GREATER, "10.3"}, - {"1:1.25-4", LESS, "1:1.25-8"}, - {"0:1.18.36", EQUAL, "1.18.36"}, - {"1.18.36", GREATER, "1.18.35"}, - {"0:1.18.36", GREATER, "1.18.35"}, - // Funky, but allowed, characters in upstream version - {"9:1.18.36:5.4-20", LESS, "10:0.5.1-22"}, - {"9:1.18.36:5.4-20", LESS, "9:1.18.36:5.5-1"}, - {"9:1.18.36:5.4-20", LESS, " 9:1.18.37:4.3-22"}, - {"1.18.36-0.17.35-18", GREATER, "1.18.36-19"}, - // Junk - {"1:1.2.13-3", LESS, "1:1.2.13-3.1"}, - {"2.0.7pre1-4", LESS, "2.0.7r-1"}, - // if a version includes a dash, it should be the debrev dash - policy says so - {"0:0-0-0", GREATER, "0-0"}, - // do we like strange versions? Yes we like strange versions… - {"0", EQUAL, "0"}, - {"0", EQUAL, "00"}, - // #205960 - {"3.0~rc1-1", LESS, "3.0-1"}, - // #573592 - debian policy 5.6.12 - {"1.0", EQUAL, "1.0-0"}, - {"0.2", LESS, "1.0-0"}, - {"1.0", LESS, "1.0-0+b1"}, - {"1.0", GREATER, "1.0-0~"}, - // "steal" the testcases from (old perl) cupt - {"1.2.3", EQUAL, "1.2.3"}, // identical - {"4.4.3-2", EQUAL, "4.4.3-2"}, // identical - {"1:2ab:5", EQUAL, "1:2ab:5"}, // this is correct... - {"7:1-a:b-5", EQUAL, "7:1-a:b-5"}, // and this - {"57:1.2.3abYZ+~-4-5", EQUAL, "57:1.2.3abYZ+~-4-5"}, // and those too - {"1.2.3", EQUAL, "0:1.2.3"}, // zero epoch - {"1.2.3", EQUAL, "1.2.3-0"}, // zero revision - {"009", EQUAL, "9"}, // zeroes… - {"009ab5", EQUAL, "9ab5"}, // there as well - {"1.2.3", LESS, "1.2.3-1"}, // added non-zero revision - {"1.2.3", LESS, "1.2.4"}, // just bigger - {"1.2.4", GREATER, "1.2.3"}, // order doesn't matter - {"1.2.24", GREATER, "1.2.3"}, // bigger, eh? - {"0.10.0", GREATER, "0.8.7"}, // bigger, eh? - {"3.2", GREATER, "2.3"}, // major number rocks - {"1.3.2a", GREATER, "1.3.2"}, // letters rock - {"0.5.0~git", LESS, "0.5.0~git2"}, // numbers rock - {"2a", LESS, "21"}, // but not in all places - {"1.3.2a", LESS, "1.3.2b"}, // but there is another letter - {"1:1.2.3", GREATER, "1.2.4"}, // epoch rocks - {"1:1.2.3", LESS, "1:1.2.4"}, // bigger anyway - {"1.2a+~bCd3", LESS, "1.2a++"}, // tilde doesn't rock - {"1.2a+~bCd3", GREATER, "1.2a+~"}, // but first is longer! - {"5:2", GREATER, "304-2"}, // epoch rocks - {"5:2", LESS, "304:2"}, // so big epoch? - {"25:2", GREATER, "3:2"}, // 25 > 3, obviously - {"1:2:123", LESS, "1:12:3"}, // 12 > 2 - {"1.2-5", LESS, "1.2-3-5"}, // 1.2 < 1.2-3 - {"5.10.0", GREATER, "5.005"}, // preceding zeroes don't matters - {"3a9.8", LESS, "3.10.2"}, // letters are before all letter symbols - {"3a9.8", GREATER, "3~10"}, // but after the tilde - {"1.4+OOo3.0.0~", LESS, "1.4+OOo3.0.0-4"}, // another tilde check - {"2.4.7-1", LESS, "2.4.7-z"}, // revision comparing - {"1.002-1+b2", GREATER, "1.00"}, // whatever... - } - - for _, c := range cases { - v1, err1 := NewVersion(c.v1) - v2, err2 := NewVersion(c.v2) - if assert.Nil(t, err1) && assert.Nil(t, err2) { - cmp := v1.Compare(v2) - assert.Equal(t, c.expected, cmp, "%s vs. %s, = %d, expected %d", c.v1, c.v2, cmp, c.expected) - - cmp = v2.Compare(v1) - assert.Equal(t, -c.expected, cmp, "%s vs. %s, = %d, expected %d", c.v2, c.v1, cmp, -c.expected) - } - } -} - -func TestVersionJson(t *testing.T) { - v, _ := NewVersion("57:1.2.3abYZ+~-4-5") - - // Marshal - json, err := v.MarshalJSON() - assert.Nil(t, err) - assert.Equal(t, "\""+v.String()+"\"", string(json)) - - // Unmarshal - var v2 Version - v2.UnmarshalJSON(json) - assert.Equal(t, v, v2) -} From 78cef02fdad4afd12a2a00df51d457c796278bfa Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 02:08:52 -0500 Subject: [PATCH 05/23] pkg: cerrors -> commonerr --- api/v1/routes.go | 30 +++++++++++------------ database/pgsql/feature.go | 8 +++--- database/pgsql/keyvalue.go | 6 ++--- database/pgsql/layer.go | 12 ++++----- database/pgsql/layer_test.go | 12 ++++----- database/pgsql/lock.go | 6 ++--- database/pgsql/namespace.go | 6 ++--- database/pgsql/notification.go | 6 ++--- database/pgsql/notification_test.go | 10 ++++---- database/pgsql/pgsql.go | 10 ++++---- database/pgsql/vulnerability.go | 18 +++++++------- database/pgsql/vulnerability_test.go | 12 ++++----- ext/vulnmdsrc/nvd/nvd.go | 10 ++++---- ext/vulnsrc/alpine/alpine.go | 4 +-- ext/vulnsrc/debian/debian.go | 6 ++--- ext/vulnsrc/oracle/oracle.go | 8 +++--- ext/vulnsrc/rhel/rhel.go | 8 +++--- ext/vulnsrc/ubuntu/ubuntu.go | 12 ++++----- notifier/notifier.go | 4 +-- {utils/errors => pkg/commonerr}/errors.go | 7 +++--- utils/http/http.go | 8 +++--- worker/detectors/feature/rpm/rpm.go | 8 +++--- worker/worker.go | 22 ++++++++--------- worker/worker_test.go | 6 ++--- 24 files changed, 120 insertions(+), 119 deletions(-) rename {utils/errors => pkg/commonerr}/errors.go (91%) diff --git a/api/v1/routes.go b/api/v1/routes.go index 344e2e28..bde7560d 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -27,8 +27,8 @@ import ( "github.com/coreos/clair/api/context" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" ) @@ -118,7 +118,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return postLayerRoute, statusUnprocessableEntity } - if _, badreq := err.(*cerrors.ErrBadRequest); badreq { + if _, badreq := err.(*commonerr.ErrBadRequest); badreq { writeResponse(w, r, http.StatusBadRequest, LayerEnvelope{Error: &Error{err.Error()}}) return postLayerRoute, http.StatusBadRequest } @@ -143,7 +143,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * _, withVulnerabilities := r.URL.Query()["vulnerabilities"] dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}}) return getLayerRoute, http.StatusNotFound } else if err != nil { @@ -159,7 +159,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { err := ctx.Store.DeleteLayer(p.ByName("layerName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}}) return deleteLayerRoute, http.StatusNotFound } else if err != nil { @@ -223,7 +223,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par } dbVulns, nextPage, err := ctx.Store.ListVulnerabilities(namespace, limit, page) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return getVulnerabilityRoute, http.StatusNotFound } else if err != nil { @@ -273,7 +273,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true) if err != nil { switch err.(type) { - case *cerrors.ErrBadRequest: + case *commonerr.ErrBadRequest: writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return postVulnerabilityRoute, http.StatusBadRequest default: @@ -290,7 +290,7 @@ func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param _, withFixedIn := r.URL.Query()["fixedIn"] dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return getVulnerabilityRoute, http.StatusNotFound } else if err != nil { @@ -334,7 +334,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true) if err != nil { switch err.(type) { - case *cerrors.ErrBadRequest: + case *commonerr.ErrBadRequest: writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return putVulnerabilityRoute, http.StatusBadRequest default: @@ -349,7 +349,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return deleteVulnerabilityRoute, http.StatusNotFound } else if err != nil { @@ -363,7 +363,7 @@ func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Pa func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) return getFixesRoute, http.StatusNotFound } else if err != nil { @@ -403,11 +403,11 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co err = ctx.Store.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []database.FeatureVersion{dbFix}) if err != nil { switch err.(type) { - case *cerrors.ErrBadRequest: + case *commonerr.ErrBadRequest: writeResponse(w, r, http.StatusBadRequest, FeatureEnvelope{Error: &Error{err.Error()}}) return putFixRoute, http.StatusBadRequest default: - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) return putFixRoute, http.StatusNotFound } @@ -422,7 +422,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) return deleteFixRoute, http.StatusNotFound } else if err != nil { @@ -468,7 +468,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params } dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}}) return deleteNotificationRoute, http.StatusNotFound } else if err != nil { @@ -484,7 +484,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { err := ctx.Store.DeleteNotification(p.ByName("notificationName")) - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}}) return deleteNotificationRoute, http.StatusNotFound } else if err != nil { diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index a0be7612..c39bd5b7 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,12 +21,12 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" ) func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { if feature.Name == "" { - return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature") + return 0, commonerr.NewBadRequestError("could not find/insert invalid Feature") } // Do cache lookup. @@ -65,7 +65,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { func (pgSQL *pgSQL) insertFeatureVersion(fv database.FeatureVersion) (id int, err error) { err = versionfmt.Valid(fv.Feature.Namespace.VersionFormat, fv.Version) if err != nil { - return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion") + return 0, commonerr.NewBadRequestError("could not find/insert invalid FeatureVersion") } // Do cache lookup. diff --git a/database/pgsql/keyvalue.go b/database/pgsql/keyvalue.go index 264774c7..14730d51 100644 --- a/database/pgsql/keyvalue.go +++ b/database/pgsql/keyvalue.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,14 +18,14 @@ import ( "database/sql" "time" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" ) // InsertKeyValue stores (or updates) a single key / value tuple. func (pgSQL *pgSQL) InsertKeyValue(key, value string) (err error) { if key == "" || value == "" { log.Warning("could not insert a flag which has an empty name or value") - return cerrors.NewBadRequestError("could not insert a flag which has an empty name or value") + return commonerr.NewBadRequestError("could not insert a flag which has an empty name or value") } defer observeQueryTime("InsertKeyValue", "all", time.Now()) diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index aab7cfe0..5b50efd3 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,8 +22,8 @@ import ( "github.com/guregu/null/zero" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" ) func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) { @@ -247,12 +247,12 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { // Verify parameters if layer.Name == "" { log.Warning("could not insert a layer which has an empty Name") - return cerrors.NewBadRequestError("could not insert a layer which has an empty Name") + return commonerr.NewBadRequestError("could not insert a layer which has an empty Name") } // Get a potentially existing layer. existingLayer, err := pgSQL.FindLayer(layer.Name, true, false) - if err != nil && err != cerrors.ErrNotFound { + if err != nil && err != commonerr.ErrNotFound { return err } else if err == nil { if existingLayer.EngineVersion >= layer.EngineVersion { @@ -271,7 +271,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { if layer.Parent != nil { if layer.Parent.ID == 0 { log.Warning("Parent is expected to be retrieved from database when inserting a layer.") - return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.") + return commonerr.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.") } parentID = zero.IntFrom(int64(layer.Parent.ID)) @@ -429,7 +429,7 @@ func (pgSQL *pgSQL) DeleteLayer(name string) error { } if affected <= 0 { - return cerrors.ErrNotFound + return commonerr.ErrNotFound } return nil diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index dcd420b7..3aa6e622 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -363,19 +363,19 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { func testInsertLayerDelete(t *testing.T, datastore database.Datastore) { err := datastore.DeleteLayer("TestInsertLayerX") - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) err = datastore.DeleteLayer("TestInsertLayer3") assert.Nil(t, err) _, err = datastore.FindLayer("TestInsertLayer3", false, false) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) _, err = datastore.FindLayer("TestInsertLayer4a", false, false) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) _, err = datastore.FindLayer("TestInsertLayer4b", true, false) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) } func cmpFV(a, b database.FeatureVersion) bool { diff --git a/database/pgsql/lock.go b/database/pgsql/lock.go index 2f491caa..f677232e 100644 --- a/database/pgsql/lock.go +++ b/database/pgsql/lock.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ package pgsql import ( "time" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" ) // Lock tries to set a temporary lock in the database. @@ -80,7 +80,7 @@ func (pgSQL *pgSQL) Unlock(name, owner string) { func (pgSQL *pgSQL) FindLock(name string) (string, time.Time, error) { if name == "" { log.Warning("could not find an invalid lock") - return "", time.Time{}, cerrors.NewBadRequestError("could not find an invalid lock") + return "", time.Time{}, commonerr.NewBadRequestError("could not find an invalid lock") } defer observeQueryTime("FindLock", "all", time.Now()) diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 941db0d1..8d4b304b 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,12 +18,12 @@ import ( "time" "github.com/coreos/clair/database" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" ) func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { if namespace.Name == "" { - return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace") + return 0, commonerr.NewBadRequestError("could not find/insert invalid Namespace") } if pgSQL.cache != nil { diff --git a/database/pgsql/notification.go b/database/pgsql/notification.go index eec59c21..6d079753 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import ( "time" "github.com/coreos/clair/database" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/guregu/null/zero" "github.com/pborman/uuid" ) @@ -242,7 +242,7 @@ func (pgSQL *pgSQL) DeleteNotification(name string) error { } if affected <= 0 { - return cerrors.ErrNotFound + return commonerr.ErrNotFound } return nil diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 476b5452..5dab2c22 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -37,7 +37,7 @@ func TestNotification(t *testing.T) { // Try to get a notification when there is none. _, err = datastore.GetAvailableNotification(time.Second) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) // Create some data. f1 := database.Feature{ @@ -126,7 +126,7 @@ func TestNotification(t *testing.T) { // Verify the renotify behaviour. if assert.Nil(t, datastore.SetNotificationNotified(notification.Name)) { _, err := datastore.GetAvailableNotification(time.Second) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) time.Sleep(50 * time.Millisecond) notificationB, err := datastore.GetAvailableNotification(20 * time.Millisecond) @@ -164,7 +164,7 @@ func TestNotification(t *testing.T) { assert.Nil(t, datastore.DeleteNotification(notification.Name)) _, err = datastore.GetAvailableNotification(time.Millisecond) - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) } // Update a vulnerability and ensure that the old/new vulnerabilities are correct. diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index aec49b7e..f1d8d7d6 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,8 +34,8 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/database/pgsql/migrations" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" ) var ( @@ -196,12 +196,12 @@ func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) func parseConnectionString(source string) (dbName string, pgSourceURL string, err error) { if source == "" { - return "", "", cerrors.NewBadRequestError("pgsql: no database connection string specified") + return "", "", commonerr.NewBadRequestError("pgsql: no database connection string specified") } sourceURL, err := url.Parse(source) if err != nil { - return "", "", cerrors.NewBadRequestError("pgsql: database connection string is not a valid URL") + return "", "", commonerr.NewBadRequestError("pgsql: database connection string is not a valid URL") } dbName = strings.TrimPrefix(sourceURL.Path, "/") @@ -280,7 +280,7 @@ func handleError(desc string, err error) error { } if err == sql.ErrNoRows { - return cerrors.ErrNotFound + return commonerr.ErrNotFound } log.Errorf("%s: %v", desc, err) diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 201fa549..17461374 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" "github.com/guregu/null/zero" ) @@ -37,7 +37,7 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID if err != nil { return nil, -1, handleError("searchNamespace", err) } else if id == 0 { - return nil, -1, cerrors.ErrNotFound + return nil, -1, commonerr.ErrNotFound } // Query. @@ -130,7 +130,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql. } if vulnerability.ID == 0 { - return vulnerability, cerrors.ErrNotFound + return vulnerability, commonerr.ErrNotFound } // Query the FixedIn FeatureVersion now. @@ -195,12 +195,12 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on // Verify parameters if vulnerability.Name == "" || vulnerability.Namespace.Name == "" { - return cerrors.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace") + return commonerr.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace") } if !onlyFixedIn && !vulnerability.Severity.IsValid() { msg := fmt.Sprintf("could not insert a vulnerability that has an invalid Severity: %s", vulnerability.Severity) log.Warning(msg) - return cerrors.NewBadRequestError(msg) + return commonerr.NewBadRequestError(msg) } for i := 0; i < len(vulnerability.FixedIn); i++ { fifv := &vulnerability.FixedIn[i] @@ -212,7 +212,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on } else if fifv.Feature.Namespace.Name != vulnerability.Namespace.Name { msg := "could not insert an invalid vulnerability that contains FixedIn FeatureVersion that are not in the same namespace as the Vulnerability" log.Warning(msg) - return cerrors.NewBadRequestError(msg) + return commonerr.NewBadRequestError(msg) } } @@ -228,7 +228,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on // Find existing vulnerability and its Vulnerability_FixedIn_Features (for update). existingVulnerability, err := findVulnerability(tx, vulnerability.Namespace.Name, vulnerability.Name, true) - if err != nil && err != cerrors.ErrNotFound { + if err != nil && err != commonerr.ErrNotFound { tx.Rollback() return err } @@ -237,7 +237,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on // Because this call tries to update FixedIn FeatureVersion, import all other data from the // existing one. if existingVulnerability.ID == 0 { - return cerrors.ErrNotFound + return commonerr.ErrNotFound } fixedIn := vulnerability.FixedIn diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index e1d8d4c4..788856ba 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -37,7 +37,7 @@ func TestFindVulnerability(t *testing.T) { // Find a vulnerability that does not exist. _, err = datastore.FindVulnerability("", "") - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) // Find a normal vulnerability. v1 := database.Vulnerability{ @@ -93,15 +93,15 @@ func TestDeleteVulnerability(t *testing.T) { // Delete non-existing Vulnerability. err = datastore.DeleteVulnerability("TestDeleteVulnerabilityNamespace1", "CVE-OPENSSL-1-DEB7") - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) err = datastore.DeleteVulnerability("debian:7", "TestDeleteVulnerabilityVulnerability1") - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) // Delete Vulnerability. err = datastore.DeleteVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") if assert.Nil(t, err) { _, err := datastore.FindVulnerability("debian:7", "CVE-OPENSSL-1-DEB7") - assert.Equal(t, cerrors.ErrNotFound, err) + assert.Equal(t, commonerr.ErrNotFound, err) } } diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 31636a77..1b98f609 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -35,7 +35,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -79,7 +79,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error { if a.localPath == "" { // Create a temporary folder to store the NVD data and create hashes struct. if a.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { - return cerrors.ErrFilesystem + return commonerr.ErrFilesystem } a.dataFeedHashes = make(map[string]string) @@ -97,7 +97,7 @@ func (a *appender) BuildCache(datastore database.Datastore) error { var nvd nvd if err = xml.NewDecoder(dataFeedReader).Decode(&nvd); err != nil { log.Errorf("could not decode NVD data feed '%s': %s", dataFeedName, err) - return cerrors.ErrCouldNotParse + return commonerr.ErrCouldNotParse } // For each entry of this data feed: @@ -179,14 +179,14 @@ func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[strin r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName)) if err != nil { log.Errorf("could not download NVD data feed file '%s': %s", dataFeedName, err) - return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload + return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload } // Un-gzip it. gr, err := gzip.NewReader(r.Body) if err != nil { log.Errorf("could not read NVD data feed file '%s': %s", dataFeedName, err) - return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload + return dataFeedReaders, dataFeedHashes, commonerr.ErrCouldNotDownload } // Store it to a file at the same time if possible. diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index a2bbf7ec..20a59033 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -31,8 +31,8 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -172,7 +172,7 @@ func (u *updater) pullRepository() (commit string, err error) { if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { u.Clean() log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) - return "", cerrors.ErrCouldNotDownload + return "", commonerr.ErrCouldNotDownload } } else { // The repository exists and it needs to be refreshed via a pull. diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index 84d4398f..a0388e72 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -31,7 +31,7 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -69,7 +69,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo r, err := http.Get(url) if err != nil { log.Errorf("could not download Debian's update: %s", err) - return resp, cerrors.ErrCouldNotDownload + return resp, commonerr.ErrCouldNotDownload } // Get the SHA-1 of the latest update's JSON data @@ -110,7 +110,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp vulnsrc.U err = json.NewDecoder(teedJSONReader).Decode(&data) if err != nil { log.Errorf("could not unmarshal Debian's JSON: %s", err) - return resp, cerrors.ErrCouldNotParse + return resp, commonerr.ErrCouldNotParse } // Calculate the hash and skip updating if the hash has been seen before. diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index 32afeaa0..569752ad 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -29,7 +29,7 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" "github.com/coreos/pkg/capnslog" ) @@ -103,7 +103,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo r, err := http.Get(ovalURI) if err != nil { log.Errorf("could not download Oracle's update list: %s", err) - return resp, cerrors.ErrCouldNotDownload + return resp, commonerr.ErrCouldNotDownload } defer r.Body.Close() @@ -126,7 +126,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo r, err := http.Get(ovalURI + elsaFilePrefix + strconv.Itoa(elsa) + ".xml") if err != nil { log.Errorf("could not download Oracle's update file: %s", err) - return resp, cerrors.ErrCouldNotDownload + return resp, commonerr.ErrCouldNotDownload } // Parse the XML. @@ -160,7 +160,7 @@ func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err = xml.NewDecoder(ovalReader).Decode(&ov) if err != nil { log.Errorf("could not decode Oracle's XML: %s", err) - err = cerrors.ErrCouldNotParse + err = commonerr.ErrCouldNotParse return } diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index 6714dcd8..f1d88986 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -31,7 +31,7 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils/types" ) @@ -107,7 +107,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo r, err := http.Get(ovalURI) if err != nil { log.Errorf("could not download RHEL's update list: %s", err) - return resp, cerrors.ErrCouldNotDownload + return resp, commonerr.ErrCouldNotDownload } // Get the list of RHSAs that we have to process. @@ -129,7 +129,7 @@ func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateRespo r, err := http.Get(ovalURI + rhsaFilePrefix + strconv.Itoa(rhsa) + ".xml") if err != nil { log.Errorf("could not download RHEL's update file: %s", err) - return resp, cerrors.ErrCouldNotDownload + return resp, commonerr.ErrCouldNotDownload } // Parse the XML. @@ -163,7 +163,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err = xml.NewDecoder(ovalReader).Decode(&ov) if err != nil { log.Errorf("could not decode RHEL's XML: %s", err) - err = cerrors.ErrCouldNotParse + err = commonerr.ErrCouldNotParse return } diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index ea0035da..0a74db4a 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -33,8 +33,8 @@ import ( "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -176,7 +176,7 @@ func (u *updater) pullRepository() (err error) { // Branch repository. if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out) - return cerrors.ErrCouldNotDownload + return commonerr.ErrCouldNotDownload } return nil @@ -187,7 +187,7 @@ func (u *updater) pullRepository() (err error) { os.RemoveAll(u.repositoryLocalPath) log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out) - return cerrors.ErrCouldNotDownload + return commonerr.ErrCouldNotDownload } return nil @@ -197,12 +197,12 @@ func getRevisionNumber(pathToRepo string) (int, error) { out, err := utils.Exec(pathToRepo, "bzr", "revno") if err != nil { log.Errorf("could not get Ubuntu repository's revision number: %s. output: %s", err, out) - return 0, cerrors.ErrCouldNotDownload + return 0, commonerr.ErrCouldNotDownload } revno, err := strconv.Atoi(strings.TrimSpace(string(out))) if err != nil { log.Errorf("could not parse Ubuntu repository's revision number: %s. output: %s", err, out) - return 0, cerrors.ErrCouldNotDownload + return 0, commonerr.ErrCouldNotDownload } return revno, nil } @@ -255,7 +255,7 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat out, err := utils.Exec(repositoryLocalPath, "bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0") if err != nil { log.Errorf("could not get Ubuntu vulnerabilities repository logs: %s. output: %s", err, out) - return nil, cerrors.ErrCouldNotDownload + return nil, commonerr.ErrCouldNotDownload } scanner := bufio.NewScanner(bytes.NewReader(out)) diff --git a/notifier/notifier.go b/notifier/notifier.go index d1fa6b2f..09f8b368 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -28,8 +28,8 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/notification" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" ) const ( @@ -127,7 +127,7 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA notification, err := datastore.GetAvailableNotification(renotifyInterval) if err != nil { // There is no notification or an error occurred. - if err != cerrors.ErrNotFound { + if err != commonerr.ErrNotFound { log.Warningf("could not get notification to send: %s", err) } diff --git a/utils/errors/errors.go b/pkg/commonerr/errors.go similarity index 91% rename from utils/errors/errors.go rename to pkg/commonerr/errors.go index 65508f2b..1e690eea 100644 --- a/utils/errors/errors.go +++ b/pkg/commonerr/errors.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package errors defines error types that are used in several modules -package errors +// Package commonerr defines reusable error types common throughout the Clair +// codebase. +package commonerr import "errors" diff --git a/utils/http/http.go b/utils/http/http.go index b02430a4..46ac5a0e 100644 --- a/utils/http/http.go +++ b/utils/http/http.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ import ( "net/http" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" ) @@ -48,11 +48,11 @@ func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) { if httpStatus == 0 { httpStatus = http.StatusInternalServerError // Try to guess the http status code from the error type - if _, isBadRequestError := err.(*cerrors.ErrBadRequest); isBadRequestError { + if _, isBadRequestError := err.(*commonerr.ErrBadRequest); isBadRequestError { httpStatus = http.StatusBadRequest } else { switch err { - case cerrors.ErrNotFound: + case commonerr.ErrNotFound: httpStatus = http.StatusNotFound case database.ErrBackendException: httpStatus = http.StatusServiceUnavailable diff --git a/worker/detectors/feature/rpm/rpm.go b/worker/detectors/feature/rpm/rpm.go index ba45f610..3a158d83 100644 --- a/worker/detectors/feature/rpm/rpm.go +++ b/worker/detectors/feature/rpm/rpm.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,8 +25,8 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker/detectors" ) @@ -55,13 +55,13 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. defer os.RemoveAll(tmpDir) if err != nil { log.Errorf("could not create temporary folder for RPM detection: %s", err) - return []database.FeatureVersion{}, cerrors.ErrFilesystem + return []database.FeatureVersion{}, commonerr.ErrFilesystem } err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700) if err != nil { log.Errorf("could not create temporary file for RPM detection: %s", err) - return []database.FeatureVersion{}, cerrors.ErrFilesystem + return []database.FeatureVersion{}, commonerr.ErrFilesystem } // Query RPM diff --git a/worker/worker.go b/worker/worker.go index c3d2de3d..984def88 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker/detectors" ) @@ -41,11 +41,11 @@ var ( // ErrUnsupported is the error that should be raised when an OS or package // manager is not supported. - ErrUnsupported = cerrors.NewBadRequestError("worker: OS and/or package manager are not supported") + ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported") // ErrParentUnknown is the error that should be raised when a parent layer // has yet to be processed for the current layer. - ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first") + ErrParentUnknown = commonerr.NewBadRequestError("worker: parent layer is unknown, it must be processed first") ) // Process detects the Namespace of a layer, the features it adds/removes, and @@ -55,15 +55,15 @@ var ( func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error { // Verify parameters. if name == "" { - return cerrors.NewBadRequestError("could not process a layer which does not have a name") + return commonerr.NewBadRequestError("could not process a layer which does not have a name") } if path == "" { - return cerrors.NewBadRequestError("could not process a layer which does not have a path") + return commonerr.NewBadRequestError("could not process a layer which does not have a path") } if imageFormat == "" { - return cerrors.NewBadRequestError("could not process a layer which does not have a format") + return commonerr.NewBadRequestError("could not process a layer which does not have a format") } log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)", @@ -71,11 +71,11 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s // Check to see if the layer is already in the database. layer, err := datastore.FindLayer(name, false, false) - if err != nil && err != cerrors.ErrNotFound { + if err != nil && err != commonerr.ErrNotFound { return err } - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { // New layer case. layer = database.Layer{Name: name, EngineVersion: Version} @@ -83,10 +83,10 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s // We need to get it with its Features in order to diff them. if parentName != "" { parent, err := datastore.FindLayer(parentName, true, false) - if err != nil && err != cerrors.ErrNotFound { + if err != nil && err != commonerr.ErrNotFound { return err } - if err == cerrors.ErrNotFound { + if err == commonerr.ErrNotFound { log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name, parentName) return ErrParentUnknown diff --git a/worker/worker_test.go b/worker/worker_test.go index fb8c270f..cc69ff11 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" - cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/clair/pkg/commonerr" // Register the required detectors. _ "github.com/coreos/clair/worker/detectors/data/docker" @@ -57,7 +57,7 @@ func TestProcessWithDistUpgrade(t *testing.T) { if layer, exists := datastore.layers[name]; exists { return layer, nil } - return database.Layer{}, cerrors.ErrNotFound + return database.Layer{}, commonerr.ErrNotFound } // Create the list of FeatureVersions that should not been upgraded from one layer to another. From f9b319089d4d5b0dfd64c6f80fc4117657270b77 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 02:33:19 -0500 Subject: [PATCH 06/23] ext: lock all drivers --- ext/notification/driver.go | 38 +++++++++++++++++++++++++----- ext/versionfmt/driver.go | 10 ++++---- ext/vulnmdsrc/driver.go | 35 ++++++++++++++++++++++------ ext/vulnsrc/driver.go | 47 ++++++++++++++++++++++++++------------ notifier/notifier.go | 8 +++---- updater/updater.go | 14 ++++++------ 6 files changed, 109 insertions(+), 43 deletions(-) diff --git a/ext/notification/driver.go b/ext/notification/driver.go index 2f3b6429..acd4d4f1 100644 --- a/ext/notification/driver.go +++ b/ext/notification/driver.go @@ -21,6 +21,8 @@ package notification import ( + "sync" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/config" @@ -30,8 +32,8 @@ import ( var ( log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/notification") - // Senders is the list of registered Senders. - Senders = make(map[string]Sender) + sendersM sync.RWMutex + senders = make(map[string]Sender) ) // Sender represents anything that can transmit notifications. @@ -46,8 +48,8 @@ type Sender interface { // RegisterSender makes a Sender available by the provided name. // -// If RegisterSender is called twice with the same name, the name is blank, or -// if the provided Sender is nil, this function panics. +// If called twice with the same name, the name is blank, or if the provided +// Sender is nil, this function panics. func RegisterSender(name string, s Sender) { if name == "" { panic("notification: could not register a Sender with an empty name") @@ -57,9 +59,33 @@ func RegisterSender(name string, s Sender) { panic("notification: could not register a nil Sender") } - if _, dup := Senders[name]; dup { + sendersM.Lock() + defer sendersM.Unlock() + + if _, dup := senders[name]; dup { panic("notification: RegisterSender called twice for " + name) } - Senders[name] = s + senders[name] = s +} + +// Senders returns the list of the registered Senders. +func Senders() map[string]Sender { + sendersM.RLock() + defer sendersM.RUnlock() + + ret := make(map[string]Sender) + for k, v := range senders { + ret[k] = v + } + + return ret +} + +// UnregisterSender removes a Sender with a particular name from the list. +func UnregisterSender(name string) { + sendersM.Lock() + defer sendersM.Unlock() + + delete(senders, name) } diff --git a/ext/versionfmt/driver.go b/ext/versionfmt/driver.go index 84073a95..3d38469f 100644 --- a/ext/versionfmt/driver.go +++ b/ext/versionfmt/driver.go @@ -63,18 +63,20 @@ type Parser interface { // if the provided Parser is nil, this function panics. func RegisterParser(name string, p Parser) { if name == "" { - panic("Could not register a Parser with an empty name") + panic("versionfmt: could not register a Parser with an empty name") } + if p == nil { - panic("Could not register a nil Parser") + panic("versionfmt: could not register a nil Parser") } parsersM.Lock() defer parsersM.Unlock() - if _, alreadyExists := parsers[name]; alreadyExists { - panic("Parser '" + name + "' is already registered") + if _, dup := parsers[name]; dup { + panic("versionfmt: RegisterParser called twice for " + name) } + parsers[name] = p } diff --git a/ext/vulnmdsrc/driver.go b/ext/vulnmdsrc/driver.go index 658125de..06d5cb78 100644 --- a/ext/vulnmdsrc/driver.go +++ b/ext/vulnmdsrc/driver.go @@ -17,12 +17,16 @@ package vulnmdsrc import ( + "sync" + "github.com/coreos/clair/database" "github.com/coreos/clair/utils/types" ) -// Appenders is the list of registered Appenders. -var Appenders = make(map[string]Appender) +var ( + appendersM sync.RWMutex + appenders = make(map[string]Appender) +) // AppendFunc is the type of a callback provided to an Appender. type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority) @@ -49,20 +53,37 @@ type Appender interface { } // RegisterAppender makes an Appender available by the provided name. -// If Register is called twice with the same name or if driver is nil, -// it panics. +// +// If called twice with the same name, the name is blank, or if the provided +// Appender is nil, this function panics. func RegisterAppender(name string, a Appender) { if name == "" { - panic("updater: could not register an Appender with an empty name") + panic("vulnmdsrc: could not register an Appender with an empty name") } if a == nil { panic("vulnmdsrc: could not register a nil Appender") } - if _, dup := Appenders[name]; dup { + appendersM.Lock() + defer appendersM.Unlock() + + if _, dup := appenders[name]; dup { panic("vulnmdsrc: RegisterAppender called twice for " + name) } - Appenders[name] = a + appenders[name] = a +} + +// Appenders returns the list of the registered Appenders. +func Appenders() map[string]Appender { + appendersM.RLock() + defer appendersM.RUnlock() + + ret := make(map[string]Appender) + for k, v := range appenders { + ret[k] = v + } + + return ret } diff --git a/ext/vulnsrc/driver.go b/ext/vulnsrc/driver.go index 132eabee..fd442416 100644 --- a/ext/vulnsrc/driver.go +++ b/ext/vulnsrc/driver.go @@ -18,21 +18,30 @@ package vulnsrc import ( "errors" + "sync" "github.com/coreos/clair/database" ) var ( - // Updaters is the list of registered Updaters. - Updaters = make(map[string]Updater) - // ErrFilesystem is returned when a fetcher fails to interact with the local filesystem. ErrFilesystem = errors.New("vulnsrc: something went wrong when interacting with the fs") // ErrGitFailure is returned when a fetcher fails to interact with git. ErrGitFailure = errors.New("vulnsrc: something went wrong when interacting with git") + + updatersM sync.RWMutex + updaters = make(map[string]Updater) ) +// UpdateResponse represents the sum of results of an update. +type UpdateResponse struct { + FlagName string + FlagValue string + Notes []string + Vulnerabilities []database.Vulnerability +} + // Updater represents anything that can fetch vulnerabilities and insert them // into a Clair datastore. type Updater interface { @@ -44,18 +53,10 @@ type Updater interface { Clean() } -// UpdateResponse represents the sum of results of an update. -type UpdateResponse struct { - FlagName string - FlagValue string - Notes []string - Vulnerabilities []database.Vulnerability -} - // RegisterUpdater makes an Updater available by the provided name. // -// If RegisterUpdater is called twice with the same name, the name is blank, or -// if the provided Updater is nil, this function panics. +// If called twice with the same name, the name is blank, or if the provided +// Updater is nil, this function panics. func RegisterUpdater(name string, u Updater) { if name == "" { panic("vulnsrc: could not register an Updater with an empty name") @@ -65,9 +66,25 @@ func RegisterUpdater(name string, u Updater) { panic("vulnsrc: could not register a nil Updater") } - if _, dup := Updaters[name]; dup { + updatersM.Lock() + defer updatersM.Unlock() + + if _, dup := updaters[name]; dup { panic("vulnsrc: RegisterUpdater called twice for " + name) } - Updaters[name] = u + updaters[name] = u +} + +// Updaters returns the list of the registered Updaters. +func Updaters() map[string]Updater { + updatersM.RLock() + defer updatersM.RUnlock() + + ret := make(map[string]Updater) + for k, v := range updaters { + ret[k] = v + } + + return ret } diff --git a/notifier/notifier.go b/notifier/notifier.go index 09f8b368..fa80482d 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -63,11 +63,11 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u defer stopper.End() // Configure registered notifiers. - for senderName, sender := range notification.Senders { + for senderName, sender := range notification.Senders() { if configured, err := sender.Configure(config); configured { log.Infof("sender '%s' configured\n", senderName) } else { - delete(notification.Senders, senderName) + notification.UnregisterSender(senderName) if err != nil { log.Errorf("could not configure notifier '%s': %s", senderName, err) } @@ -75,7 +75,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u } // Do not run the updater if there is no notifier enabled. - if len(notification.Senders) == 0 { + if len(notification.Senders()) == 0 { log.Infof("notifier service is disabled") return } @@ -149,7 +149,7 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { // Send notification. - for senderName, sender := range notification.Senders { + for senderName, sender := range notification.Senders() { var attempts int var backOff time.Duration for { diff --git a/updater/updater.go b/updater/updater.go index 01feb0c2..a7b4e558 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -152,10 +152,10 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S } // Clean resources. - for _, appenders := range vulnmdsrc.Appenders { + for _, appenders := range vulnmdsrc.Appenders() { appenders.Clean() } - for _, updaters := range vulnsrc.Updaters { + for _, updaters := range vulnsrc.Updaters() { updaters.Clean() } @@ -215,7 +215,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Fetch updates in parallel. log.Info("fetching vulnerability updates") var responseC = make(chan *vulnsrc.UpdateResponse, 0) - for n, u := range vulnsrc.Updaters { + for n, u := range vulnsrc.Updaters() { go func(name string, u vulnsrc.Updater) { response, err := u.Update(datastore) if err != nil { @@ -231,7 +231,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st } // Collect results of updates. - for i := 0; i < len(vulnsrc.Updaters); i++ { + for i := 0; i < len(vulnsrc.Updaters()); i++ { resp := <-responseC if resp != nil { vulnerabilities = append(vulnerabilities, doVulnerabilitiesNamespacing(resp.Vulnerabilities)...) @@ -248,7 +248,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st // Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel. func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability { - if len(vulnmdsrc.Appenders) == 0 { + if len(vulnmdsrc.Appenders()) == 0 { return vulnerabilities } @@ -264,9 +264,9 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner } var wg sync.WaitGroup - wg.Add(len(vulnmdsrc.Appenders)) + wg.Add(len(vulnmdsrc.Appenders())) - for n, a := range vulnmdsrc.Appenders { + for n, a := range vulnmdsrc.Appenders() { go func(name string, appender vulnmdsrc.Appender) { defer wg.Done() From d9be34c3c4cf7a8d1865abd064c59dd3f24f51bd Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 03:07:35 -0500 Subject: [PATCH 07/23] ext: data detector -> imagefmt --- cmd/clair/main.go | 7 +- ext/imagefmt/aci/aci.go | 40 +++++++ ext/imagefmt/docker/docker.go | 32 ++++++ ext/imagefmt/driver.go | 150 +++++++++++++++++++++++++ worker/detectors/data.go | 122 -------------------- worker/detectors/data/aci/aci.go | 41 ------- worker/detectors/data/docker/docker.go | 41 ------- worker/worker.go | 10 +- 8 files changed, 229 insertions(+), 214 deletions(-) create mode 100644 ext/imagefmt/aci/aci.go create mode 100644 ext/imagefmt/docker/docker.go create mode 100644 ext/imagefmt/driver.go delete mode 100644 worker/detectors/data.go delete mode 100644 worker/detectors/data/aci/aci.go delete mode 100644 worker/detectors/data/docker/docker.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 8d3e35f5..9ac8a2e4 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,6 +26,8 @@ import ( "github.com/coreos/clair/config" // Register extensions. + _ "github.com/coreos/clair/ext/imagefmt/aci" + _ "github.com/coreos/clair/ext/imagefmt/docker" _ "github.com/coreos/clair/ext/notification/webhook" _ "github.com/coreos/clair/ext/vulnmdsrc/nvd" _ "github.com/coreos/clair/ext/vulnsrc/alpine" @@ -34,9 +36,6 @@ import ( _ "github.com/coreos/clair/ext/vulnsrc/rhel" _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" - _ "github.com/coreos/clair/worker/detectors/data/aci" - _ "github.com/coreos/clair/worker/detectors/data/docker" - _ "github.com/coreos/clair/worker/detectors/feature/apk" _ "github.com/coreos/clair/worker/detectors/feature/dpkg" _ "github.com/coreos/clair/worker/detectors/feature/rpm" diff --git a/ext/imagefmt/aci/aci.go b/ext/imagefmt/aci/aci.go new file mode 100644 index 00000000..38e26217 --- /dev/null +++ b/ext/imagefmt/aci/aci.go @@ -0,0 +1,40 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "io" + "path/filepath" + + "github.com/coreos/clair/ext/imagefmt" + "github.com/coreos/clair/pkg/tarutil" +) + +type format struct{} + +func init() { + imagefmt.RegisterExtractor("aci", &format{}) +} + +func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) { + // All contents are inside a "rootfs" directory, so this needs to be + // prepended to each filename. + var filenames []string + for _, filename := range toExtract { + filenames = append(filenames, filepath.Join("rootfs/", filename)) + } + + return tarutil.ExtractFiles(layerReader, filenames) +} diff --git a/ext/imagefmt/docker/docker.go b/ext/imagefmt/docker/docker.go new file mode 100644 index 00000000..94f866fd --- /dev/null +++ b/ext/imagefmt/docker/docker.go @@ -0,0 +1,32 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "io" + + "github.com/coreos/clair/ext/imagefmt" + "github.com/coreos/clair/pkg/tarutil" +) + +type format struct{} + +func init() { + imagefmt.RegisterExtractor("docker", &format{}) +} + +func (f format) ExtractFiles(layerReader io.ReadCloser, toExtract []string) (tarutil.FilesMap, error) { + return tarutil.ExtractFiles(layerReader, toExtract) +} diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go new file mode 100644 index 00000000..81d8ce3f --- /dev/null +++ b/ext/imagefmt/driver.go @@ -0,0 +1,150 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notification fetches notifications from the database and informs the +// specified remote handler about their existences, inviting the third party to +// actively query the API about it. + +// Package imagefmt exposes functions to dynamically register methods to +// detect different types of container image formats. +package imagefmt + +import ( + "fmt" + "io" + "math" + "net/http" + "os" + "strings" + "sync" + + "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/tarutil" + "github.com/coreos/pkg/capnslog" +) + +var ( + // ErrCouldNotFindLayer is returned when we could not download or open the layer file. + ErrCouldNotFindLayer = commonerr.NewBadRequestError("could not find layer") + + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/imagefmt") + + extractorsM sync.RWMutex + extractors = make(map[string]Extractor) +) + +// Extractor represents an ability to extract files from a particular container +// image format. +type Extractor interface { + // ExtractFiles produces a tarutil.FilesMap from a image layer. + ExtractFiles(layer io.ReadCloser, filenames []string) (tarutil.FilesMap, error) +} + +// RegisterExtractor makes a extractor available by the provided name. +// +// If called twice with the same name, the name is blank, or if the provided +// Extractor is nil, this function panics. +func RegisterExtractor(name string, d Extractor) { + extractorsM.Lock() + defer extractorsM.Unlock() + + if name == "" { + panic("imagefmt: could not register a extractor with an empty name") + } + + if d == nil { + panic("imagefmt: could not register a nil extractor") + } + + // Enforce lowercase names, so that they can be reliably be found in a map. + name = strings.ToLower(name) + + if _, dup := extractors[name]; dup { + panic("imagefmt: RegisterExtractor called twice for " + name) + } + + extractors[name] = d +} + +// Extractors returns the list of the registered extractors. +func Extractors() map[string]Extractor { + extractorsM.RLock() + defer extractorsM.RUnlock() + + ret := make(map[string]Extractor) + for k, v := range extractors { + ret[k] = v + } + + return ret +} + +// UnregisterExtractor removes a Extractor with a particular name from the list. +func UnregisterExtractor(name string) { + extractorsM.Lock() + defer extractorsM.Unlock() + delete(extractors, name) +} + +// Extract streams an image layer from disk or over HTTP, determines the +// image format, then extracts the files specified. +func Extract(format, path string, headers map[string]string, toExtract []string) (tarutil.FilesMap, error) { + var layerReader io.ReadCloser + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + // Create a new HTTP request object. + request, err := http.NewRequest("GET", path, nil) + if err != nil { + return nil, ErrCouldNotFindLayer + } + + // Set any provided HTTP Headers. + if headers != nil { + for k, v := range headers { + request.Header.Set(k, v) + } + } + + // Send the request and handle the response. + r, err := http.DefaultClient.Do(request) + if err != nil { + log.Warningf("could not download layer: %s", err) + return nil, ErrCouldNotFindLayer + } + + // Fail if we don't receive a 2xx HTTP status code. + if math.Floor(float64(r.StatusCode/100)) != 2 { + log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode) + return nil, ErrCouldNotFindLayer + } + + layerReader = r.Body + } else { + var err error + layerReader, err = os.Open(path) + if err != nil { + return nil, ErrCouldNotFindLayer + } + } + defer layerReader.Close() + + if extractor, exists := Extractors()[strings.ToLower(format)]; exists { + files, err := extractor.ExtractFiles(layerReader, toExtract) + if err != nil { + return nil, err + } + return files, nil + } + + return nil, commonerr.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format)) +} diff --git a/worker/detectors/data.go b/worker/detectors/data.go deleted file mode 100644 index 376a2548..00000000 --- a/worker/detectors/data.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package detectors exposes functions to register and use container -// information extractors. -package detectors - -import ( - "fmt" - "io" - "math" - "net/http" - "os" - "strings" - "sync" - - cerrors "github.com/coreos/clair/utils/errors" - "github.com/coreos/pkg/capnslog" -) - -// The DataDetector interface defines a way to detect the required data from input path -type DataDetector interface { - //Support check if the input path and format are supported by the underling detector - Supported(path string, format string) bool - // Detect detects the required data from input path - Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) -} - -var ( - dataDetectorsLock sync.Mutex - dataDetectors = make(map[string]DataDetector) - - log = capnslog.NewPackageLogger("github.com/coreos/clair", "detectors") - - // ErrCouldNotFindLayer is returned when we could not download or open the layer file. - ErrCouldNotFindLayer = cerrors.NewBadRequestError("could not find layer") -) - -// RegisterDataDetector provides a way to dynamically register an implementation of a -// DataDetector. -// -// If RegisterDataDetector is called twice with the same name if DataDetector is nil, -// or if the name is blank, it panics. -func RegisterDataDetector(name string, f DataDetector) { - if name == "" { - panic("Could not register a DataDetector with an empty name") - } - if f == nil { - panic("Could not register a nil DataDetector") - } - - dataDetectorsLock.Lock() - defer dataDetectorsLock.Unlock() - - if _, alreadyExists := dataDetectors[name]; alreadyExists { - panic(fmt.Sprintf("Detector '%s' is already registered", name)) - } - dataDetectors[name] = f -} - -// DetectData finds the Data of the layer by using every registered DataDetector -func DetectData(format, path string, headers map[string]string, toExtract []string, maxFileSize int64) (data map[string][]byte, err error) { - var layerReader io.ReadCloser - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { - // Create a new HTTP request object. - request, err := http.NewRequest("GET", path, nil) - if err != nil { - return nil, ErrCouldNotFindLayer - } - - // Set any provided HTTP Headers. - if headers != nil { - for k, v := range headers { - request.Header.Set(k, v) - } - } - - // Send the request and handle the response. - r, err := http.DefaultClient.Do(request) - if err != nil { - log.Warningf("could not download layer: %s", err) - return nil, ErrCouldNotFindLayer - } - - // Fail if we don't receive a 2xx HTTP status code. - if math.Floor(float64(r.StatusCode/100)) != 2 { - log.Warningf("could not download layer: got status code %d, expected 2XX", r.StatusCode) - return nil, ErrCouldNotFindLayer - } - - layerReader = r.Body - } else { - layerReader, err = os.Open(path) - if err != nil { - return nil, ErrCouldNotFindLayer - } - } - defer layerReader.Close() - - for _, detector := range dataDetectors { - if detector.Supported(path, format) { - data, err = detector.Detect(layerReader, toExtract, maxFileSize) - if err != nil { - return nil, err - } - return data, nil - } - } - - return nil, cerrors.NewBadRequestError(fmt.Sprintf("unsupported image format '%s'", format)) -} diff --git a/worker/detectors/data/aci/aci.go b/worker/detectors/data/aci/aci.go deleted file mode 100644 index 78551aa7..00000000 --- a/worker/detectors/data/aci/aci.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package aci - -import ( - "io" - "strings" - - "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" -) - -// ACIDataDetector implements DataDetector and detects layer data in 'aci' format -type ACIDataDetector struct{} - -func init() { - detectors.RegisterDataDetector("aci", &ACIDataDetector{}) -} - -func (detector *ACIDataDetector) Supported(path string, format string) bool { - if strings.EqualFold(format, "ACI") { - return true - } - return false -} - -func (detector *ACIDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "rootfs/", toExtract, maxFileSize) -} diff --git a/worker/detectors/data/docker/docker.go b/worker/detectors/data/docker/docker.go deleted file mode 100644 index d70de3bc..00000000 --- a/worker/detectors/data/docker/docker.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package docker - -import ( - "io" - "strings" - - "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" -) - -// DockerDataDetector implements DataDetector and detects layer data in 'Docker' format -type DockerDataDetector struct{} - -func init() { - detectors.RegisterDataDetector("Docker", &DockerDataDetector{}) -} - -func (detector *DockerDataDetector) Supported(path string, format string) bool { - if strings.EqualFold(format, "Docker") { - return true - } - return false -} - -func (detector *DockerDataDetector) Detect(layerReader io.ReadCloser, toExtract []string, maxFileSize int64) (map[string][]byte, error) { - return utils.SelectivelyExtractArchive(layerReader, "", toExtract, maxFileSize) -} diff --git a/worker/worker.go b/worker/worker.go index 984def88..904155ee 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -20,6 +20,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" "github.com/coreos/clair/worker/detectors" @@ -29,11 +30,6 @@ const ( // Version (integer) represents the worker version. // Increased each time the engine changes. Version = 3 - - // maxFileSize enforces a maximum size of a single file within a tarball that - // will be extracted. This protects against malicious layers that may contain - // extremely large package database files. - maxFileSize = 200 * 1024 * 1024 // 200 MiB ) var ( @@ -116,12 +112,14 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s // detectContent downloads a layer's archive and extracts its Namespace and Features. func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { - data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) + files, err := imagefmt.Extract(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...)) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) return } + data := map[string][]byte(files) + // Detect namespace. namespace = detectNamespace(name, data, parent) From fb193e1fdecad209e17a890e6727d04931015e0b Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 16:48:12 -0500 Subject: [PATCH 08/23] ext: namespace detector -> featurens --- cmd/clair/main.go | 11 +- .../featurens}/alpinerelease/alpinerelease.go | 21 +-- .../alpinerelease/alpinerelease_test.go | 21 +-- .../featurens}/aptsources/aptsources.go | 31 ++--- .../featurens}/aptsources/aptsources_test.go | 13 +- ext/featurens/driver.go | 124 ++++++++++++++++++ .../featurens}/lsbrelease/lsbrelease.go | 33 ++--- .../featurens}/lsbrelease/lsbrelease_test.go | 15 ++- .../featurens}/osrelease/osrelease.go | 53 ++++---- .../featurens}/osrelease/osrelease_test.go | 17 +-- .../featurens}/redhatrelease/redhatrelease.go | 52 ++++---- .../redhatrelease/redhatrelease_test.go | 19 +-- worker/detectors/namespace.go | 86 ------------ worker/detectors/namespace/test.go | 45 ------- worker/worker.go | 27 ++-- 15 files changed, 283 insertions(+), 285 deletions(-) rename {worker/detectors/namespace => ext/featurens}/alpinerelease/alpinerelease.go (73%) rename {worker/detectors/namespace => ext/featurens}/alpinerelease/alpinerelease_test.go (63%) rename {worker/detectors/namespace => ext/featurens}/aptsources/aptsources.go (71%) rename {worker/detectors/namespace => ext/featurens}/aptsources/aptsources_test.go (79%) create mode 100644 ext/featurens/driver.go rename {worker/detectors/namespace => ext/featurens}/lsbrelease/lsbrelease.go (73%) rename {worker/detectors/namespace => ext/featurens}/lsbrelease/lsbrelease_test.go (79%) rename {worker/detectors/namespace => ext/featurens}/osrelease/osrelease.go (55%) rename {worker/detectors/namespace => ext/featurens}/osrelease/osrelease_test.go (85%) rename {worker/detectors/namespace => ext/featurens}/redhatrelease/redhatrelease.go (58%) rename {worker/detectors/namespace => ext/featurens}/redhatrelease/redhatrelease_test.go (77%) delete mode 100644 worker/detectors/namespace.go delete mode 100644 worker/detectors/namespace/test.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 9ac8a2e4..2688880e 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -26,6 +26,11 @@ import ( "github.com/coreos/clair/config" // Register extensions. + _ "github.com/coreos/clair/ext/featurens/alpinerelease" + _ "github.com/coreos/clair/ext/featurens/aptsources" + _ "github.com/coreos/clair/ext/featurens/lsbrelease" + _ "github.com/coreos/clair/ext/featurens/osrelease" + _ "github.com/coreos/clair/ext/featurens/redhatrelease" _ "github.com/coreos/clair/ext/imagefmt/aci" _ "github.com/coreos/clair/ext/imagefmt/docker" _ "github.com/coreos/clair/ext/notification/webhook" @@ -40,12 +45,6 @@ import ( _ "github.com/coreos/clair/worker/detectors/feature/dpkg" _ "github.com/coreos/clair/worker/detectors/feature/rpm" - _ "github.com/coreos/clair/worker/detectors/namespace/alpinerelease" - _ "github.com/coreos/clair/worker/detectors/namespace/aptsources" - _ "github.com/coreos/clair/worker/detectors/namespace/lsbrelease" - _ "github.com/coreos/clair/worker/detectors/namespace/osrelease" - _ "github.com/coreos/clair/worker/detectors/namespace/redhatrelease" - _ "github.com/coreos/clair/database/pgsql" ) diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease.go b/ext/featurens/alpinerelease/alpinerelease.go similarity index 73% rename from worker/detectors/namespace/alpinerelease/alpinerelease.go rename to ext/featurens/alpinerelease/alpinerelease.go index 0730ad50..fafe5c9f 100644 --- a/worker/detectors/namespace/alpinerelease/alpinerelease.go +++ b/ext/featurens/alpinerelease/alpinerelease.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package alpinerelease implements a featurens.Detector for Alpine Linux based +// container image layers. package alpinerelease import ( @@ -21,8 +23,9 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) const ( @@ -33,15 +36,13 @@ const ( var versionRegexp = regexp.MustCompile(`^(\d)+\.(\d)+\.(\d)+$`) func init() { - detectors.RegisterNamespaceDetector("alpine-release", &detector{}) + featurens.RegisterDetector("alpine-release", &detector{}) } -// detector implements NamespaceDetector by reading the current version of -// Alpine Linux from /etc/alpine-release. type detector struct{} -func (d *detector) Detect(data map[string][]byte) *database.Namespace { - file, exists := data[alpineReleasePath] +func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { + file, exists := files[alpineReleasePath] if exists { scanner := bufio.NewScanner(bytes.NewBuffer(file)) for scanner.Scan() { @@ -52,14 +53,14 @@ func (d *detector) Detect(data map[string][]byte) *database.Namespace { return &database.Namespace{ Name: osName + ":" + "v" + versionNumbers[0] + "." + versionNumbers[1], VersionFormat: dpkg.ParserName, - } + }, nil } } } - return nil + return nil, nil } -func (d *detector) GetRequiredFiles() []string { +func (d detector) RequiredFilenames() []string { return []string{alpineReleasePath} } diff --git a/worker/detectors/namespace/alpinerelease/alpinerelease_test.go b/ext/featurens/alpinerelease/alpinerelease_test.go similarity index 63% rename from worker/detectors/namespace/alpinerelease/alpinerelease_test.go rename to ext/featurens/alpinerelease/alpinerelease_test.go index 3052ce03..110130a0 100644 --- a/worker/detectors/namespace/alpinerelease/alpinerelease_test.go +++ b/ext/featurens/alpinerelease/alpinerelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,34 +18,35 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/namespace" + "github.com/coreos/clair/ext/featurens" + "github.com/coreos/clair/pkg/tarutil" ) -func TestAlpineReleaseNamespaceDetection(t *testing.T) { - testData := []namespace.TestData{ +func TestDetector(t *testing.T) { + testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "alpine:v3.3"}, - Data: map[string][]byte{"etc/alpine-release": []byte(`3.3.4`)}, + Files: tarutil.FilesMap{"etc/alpine-release": []byte(`3.3.4`)}, }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v3.4"}, - Data: map[string][]byte{"etc/alpine-release": []byte(`3.4.0`)}, + Files: tarutil.FilesMap{"etc/alpine-release": []byte(`3.4.0`)}, }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3"}, - Data: map[string][]byte{"etc/alpine-release": []byte(`0.3.4`)}, + Files: tarutil.FilesMap{"etc/alpine-release": []byte(`0.3.4`)}, }, { ExpectedNamespace: &database.Namespace{Name: "alpine:v0.3"}, - Data: map[string][]byte{"etc/alpine-release": []byte(` + Files: tarutil.FilesMap{"etc/alpine-release": []byte(` 0.3.4 `)}, }, { ExpectedNamespace: nil, - Data: map[string][]byte{}, + Files: tarutil.FilesMap{}, }, } - namespace.TestDetector(t, &detector{}, testData) + featurens.TestDetector(t, &detector{}, testData) } diff --git a/worker/detectors/namespace/aptsources/aptsources.go b/ext/featurens/aptsources/aptsources.go similarity index 71% rename from worker/detectors/namespace/aptsources/aptsources.go rename to ext/featurens/aptsources/aptsources.go index 3c201c4c..3330ec69 100644 --- a/worker/detectors/namespace/aptsources/aptsources.go +++ b/ext/featurens/aptsources/aptsources.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package aptsources implements a featurens.Detector for apt based container +// image layers. +// +// This detector is necessary to determine the precise Debian version when it +// is an unstable version for instance. package aptsources import ( @@ -19,25 +24,21 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) -// AptSourcesNamespaceDetector implements NamespaceDetector and detects the Namespace from the -// /etc/apt/sources.list file. -// -// This detector is necessary to determine precise Debian version when it is -// an unstable version for instance. -type AptSourcesNamespaceDetector struct{} +type detector struct{} func init() { - detectors.RegisterNamespaceDetector("apt-sources", &AptSourcesNamespaceDetector{}) + featurens.RegisterDetector("apt-sources", &detector{}) } -func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { - f, hasFile := data["etc/apt/sources.list"] +func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { + f, hasFile := files["etc/apt/sources.list"] if !hasFile { - return nil + return nil, nil } var OS, version string @@ -79,11 +80,11 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat return &database.Namespace{ Name: OS + ":" + version, VersionFormat: dpkg.ParserName, - } + }, nil } - return nil + return nil, nil } -func (detector *AptSourcesNamespaceDetector) GetRequiredFiles() []string { +func (d detector) RequiredFilenames() []string { return []string{"etc/apt/sources.list"} } diff --git a/worker/detectors/namespace/aptsources/aptsources_test.go b/ext/featurens/aptsources/aptsources_test.go similarity index 79% rename from worker/detectors/namespace/aptsources/aptsources_test.go rename to ext/featurens/aptsources/aptsources_test.go index 238d1a9b..cd26e095 100644 --- a/worker/detectors/namespace/aptsources/aptsources_test.go +++ b/ext/featurens/aptsources/aptsources_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/namespace" + "github.com/coreos/clair/ext/featurens" + "github.com/coreos/clair/pkg/tarutil" ) -func TestAptSourcesNamespaceDetector(t *testing.T) { - testData := []namespace.TestData{ +func TestDetector(t *testing.T) { + testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "debian:unstable"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux stretch/sid" NAME="Debian GNU/Linux" @@ -38,5 +39,5 @@ BUG_REPORT_URL="https://bugs.debian.org/"`), }, } - namespace.TestDetector(t, &AptSourcesNamespaceDetector{}, testData) + featurens.TestDetector(t, &detector{}, testData) } diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go new file mode 100644 index 00000000..e1187a59 --- /dev/null +++ b/ext/featurens/driver.go @@ -0,0 +1,124 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package featurens exposes functions to dynamically register methods for +// determining a namespace for features present in an image layer. +package featurens + +import ( + "sync" + "testing" + + "github.com/coreos/pkg/capnslog" + "github.com/stretchr/testify/assert" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/tarutil" +) + +var ( + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurens") + + detectorsM sync.RWMutex + detectors = make(map[string]Detector) +) + +// Detector represents an ability to detect a namespace used for organizing +// features present in an image layer. +type Detector interface { + // Detect attempts to determine a Namespace from a FilesMap of an image + // layer. + Detect(tarutil.FilesMap) (*database.Namespace, error) + + // RequireFilenames returns the list of files required to be in the FilesMap + // provided to the Detect method. + // TODO(jzelinskie): strip "/" prefix + RequiredFilenames() []string +} + +// RegisterDetector makes a detector available by the provided name. +// +// If called twice with the same name, the name is blank, or if the provided +// Detector is nil, this function panics. +func RegisterDetector(name string, d Detector) { + if name == "" { + panic("namespace: could not register a Detector with an empty name") + } + if d == nil { + panic("namespace: could not register a nil Detector") + } + + detectorsM.Lock() + defer detectorsM.Unlock() + + if _, dup := detectors[name]; dup { + panic("namespace: RegisterDetector called twice for " + name) + } + + detectors[name] = d +} + +// Detect iterators through all registered Detectors and returns the first +// non-nil detected namespace. +func Detect(files tarutil.FilesMap) (*database.Namespace, error) { + detectorsM.RLock() + defer detectorsM.RUnlock() + + for name, detector := range detectors { + namespace, err := detector.Detect(files) + if err != nil { + log.Warningf("failed while attempting to detect namespace %s: %s", name, err) + return nil, err + } + + if namespace != nil { + log.Debugf("detected namespace %s: %#v", name, namespace) + return namespace, nil + } + } + + return nil, nil +} + +// RequiredFilenames returns the total list of files required for all +// registered Detectors. +func RequiredFilenames() (files []string) { + for _, detector := range detectors { + files = append(files, detector.RequiredFilenames()...) + } + + return +} + +// TestData represents the data used to test an implementation of +// NameSpaceDetector. +type TestData struct { + Files tarutil.FilesMap + ExpectedNamespace *database.Namespace +} + +// TestDetector runs a Detector on each provided instance of TestData and +// asserts the output to be equal to the expected output. +func TestDetector(t *testing.T, d Detector, testData []TestData) { + for _, td := range testData { + namespace, err := d.Detect(td.Files) + assert.Nil(t, err) + + if namespace == nil { + assert.Equal(t, td.ExpectedNamespace, namespace) + } else { + assert.Equal(t, td.ExpectedNamespace.Name, namespace.Name) + } + } +} diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease.go b/ext/featurens/lsbrelease/lsbrelease.go similarity index 73% rename from worker/detectors/namespace/lsbrelease/lsbrelease.go rename to ext/featurens/lsbrelease/lsbrelease.go index 8f9ba81a..d883215f 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease.go +++ b/ext/featurens/lsbrelease/lsbrelease.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package lsbrelease implements a featurens.Detector for container image +// layers containing an lsb-release file. +// +// This detector is necessary for detecting Ubuntu Precise. package lsbrelease import ( @@ -20,9 +24,10 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) var ( @@ -30,20 +35,16 @@ var ( lsbReleaseVersionRegexp = regexp.MustCompile(`^DISTRIB_RELEASE=(.*)`) ) -// LsbReleaseNamespaceDetector implements NamespaceDetector and detects the -// Namespace from the /etc/lsb-release file. -// -// This detector is necessary for Ubuntu Precise. -type LsbReleaseNamespaceDetector struct{} +type detector struct{} func init() { - detectors.RegisterNamespaceDetector("lsb-release", &LsbReleaseNamespaceDetector{}) + featurens.RegisterDetector("lsb-release", &detector{}) } -func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { - f, hasFile := data["etc/lsb-release"] +func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { + f, hasFile := files["etc/lsb-release"] if !hasFile { - return nil + return nil, nil } var OS, version string @@ -79,19 +80,19 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat case "centos", "rhel", "fedora", "amzn", "ol", "oracle": versionFormat = rpm.ParserName default: - return nil + return nil, nil } if OS != "" && version != "" { return &database.Namespace{ Name: OS + ":" + version, VersionFormat: versionFormat, - } + }, nil } - return nil + + return nil, nil } -// GetRequiredFiles returns the list of files that are required for Detect() -func (detector *LsbReleaseNamespaceDetector) GetRequiredFiles() []string { +func (d *detector) RequiredFilenames() []string { return []string{"etc/lsb-release"} } diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go b/ext/featurens/lsbrelease/lsbrelease_test.go similarity index 79% rename from worker/detectors/namespace/lsbrelease/lsbrelease_test.go rename to ext/featurens/lsbrelease/lsbrelease_test.go index 6d24f629..62ae52db 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go +++ b/ext/featurens/lsbrelease/lsbrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/namespace" + "github.com/coreos/clair/ext/featurens" + "github.com/coreos/clair/pkg/tarutil" ) -func TestLsbReleaseNamespaceDetector(t *testing.T) { - testData := []namespace.TestData{ +func TestDetector(t *testing.T) { + testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "ubuntu:12.04"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/lsb-release": []byte( `DISTRIB_ID=Ubuntu DISTRIB_RELEASE=12.04 @@ -35,7 +36,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`), }, { // We don't care about the minor version of Debian ExpectedNamespace: &database.Namespace{Name: "debian:7"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/lsb-release": []byte( `DISTRIB_ID=Debian DISTRIB_RELEASE=7.1 @@ -45,5 +46,5 @@ DISTRIB_DESCRIPTION="Debian 7.1"`), }, } - namespace.TestDetector(t, &LsbReleaseNamespaceDetector{}, testData) + featurens.TestDetector(t, &detector{}, testData) } diff --git a/worker/detectors/namespace/osrelease/osrelease.go b/ext/featurens/osrelease/osrelease.go similarity index 55% rename from worker/detectors/namespace/osrelease/osrelease.go rename to ext/featurens/osrelease/osrelease.go index d6467bdf..1139739d 100644 --- a/worker/detectors/namespace/osrelease/osrelease.go +++ b/ext/featurens/osrelease/osrelease.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package osrelease implements a featurens.Detector for container image +// layers containing an os-release file. +// +// This detector is typically useful for detecting Debian or Ubuntu. package osrelease import ( @@ -20,40 +24,41 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) var ( - //log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/namespace/osrelease") - osReleaseOSRegexp = regexp.MustCompile(`^ID=(.*)`) osReleaseVersionRegexp = regexp.MustCompile(`^VERSION_ID=(.*)`) + + // blacklistFilenames are files that should exclude this detector. + blacklistFilenames = []string{ + "etc/oracle-release", + "etc/redhat-release", + "usr/lib/centos-release", + } ) -// OsReleaseNamespaceDetector implements NamespaceDetector and detects the OS from the -// /etc/os-release and usr/lib/os-release files. -type OsReleaseNamespaceDetector struct{} +type detector struct{} func init() { - detectors.RegisterNamespaceDetector("os-release", &OsReleaseNamespaceDetector{}) + featurens.RegisterDetector("os-release", &detector{}) } -// Detect tries to detect OS/Version using "/etc/os-release" and "/usr/lib/os-release" -// Typically for Debian / Ubuntu -// /etc/debian_version can't be used, it does not make any difference between testing and unstable, it returns stretch/sid -func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { +func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { var OS, version string - for _, filePath := range detector.getExcludeFiles() { - if _, hasFile := data[filePath]; hasFile { - return nil + for _, filePath := range blacklistFilenames { + if _, hasFile := files[filePath]; hasFile { + return nil, nil } } - for _, filePath := range detector.GetRequiredFiles() { - f, hasFile := data[filePath] + for _, filePath := range d.RequiredFilenames() { + f, hasFile := files[filePath] if !hasFile { continue } @@ -82,24 +87,18 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data case "centos", "rhel", "fedora", "amzn", "ol", "oracle": versionFormat = rpm.ParserName default: - return nil + return nil, nil } if OS != "" && version != "" { return &database.Namespace{ Name: OS + ":" + version, VersionFormat: versionFormat, - } + }, nil } - return nil + return nil, nil } -// GetRequiredFiles returns the list of files that are required for Detect() -func (detector *OsReleaseNamespaceDetector) GetRequiredFiles() []string { +func (d detector) RequiredFilenames() []string { return []string{"etc/os-release", "usr/lib/os-release"} } - -// getExcludeFiles returns the list of files that are ought to exclude this detector from Detect() -func (detector *OsReleaseNamespaceDetector) getExcludeFiles() []string { - return []string{"etc/oracle-release", "etc/redhat-release", "usr/lib/centos-release"} -} diff --git a/worker/detectors/namespace/osrelease/osrelease_test.go b/ext/featurens/osrelease/osrelease_test.go similarity index 85% rename from worker/detectors/namespace/osrelease/osrelease_test.go rename to ext/featurens/osrelease/osrelease_test.go index f02f409a..2aaa0ec7 100644 --- a/worker/detectors/namespace/osrelease/osrelease_test.go +++ b/ext/featurens/osrelease/osrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,14 +18,15 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/namespace" + "github.com/coreos/clair/ext/featurens" + "github.com/coreos/clair/pkg/tarutil" ) -func TestOsReleaseNamespaceDetector(t *testing.T) { - testData := []namespace.TestData{ +func TestDetector(t *testing.T) { + testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "debian:8"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" NAME="Debian GNU/Linux" @@ -39,7 +40,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`), }, { ExpectedNamespace: &database.Namespace{Name: "ubuntu:15.10"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/os-release": []byte( `NAME="Ubuntu" VERSION="15.10 (Wily Werewolf)" @@ -54,7 +55,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`), }, { // Doesn't have quotes around VERSION_ID ExpectedNamespace: &database.Namespace{Name: "fedora:20"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/os-release": []byte( `NAME=Fedora VERSION="20 (Heisenbug)" @@ -73,5 +74,5 @@ REDHAT_SUPPORT_PRODUCT_VERSION=20`), }, } - namespace.TestDetector(t, &OsReleaseNamespaceDetector{}, testData) + featurens.TestDetector(t, &detector{}, testData) } diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease.go b/ext/featurens/redhatrelease/redhatrelease.go similarity index 58% rename from worker/detectors/namespace/redhatrelease/redhatrelease.go rename to ext/featurens/redhatrelease/redhatrelease.go index 4271c47c..2b15c1e9 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease.go +++ b/ext/featurens/redhatrelease/redhatrelease.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package redhatrelease implements a featurens.Detector for container image +// layers containing an redhat-release-like files. +// +// This detector is typically useful for detecting CentOS and Red-Hat like +// systems. package redhatrelease import ( @@ -19,77 +24,64 @@ import ( "strings" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/worker/detectors" - "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair/pkg/tarutil" ) var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/namespace/redhatrelease") - oracleReleaseRegexp = regexp.MustCompile(`(?P[^\s]*) (Linux Server release) (?P[\d]+)`) centosReleaseRegexp = regexp.MustCompile(`(?P[^\s]*) (Linux release|release) (?P[\d]+)`) redhatReleaseRegexp = regexp.MustCompile(`(?PRed Hat Enterprise Linux) (Client release|Server release|Workstation release) (?P[\d]+)`) ) -// RedhatReleaseNamespaceDetector implements NamespaceDetector and detects the OS from the -// /etc/oracle-release, /etc/centos-release, /etc/redhat-release and /etc/system-release files. -// -// Typically for CentOS and Red-Hat like systems -// eg. CentOS release 5.11 (Final) -// eg. CentOS release 6.6 (Final) -// eg. CentOS Linux release 7.1.1503 (Core) -// eg. Oracle Linux Server release 7.3 -// eg. Red Hat Enterprise Linux Server release 7.2 (Maipo) -type RedhatReleaseNamespaceDetector struct{} +type detector struct{} func init() { - detectors.RegisterNamespaceDetector("redhat-release", &RedhatReleaseNamespaceDetector{}) + featurens.RegisterDetector("redhat-release", &detector{}) } -func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { - for _, filePath := range detector.GetRequiredFiles() { - f, hasFile := data[filePath] +func (d detector) Detect(files tarutil.FilesMap) (*database.Namespace, error) { + for _, filePath := range d.RequiredFilenames() { + f, hasFile := files[filePath] if !hasFile { continue } var r []string - // try for Oracle Linux + // Attempt to match Oracle Linux. r = oracleReleaseRegexp.FindStringSubmatch(string(f)) if len(r) == 4 { return &database.Namespace{ Name: strings.ToLower(r[1]) + ":" + r[3], VersionFormat: rpm.ParserName, - } + }, nil } - // try for RHEL + // Attempt to match RHEL. r = redhatReleaseRegexp.FindStringSubmatch(string(f)) if len(r) == 4 { - // TODO(vbatts) this is a hack until https://github.com/coreos/clair/pull/193 + // TODO(vbatts): this is a hack until https://github.com/coreos/clair/pull/193 return &database.Namespace{ Name: "centos" + ":" + r[3], VersionFormat: rpm.ParserName, - } + }, nil } - // then try centos first + // Atempt to match CentOS. r = centosReleaseRegexp.FindStringSubmatch(string(f)) if len(r) == 4 { return &database.Namespace{ Name: strings.ToLower(r[1]) + ":" + r[3], VersionFormat: rpm.ParserName, - } + }, nil } - } - return nil + return nil, nil } -// GetRequiredFiles returns the list of files that are required for Detect() -func (detector *RedhatReleaseNamespaceDetector) GetRequiredFiles() []string { +func (d detector) RequiredFilenames() []string { return []string{"etc/oracle-release", "etc/centos-release", "etc/redhat-release", "etc/system-release"} } diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go b/ext/featurens/redhatrelease/redhatrelease_test.go similarity index 77% rename from worker/detectors/namespace/redhatrelease/redhatrelease_test.go rename to ext/featurens/redhatrelease/redhatrelease_test.go index dc360545..a6977f2c 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go +++ b/ext/featurens/redhatrelease/redhatrelease_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,36 +18,37 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/namespace" + "github.com/coreos/clair/ext/featurens" + "github.com/coreos/clair/pkg/tarutil" ) -func TestRedhatReleaseNamespaceDetector(t *testing.T) { - testData := []namespace.TestData{ +func TestDetector(t *testing.T) { + testData := []featurens.TestData{ { ExpectedNamespace: &database.Namespace{Name: "oracle:6"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/oracle-release": []byte(`Oracle Linux Server release 6.8`), }, }, { ExpectedNamespace: &database.Namespace{Name: "oracle:7"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/oracle-release": []byte(`Oracle Linux Server release 7.2`), }, }, { ExpectedNamespace: &database.Namespace{Name: "centos:6"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/centos-release": []byte(`CentOS release 6.6 (Final)`), }, }, { ExpectedNamespace: &database.Namespace{Name: "centos:7"}, - Data: map[string][]byte{ + Files: tarutil.FilesMap{ "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), }, }, } - namespace.TestDetector(t, &RedhatReleaseNamespaceDetector{}, testData) + featurens.TestDetector(t, &detector{}, testData) } diff --git a/worker/detectors/namespace.go b/worker/detectors/namespace.go deleted file mode 100644 index e77ef23d..00000000 --- a/worker/detectors/namespace.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package detectors exposes functions to register and use container -// information extractors. -package detectors - -import ( - "fmt" - "sync" - - "github.com/coreos/clair/database" - "github.com/coreos/pkg/capnslog" -) - -// The NamespaceDetector interface defines a way to detect a Namespace from input data. -// A namespace is usually made of an Operating System name and its version. -type NamespaceDetector interface { - // Detect detects a Namespace and its version from input data. - Detect(map[string][]byte) *database.Namespace - // GetRequiredFiles returns the list of files required for Detect, without - // leading /. - GetRequiredFiles() []string -} - -var ( - nlog = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors") - - namespaceDetectorsLock sync.Mutex - namespaceDetectors = make(map[string]NamespaceDetector) -) - -// RegisterNamespaceDetector provides a way to dynamically register an implementation of a -// NamespaceDetector. -// -// If RegisterNamespaceDetector is called twice with the same name if NamespaceDetector is nil, -// or if the name is blank, it panics. -func RegisterNamespaceDetector(name string, f NamespaceDetector) { - if name == "" { - panic("Could not register a NamespaceDetector with an empty name") - } - if f == nil { - panic("Could not register a nil NamespaceDetector") - } - - namespaceDetectorsLock.Lock() - defer namespaceDetectorsLock.Unlock() - - if _, alreadyExists := namespaceDetectors[name]; alreadyExists { - panic(fmt.Sprintf("Detector '%s' is already registered", name)) - } - namespaceDetectors[name] = f -} - -// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector. -func DetectNamespace(data map[string][]byte) *database.Namespace { - for name, detector := range namespaceDetectors { - if namespace := detector.Detect(data); namespace != nil { - nlog.Debugf("detector: %q; namespace: %q\n", name, namespace.Name) - return namespace - } - } - - return nil -} - -// GetRequiredFilesNamespace returns the list of files required for DetectNamespace for every -// registered NamespaceDetector, without leading /. -func GetRequiredFilesNamespace() (files []string) { - for _, detector := range namespaceDetectors { - files = append(files, detector.GetRequiredFiles()...) - } - - return -} diff --git a/worker/detectors/namespace/test.go b/worker/detectors/namespace/test.go deleted file mode 100644 index 89e33ec5..00000000 --- a/worker/detectors/namespace/test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2016 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package namespace implements utilities common to implementations of -// NamespaceDetector. -package namespace - -import ( - "testing" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors" - "github.com/stretchr/testify/assert" -) - -// TestData represents the data used to test an implementation of -// NameSpaceDetector. -type TestData struct { - Data map[string][]byte - ExpectedNamespace *database.Namespace -} - -// TestDetector runs a detector on each provided instance of TestData and -// asserts the output to be equal to the expected output. -func TestDetector(t *testing.T, detector detectors.NamespaceDetector, testData []TestData) { - for _, td := range testData { - detectedNamespace := detector.Detect(td.Data) - if detectedNamespace == nil { - assert.Equal(t, td.ExpectedNamespace, detectedNamespace) - } else { - assert.Equal(t, td.ExpectedNamespace.Name, detectedNamespace.Name) - } - } -} diff --git a/worker/worker.go b/worker/worker.go index 904155ee..d9e8f371 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -20,8 +20,10 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/tarutil" "github.com/coreos/clair/utils" "github.com/coreos/clair/worker/detectors" ) @@ -110,20 +112,23 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s return datastore.InsertLayer(layer) } -// detectContent downloads a layer's archive and extracts its Namespace and Features. +// detectContent downloads a layer's archive and extracts its Namespace and +// Features. func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { - files, err := imagefmt.Extract(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...)) + totalRequiredFiles := append(detectors.GetRequiredFilesFeatures(), featurens.RequiredFilenames()...) + files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) return } - data := map[string][]byte(files) - - // Detect namespace. - namespace = detectNamespace(name, data, parent) + namespace, err = detectNamespace(name, files, parent) + if err != nil { + return + } // Detect features. + data := map[string][]byte(files) featureVersions, err = detectFeatureVersions(name, data, namespace, parent) if err != nil { return @@ -135,15 +140,17 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa return } -func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) { - // Use registered detectors to get the Namespace. - namespace = detectors.DetectNamespace(data) +func detectNamespace(name string, files tarutil.FilesMap, parent *database.Layer) (namespace *database.Namespace, err error) { + namespace, err = featurens.Detect(files) + if err != nil { + return + } if namespace != nil { log.Debugf("layer %s: detected namespace %q", name, namespace.Name) return } - // Use the parent's Namespace. + // Fallback to the parent's namespace. if parent != nil { namespace = parent.Namespace if namespace != nil { From 71a8b542f95cf34746d1b544a0fe1790a9f6eb09 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 16:48:46 -0500 Subject: [PATCH 09/23] ext: misc doc comment fixes --- ext/imagefmt/aci/aci.go | 2 ++ ext/imagefmt/docker/docker.go | 2 ++ ext/imagefmt/driver.go | 6 +++--- ext/versionfmt/dpkg/parser.go | 4 +++- ext/versionfmt/rpm/parser.go | 4 +++- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ext/imagefmt/aci/aci.go b/ext/imagefmt/aci/aci.go index 38e26217..ed6fd5b8 100644 --- a/ext/imagefmt/aci/aci.go +++ b/ext/imagefmt/aci/aci.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package aci implements an imagefmt.Extractor for appc formatted container +// image layers. package aci import ( diff --git a/ext/imagefmt/docker/docker.go b/ext/imagefmt/docker/docker.go index 94f866fd..e69ee2e4 100644 --- a/ext/imagefmt/docker/docker.go +++ b/ext/imagefmt/docker/docker.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package docker implements an imagefmt.Extractor for docker formatted +// container image layers. package docker import ( diff --git a/ext/imagefmt/driver.go b/ext/imagefmt/driver.go index 81d8ce3f..6b14f48e 100644 --- a/ext/imagefmt/driver.go +++ b/ext/imagefmt/driver.go @@ -51,7 +51,7 @@ type Extractor interface { ExtractFiles(layer io.ReadCloser, filenames []string) (tarutil.FilesMap, error) } -// RegisterExtractor makes a extractor available by the provided name. +// RegisterExtractor makes an extractor available by the provided name. // // If called twice with the same name, the name is blank, or if the provided // Extractor is nil, this function panics. @@ -60,11 +60,11 @@ func RegisterExtractor(name string, d Extractor) { defer extractorsM.Unlock() if name == "" { - panic("imagefmt: could not register a extractor with an empty name") + panic("imagefmt: could not register an Extractor with an empty name") } if d == nil { - panic("imagefmt: could not register a nil extractor") + panic("imagefmt: could not register a nil Extractor") } // Enforce lowercase names, so that they can be reliably be found in a map. diff --git a/ext/versionfmt/dpkg/parser.go b/ext/versionfmt/dpkg/parser.go index d170c9ca..42fbd45e 100644 --- a/ext/versionfmt/dpkg/parser.go +++ b/ext/versionfmt/dpkg/parser.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package dpkg implements a versionfmt.Parser for version numbers used in dpkg +// based software packages. package dpkg import ( diff --git a/ext/versionfmt/rpm/parser.go b/ext/versionfmt/rpm/parser.go index 42fb71ab..a05f14d6 100644 --- a/ext/versionfmt/rpm/parser.go +++ b/ext/versionfmt/rpm/parser.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package rpm implements a versionfmt.Parser for version numbers used in rpm +// based software packages. package rpm import ( From cda3d4819c261932ad24196f51f4a4b4fec022bd Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 18:49:02 -0500 Subject: [PATCH 10/23] ext: feature detector -> featurefmt --- cmd/clair/main.go | 12 +- .../feature => ext/featurefmt}/apk/apk.go | 18 +-- .../featurefmt}/apk/apk_test.go | 13 +- .../featurefmt}/apk/testdata/installed | 0 .../feature => ext/featurefmt}/dpkg/dpkg.go | 22 ++- .../featurefmt}/dpkg/dpkg_test.go | 13 +- .../featurefmt}/dpkg/testdata/status | 0 ext/featurefmt/driver.go | 126 ++++++++++++++++++ .../feature => ext/featurefmt}/rpm/rpm.go | 21 ++- .../featurefmt}/rpm/rpm_test.go | 13 +- .../featurefmt}/rpm/testdata/Packages | Bin worker/detectors/feature/test.go | 56 -------- worker/detectors/features.go | 79 ----------- worker/worker.go | 11 +- worker/worker_test.go | 8 +- 15 files changed, 191 insertions(+), 201 deletions(-) rename {worker/detectors/feature => ext/featurefmt}/apk/apk.go (82%) rename {worker/detectors/feature => ext/featurefmt}/apk/apk_test.go (85%) rename {worker/detectors/feature => ext/featurefmt}/apk/testdata/installed (100%) rename {worker/detectors/feature => ext/featurefmt}/dpkg/dpkg.go (82%) rename {worker/detectors/feature => ext/featurefmt}/dpkg/dpkg_test.go (81%) rename {worker/detectors/feature => ext/featurefmt}/dpkg/testdata/status (100%) create mode 100644 ext/featurefmt/driver.go rename {worker/detectors/feature => ext/featurefmt}/rpm/rpm.go (83%) rename {worker/detectors/feature => ext/featurefmt}/rpm/rpm_test.go (80%) rename {worker/detectors/feature => ext/featurefmt}/rpm/testdata/Packages (100%) delete mode 100644 worker/detectors/feature/test.go delete mode 100644 worker/detectors/features.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 2688880e..74e764cf 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -25,7 +25,13 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/config" + // Register database driver. + _ "github.com/coreos/clair/database/pgsql" + // Register extensions. + _ "github.com/coreos/clair/ext/featurefmt/apk" + _ "github.com/coreos/clair/ext/featurefmt/dpkg" + _ "github.com/coreos/clair/ext/featurefmt/rpm" _ "github.com/coreos/clair/ext/featurens/alpinerelease" _ "github.com/coreos/clair/ext/featurens/aptsources" _ "github.com/coreos/clair/ext/featurens/lsbrelease" @@ -40,12 +46,6 @@ import ( _ "github.com/coreos/clair/ext/vulnsrc/oracle" _ "github.com/coreos/clair/ext/vulnsrc/rhel" _ "github.com/coreos/clair/ext/vulnsrc/ubuntu" - - _ "github.com/coreos/clair/worker/detectors/feature/apk" - _ "github.com/coreos/clair/worker/detectors/feature/dpkg" - _ "github.com/coreos/clair/worker/detectors/feature/rpm" - - _ "github.com/coreos/clair/database/pgsql" ) var log = capnslog.NewPackageLogger("github.com/coreos/clair/cmd/clair", "main") diff --git a/worker/detectors/feature/apk/apk.go b/ext/featurefmt/apk/apk.go similarity index 82% rename from worker/detectors/feature/apk/apk.go rename to ext/featurefmt/apk/apk.go index 1d8594f2..2da9ebc6 100644 --- a/worker/detectors/feature/apk/apk.go +++ b/ext/featurefmt/apk/apk.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package apk implements a featurefmt.Lister for APK packages. package apk import ( @@ -21,21 +22,22 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) -var log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages") +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/apk") func init() { - detectors.RegisterFeaturesDetector("apk", &detector{}) + featurefmt.RegisterLister("apk", &lister{}) } -type detector struct{} +type lister struct{} -func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { - file, exists := data["lib/apk/db/installed"] +func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { + file, exists := files["lib/apk/db/installed"] if !exists { return []database.FeatureVersion{}, nil } @@ -83,6 +85,6 @@ func (d *detector) Detect(data map[string][]byte) ([]database.FeatureVersion, er return pkgs, nil } -func (d *detector) GetRequiredFiles() []string { +func (l lister) RequiredFilenames() []string { return []string{"lib/apk/db/installed"} } diff --git a/worker/detectors/feature/apk/apk_test.go b/ext/featurefmt/apk/apk_test.go similarity index 85% rename from worker/detectors/feature/apk/apk_test.go rename to ext/featurefmt/apk/apk_test.go index ccf9b4c0..6dbde3e6 100644 --- a/worker/detectors/feature/apk/apk_test.go +++ b/ext/featurefmt/apk/apk_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,12 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/feature" + "github.com/coreos/clair/ext/featurefmt" + "github.com/coreos/clair/pkg/tarutil" ) func TestAPKFeatureDetection(t *testing.T) { - testData := []feature.TestData{ + testData := []featurefmt.TestData{ { FeatureVersions: []database.FeatureVersion{ { @@ -70,10 +71,10 @@ func TestAPKFeatureDetection(t *testing.T) { Version: "0.7-r0", }, }, - Data: map[string][]byte{ - "lib/apk/db/installed": feature.LoadFileForTest("apk/testdata/installed"), + Files: tarutil.FilesMap{ + "lib/apk/db/installed": featurefmt.LoadFileForTest("apk/testdata/installed"), }, }, } - feature.TestDetector(t, &detector{}, testData) + featurefmt.TestLister(t, &lister{}, testData) } diff --git a/worker/detectors/feature/apk/testdata/installed b/ext/featurefmt/apk/testdata/installed similarity index 100% rename from worker/detectors/feature/apk/testdata/installed rename to ext/featurefmt/apk/testdata/installed diff --git a/worker/detectors/feature/dpkg/dpkg.go b/ext/featurefmt/dpkg/dpkg.go similarity index 82% rename from worker/detectors/feature/dpkg/dpkg.go rename to ext/featurefmt/dpkg/dpkg.go index f6a03dda..4e72dfc7 100644 --- a/worker/detectors/feature/dpkg/dpkg.go +++ b/ext/featurefmt/dpkg/dpkg.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package dpkg implements a featurefmt.Lister for dpkg packages. package dpkg import ( @@ -22,28 +23,27 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/worker/detectors" + "github.com/coreos/clair/pkg/tarutil" ) var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker/detectors/packages") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/dpkg") dpkgSrcCaptureRegexp = regexp.MustCompile(`Source: (?P[^\s]*)( \((?P.*)\))?`) dpkgSrcCaptureRegexpNames = dpkgSrcCaptureRegexp.SubexpNames() ) -// DpkgFeaturesDetector implements FeaturesDetector and detects dpkg packages -type DpkgFeaturesDetector struct{} +type lister struct{} func init() { - detectors.RegisterFeaturesDetector("dpkg", &DpkgFeaturesDetector{}) + featurefmt.RegisterLister("dpkg", &lister{}) } -// Detect detects packages using var/lib/dpkg/status from the input data -func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { - f, hasFile := data["var/lib/dpkg/status"] +func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { + f, hasFile := files["var/lib/dpkg/status"] if !hasFile { return []database.FeatureVersion{}, nil } @@ -116,8 +116,6 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database return packages, nil } -// GetRequiredFiles returns the list of files required for Detect, without -// leading / -func (detector *DpkgFeaturesDetector) GetRequiredFiles() []string { +func (l lister) RequiredFilenames() []string { return []string{"var/lib/dpkg/status"} } diff --git a/worker/detectors/feature/dpkg/dpkg_test.go b/ext/featurefmt/dpkg/dpkg_test.go similarity index 81% rename from worker/detectors/feature/dpkg/dpkg_test.go rename to ext/featurefmt/dpkg/dpkg_test.go index 30f8ec5e..a9c3a8cf 100644 --- a/worker/detectors/feature/dpkg/dpkg_test.go +++ b/ext/featurefmt/dpkg/dpkg_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,12 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/feature" + "github.com/coreos/clair/ext/featurefmt" + "github.com/coreos/clair/pkg/tarutil" ) func TestDpkgFeatureDetection(t *testing.T) { - testData := []feature.TestData{ + testData := []featurefmt.TestData{ // Test an Ubuntu dpkg status file { FeatureVersions: []database.FeatureVersion{ @@ -40,11 +41,11 @@ func TestDpkgFeatureDetection(t *testing.T) { Version: "5.1.1-12ubuntu1", // The version comes from the "Source:" line }, }, - Data: map[string][]byte{ - "var/lib/dpkg/status": feature.LoadFileForTest("dpkg/testdata/status"), + Files: tarutil.FilesMap{ + "var/lib/dpkg/status": featurefmt.LoadFileForTest("dpkg/testdata/status"), }, }, } - feature.TestDetector(t, &DpkgFeaturesDetector{}, testData) + featurefmt.TestLister(t, &lister{}, testData) } diff --git a/worker/detectors/feature/dpkg/testdata/status b/ext/featurefmt/dpkg/testdata/status similarity index 100% rename from worker/detectors/feature/dpkg/testdata/status rename to ext/featurefmt/dpkg/testdata/status diff --git a/ext/featurefmt/driver.go b/ext/featurefmt/driver.go new file mode 100644 index 00000000..5b9fc2eb --- /dev/null +++ b/ext/featurefmt/driver.go @@ -0,0 +1,126 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package featurefmt exposes functions to dynamically register methods for +// determining the features present in an image layer. +package featurefmt + +import ( + "io/ioutil" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/tarutil" + "github.com/stretchr/testify/assert" +) + +var ( + listersM sync.RWMutex + listers = make(map[string]Lister) +) + +// Lister represents an ability to list the features present in an image layer. +type Lister interface { + // ListFeatures produces a list of FeatureVersions present in an image layer. + ListFeatures(tarutil.FilesMap) ([]database.FeatureVersion, error) + + // RequiredFilenames returns the list of files required to be in the FilesMap + // provided to the ListFeatures method. + // + // Filenames must not begin with "/". + RequiredFilenames() []string +} + +// RegisterLister makes a Lister available by the provided name. +// +// If called twice with the same name, the name is blank, or if the provided +// Lister is nil, this function panics. +func RegisterLister(name string, l Lister) { + if name == "" { + panic("featurefmt: could not register a Lister with an empty name") + } + if l == nil { + panic("featurefmt: could not register a nil Lister") + } + + listersM.Lock() + defer listersM.Unlock() + + if _, dup := listers[name]; dup { + panic("featurefmt: RegisterLister called twice for " + name) + } + + listers[name] = l +} + +// ListFeatures produces the list of FeatureVersions in an image layer using +// every registered Lister. +func ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { + listersM.RLock() + defer listersM.RUnlock() + + var totalFeatures []database.FeatureVersion + for _, lister := range listers { + features, err := lister.ListFeatures(files) + if err != nil { + return []database.FeatureVersion{}, err + } + totalFeatures = append(totalFeatures, features...) + } + + return totalFeatures, nil +} + +// RequiredFilenames returns the total list of files required for all +// registered Listers. +func RequiredFilenames() (files []string) { + listersM.RLock() + defer listersM.RUnlock() + + for _, lister := range listers { + files = append(files, lister.RequiredFilenames()...) + } + + return +} + +// TestData represents the data used to test an implementation of Lister. +type TestData struct { + Files tarutil.FilesMap + FeatureVersions []database.FeatureVersion +} + +// LoadFileForTest can be used in order to obtain the []byte contents of a file +// that is meant to be used for test data. +func LoadFileForTest(name string) []byte { + _, filename, _, _ := runtime.Caller(0) + d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name) + return d +} + +// TestLister runs a Lister on each provided instance of TestData and asserts +// the ouput to be equal to the expected output. +func TestLister(t *testing.T, l Lister, testData []TestData) { + for _, td := range testData { + featureVersions, err := l.ListFeatures(td.Files) + if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.FeatureVersions)) { + for _, expectedFeatureVersion := range td.FeatureVersions { + assert.Contains(t, featureVersions, expectedFeatureVersion) + } + } + } +} diff --git a/worker/detectors/feature/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go similarity index 83% rename from worker/detectors/feature/rpm/rpm.go rename to ext/featurefmt/rpm/rpm.go index 3a158d83..ffcdaaf7 100644 --- a/worker/detectors/feature/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package rpm implements a featurefmt.Lister for rpm packages. package rpm import ( @@ -23,26 +24,24 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/pkg/commonerr" + "github.com/coreos/clair/pkg/tarutil" "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" ) -var log = capnslog.NewPackageLogger("github.com/coreos/clair", "rpm") +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/rpm") -// RpmFeaturesDetector implements FeaturesDetector and detects rpm packages -// It requires the "rpm" binary to be in the PATH -type RpmFeaturesDetector struct{} +type lister struct{} func init() { - detectors.RegisterFeaturesDetector("rpm", &RpmFeaturesDetector{}) + featurefmt.RegisterLister("rpm", &lister{}) } -// Detect detects packages using var/lib/rpm/Packages from the input data -func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { - f, hasFile := data["var/lib/rpm/Packages"] +func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, error) { + f, hasFile := files["var/lib/rpm/Packages"] if !hasFile { return []database.FeatureVersion{}, nil } @@ -116,8 +115,6 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. return packages, nil } -// GetRequiredFiles returns the list of files required for Detect, without -// leading / -func (detector *RpmFeaturesDetector) GetRequiredFiles() []string { +func (l lister) RequiredFilenames() []string { return []string{"var/lib/rpm/Packages"} } diff --git a/worker/detectors/feature/rpm/rpm_test.go b/ext/featurefmt/rpm/rpm_test.go similarity index 80% rename from worker/detectors/feature/rpm/rpm_test.go rename to ext/featurefmt/rpm/rpm_test.go index d937eaf9..1b6f531c 100644 --- a/worker/detectors/feature/rpm/rpm_test.go +++ b/ext/featurefmt/rpm/rpm_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,12 @@ import ( "testing" "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors/feature" + "github.com/coreos/clair/ext/featurefmt" + "github.com/coreos/clair/pkg/tarutil" ) func TestRpmFeatureDetection(t *testing.T) { - testData := []feature.TestData{ + testData := []featurefmt.TestData{ // Test a CentOS 7 RPM database // Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb { @@ -38,11 +39,11 @@ func TestRpmFeatureDetection(t *testing.T) { Version: "3.2-18.el7", }, }, - Data: map[string][]byte{ - "var/lib/rpm/Packages": feature.LoadFileForTest("rpm/testdata/Packages"), + Files: tarutil.FilesMap{ + "var/lib/rpm/Packages": featurefmt.LoadFileForTest("rpm/testdata/Packages"), }, }, } - feature.TestDetector(t, &RpmFeaturesDetector{}, testData) + featurefmt.TestLister(t, &lister{}, testData) } diff --git a/worker/detectors/feature/rpm/testdata/Packages b/ext/featurefmt/rpm/testdata/Packages similarity index 100% rename from worker/detectors/feature/rpm/testdata/Packages rename to ext/featurefmt/rpm/testdata/Packages diff --git a/worker/detectors/feature/test.go b/worker/detectors/feature/test.go deleted file mode 100644 index 45335d7d..00000000 --- a/worker/detectors/feature/test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2016 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package feature implements utilities common to implementations of -// FeatureDetector. -package feature - -import ( - "io/ioutil" - "path/filepath" - "runtime" - "testing" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/worker/detectors" - "github.com/stretchr/testify/assert" -) - -// TestData represents the data used to test an implementation of -// FeatureDetector. -type TestData struct { - Data map[string][]byte - FeatureVersions []database.FeatureVersion -} - -// LoadFileForTest can be used in order to obtain the []byte contents of a file -// that is meant to be used for test data. -func LoadFileForTest(name string) []byte { - _, filename, _, _ := runtime.Caller(0) - d, _ := ioutil.ReadFile(filepath.Join(filepath.Dir(filename)) + "/" + name) - return d -} - -// TestDetector runs a detector on each provided instance of TestData and -// asserts the ouput to be equal to the expected output. -func TestDetector(t *testing.T, detector detectors.FeaturesDetector, testData []TestData) { - for _, td := range testData { - featureVersions, err := detector.Detect(td.Data) - if assert.Nil(t, err) && assert.Len(t, featureVersions, len(td.FeatureVersions)) { - for _, expectedFeatureVersion := range td.FeatureVersions { - assert.Contains(t, featureVersions, expectedFeatureVersion) - } - } - } -} diff --git a/worker/detectors/features.go b/worker/detectors/features.go deleted file mode 100644 index da8d8c06..00000000 --- a/worker/detectors/features.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package detectors - -import ( - "fmt" - "sync" - - "github.com/coreos/clair/database" -) - -// The FeaturesDetector interface defines a way to detect packages from input data. -type FeaturesDetector interface { - // Detect detects a list of FeatureVersion from the input data. - Detect(map[string][]byte) ([]database.FeatureVersion, error) - // GetRequiredFiles returns the list of files required for Detect, without - // leading /. - GetRequiredFiles() []string -} - -var ( - featuresDetectorsLock sync.Mutex - featuresDetectors = make(map[string]FeaturesDetector) -) - -// RegisterFeaturesDetector makes a FeaturesDetector available for DetectFeatures. -func RegisterFeaturesDetector(name string, f FeaturesDetector) { - if name == "" { - panic("Could not register a FeaturesDetector with an empty name") - } - if f == nil { - panic("Could not register a nil FeaturesDetector") - } - - featuresDetectorsLock.Lock() - defer featuresDetectorsLock.Unlock() - - if _, alreadyExists := featuresDetectors[name]; alreadyExists { - panic(fmt.Sprintf("Detector '%s' is already registered", name)) - } - featuresDetectors[name] = f -} - -// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector. -func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) { - var packages []database.FeatureVersion - - for _, detector := range featuresDetectors { - pkgs, err := detector.Detect(data) - if err != nil { - return []database.FeatureVersion{}, err - } - packages = append(packages, pkgs...) - } - - return packages, nil -} - -// GetRequiredFilesFeatures returns the list of files required for Detect for every -// registered FeaturesDetector, without leading /. -func GetRequiredFilesFeatures() (files []string) { - for _, detector := range featuresDetectors { - files = append(files, detector.GetRequiredFiles()...) - } - - return -} diff --git a/worker/worker.go b/worker/worker.go index d9e8f371..e6f00932 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -20,12 +20,12 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurens" "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" "github.com/coreos/clair/utils" - "github.com/coreos/clair/worker/detectors" ) const ( @@ -115,7 +115,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s // detectContent downloads a layer's archive and extracts its Namespace and // Features. func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { - totalRequiredFiles := append(detectors.GetRequiredFilesFeatures(), featurens.RequiredFilenames()...) + totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...) files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) @@ -128,8 +128,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa } // Detect features. - data := map[string][]byte(files) - featureVersions, err = detectFeatureVersions(name, data, namespace, parent) + featureVersions, err = detectFeatureVersions(name, files, namespace, parent) if err != nil { return } @@ -162,12 +161,12 @@ func detectNamespace(name string, files tarutil.FilesMap, parent *database.Layer return } -func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { +func detectFeatureVersions(name string, files tarutil.FilesMap, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { // TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that // some detectors would need it in order to produce the entire feature list (if they can only // detect a diff). Also, we should probably pass the detected namespace so detectors could // make their own decision. - features, err = detectors.DetectFeatures(data) + features, err = featurefmt.ListFeatures(files) if err != nil { return } diff --git a/worker/worker_test.go b/worker/worker_test.go index cc69ff11..26e104fa 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -26,10 +26,10 @@ import ( "github.com/coreos/clair/pkg/commonerr" // Register the required detectors. - _ "github.com/coreos/clair/worker/detectors/data/docker" - _ "github.com/coreos/clair/worker/detectors/feature/dpkg" - _ "github.com/coreos/clair/worker/detectors/namespace/aptsources" - _ "github.com/coreos/clair/worker/detectors/namespace/osrelease" + _ "github.com/coreos/clair/ext/featurefmt/dpkg" + _ "github.com/coreos/clair/ext/featurens/aptsources" + _ "github.com/coreos/clair/ext/featurens/osrelease" + _ "github.com/coreos/clair/ext/imagefmt/docker" ) type mockDatastore struct { From 03b8cd9a4584db0ca18032bf109469ceb2adc3d3 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 18:49:17 -0500 Subject: [PATCH 11/23] ext/featurens: add missing lock --- ext/featurens/driver.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ext/featurens/driver.go b/ext/featurens/driver.go index e1187a59..db066cbb 100644 --- a/ext/featurens/driver.go +++ b/ext/featurens/driver.go @@ -41,9 +41,10 @@ type Detector interface { // layer. Detect(tarutil.FilesMap) (*database.Namespace, error) - // RequireFilenames returns the list of files required to be in the FilesMap + // RequiredFilenames returns the list of files required to be in the FilesMap // provided to the Detect method. - // TODO(jzelinskie): strip "/" prefix + // + // Filenames must not begin with "/". RequiredFilenames() []string } @@ -94,6 +95,9 @@ func Detect(files tarutil.FilesMap) (*database.Namespace, error) { // RequiredFilenames returns the total list of files required for all // registered Detectors. func RequiredFilenames() (files []string) { + detectorsM.RLock() + defer detectorsM.RUnlock() + for _, detector := range detectors { files = append(files, detector.RequiredFilenames()...) } @@ -101,8 +105,7 @@ func RequiredFilenames() (files []string) { return } -// TestData represents the data used to test an implementation of -// NameSpaceDetector. +// TestData represents the data used to test an implementation of Detector. type TestData struct { Files tarutil.FilesMap ExpectedNamespace *database.Namespace From 02e2c5823670d9587a2143a231adfa3cd38a87bb Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 13 Jan 2017 18:55:40 -0500 Subject: [PATCH 12/23] utils/http: remove unused pkg --- utils/http/http.go | 77 ---------------------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 utils/http/http.go diff --git a/utils/http/http.go b/utils/http/http.go deleted file mode 100644 index 46ac5a0e..00000000 --- a/utils/http/http.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package http provides utility functions for HTTP servers and clients. -package http - -import ( - "encoding/json" - "io" - "net/http" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/pkg/tarutil" - "github.com/coreos/clair/worker" -) - -// MaxBodySize is the maximum number of bytes that ParseHTTPBody reads from an http.Request.Body. -const MaxBodySize int64 = 1048576 - -// WriteHTTP writes a JSON-encoded object to a http.ResponseWriter, as well as -// a HTTP status code. -func WriteHTTP(w http.ResponseWriter, httpStatus int, v interface{}) { - w.WriteHeader(httpStatus) - if v != nil { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - result, _ := json.Marshal(v) - w.Write(result) - } -} - -// WriteHTTPError writes an error, wrapped in the Message field of a JSON-encoded -// object to a http.ResponseWriter, as well as a HTTP status code. -// If the status code is 0, handleError tries to guess the proper HTTP status -// code from the error type. -func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) { - if httpStatus == 0 { - httpStatus = http.StatusInternalServerError - // Try to guess the http status code from the error type - if _, isBadRequestError := err.(*commonerr.ErrBadRequest); isBadRequestError { - httpStatus = http.StatusBadRequest - } else { - switch err { - case commonerr.ErrNotFound: - httpStatus = http.StatusNotFound - case database.ErrBackendException: - httpStatus = http.StatusServiceUnavailable - case worker.ErrParentUnknown, worker.ErrUnsupported, tarutil.ErrCouldNotExtract, tarutil.ErrExtractedFileTooBig: - httpStatus = http.StatusBadRequest - } - } - } - - WriteHTTP(w, httpStatus, struct{ Message string }{Message: err.Error()}) -} - -// ParseHTTPBody reads a JSON-encoded body from a http.Request and unmarshals it -// into the provided object. -func ParseHTTPBody(r *http.Request, v interface{}) (int, error) { - defer r.Body.Close() - err := json.NewDecoder(io.LimitReader(r.Body, MaxBodySize)).Decode(v) - if err != nil { - return http.StatusUnsupportedMediaType, err - } - return 0, nil -} From 343e24eb7eb6336dca94df7b43499dfef08ee4fe Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 15 Jan 2017 10:52:13 -0500 Subject: [PATCH 13/23] clair: remove `types` package This removes the `types` package instead moving the contents to the top-level clair package. This change also renames the `Priority` type to `Severity` in order to reduce confusion. This change also removes the IsValid method and replaces it with a safe constructor to avoid the creation of invalid values. Many docstrings were tweaked in the making of this commit. --- api/v1/models.go | 10 +- clair.go | 75 -------- cmd/clair/main.go | 98 ++++++++--- database/database.go | 163 +++++++++++------- database/models.go | 6 +- database/pgsql/complex_test.go | 6 +- database/pgsql/layer_test.go | 4 +- database/pgsql/notification_test.go | 4 +- database/pgsql/vulnerability.go | 12 +- database/pgsql/vulnerability_test.go | 22 +-- ext/vulnmdsrc/driver.go | 4 +- ext/vulnmdsrc/nvd/nvd.go | 42 ++--- ext/vulnsrc/alpine/alpine.go | 6 +- ext/vulnsrc/debian/debian.go | 30 ++-- ext/vulnsrc/debian/debian_test.go | 10 +- ext/vulnsrc/oracle/oracle.go | 29 ++-- ext/vulnsrc/oracle/oracle_test.go | 8 +- ext/vulnsrc/rhel/rhel.go | 24 ++- ext/vulnsrc/rhel/rhel_test.go | 8 +- ext/vulnsrc/ubuntu/ubuntu.go | 28 +-- ext/vulnsrc/ubuntu/ubuntu_test.go | 6 +- utils/types/priority.go => severity.go | 99 +++++++---- .../priority_test.go => severity_test.go | 21 ++- updater/updater.go | 6 +- 24 files changed, 364 insertions(+), 357 deletions(-) delete mode 100644 clair.go rename utils/types/priority.go => severity.go (56%) rename utils/types/priority_test.go => severity_test.go (59%) diff --git a/api/v1/models.go b/api/v1/models.go index bb480ac1..e3b1e1fb 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,9 +24,9 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/fernet/fernet-go" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" - "github.com/coreos/clair/utils/types" ) var log = capnslog.NewPackageLogger("github.com/coreos/clair", "v1") @@ -109,9 +109,9 @@ type Vulnerability struct { } func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { - severity := types.Priority(v.Severity) - if !severity.IsValid() { - return database.Vulnerability{}, errors.New("Invalid severity") + severity, err := clair.NewSeverity(v.Severity) + if err != nil { + return database.Vulnerability{}, err } var dbFeatures []database.FeatureVersion diff --git a/clair.go b/clair.go deleted file mode 100644 index 788ab617..00000000 --- a/clair.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package clair implements the ability to boot Clair with your own imports -// that can dynamically register additional functionality. -package clair - -import ( - "math/rand" - "os" - "os/signal" - "syscall" - "time" - - "github.com/coreos/clair/api" - "github.com/coreos/clair/api/context" - "github.com/coreos/clair/config" - "github.com/coreos/clair/database" - "github.com/coreos/clair/notifier" - "github.com/coreos/clair/updater" - "github.com/coreos/clair/utils" - "github.com/coreos/pkg/capnslog" -) - -var log = capnslog.NewPackageLogger("github.com/coreos/clair", "main") - -// Boot starts Clair. By exporting this function, anyone can import their own -// custom fetchers/updaters into their own package and then call clair.Boot. -func Boot(config *config.Config) { - rand.Seed(time.Now().UnixNano()) - st := utils.NewStopper() - - // Open database - db, err := database.Open(config.Database) - if err != nil { - log.Fatal(err) - } - defer db.Close() - - // Start notifier - st.Begin() - go notifier.Run(config.Notifier, db, st) - - // Start API - st.Begin() - go api.Run(config.API, &context.RouteContext{db, config.API}, st) - st.Begin() - go api.RunHealth(config.API, &context.RouteContext{db, config.API}, st) - - // Start updater - st.Begin() - go updater.Run(config.Updater, db, st) - - // Wait for interruption and shutdown gracefully. - waitForSignals(syscall.SIGINT, syscall.SIGTERM) - log.Info("Received interruption, gracefully stopping ...") - st.Stop() -} - -func waitForSignals(signals ...os.Signal) { - interrupts := make(chan os.Signal, 1) - signal.Notify(interrupts, signals...) - <-interrupts -} diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 74e764cf..22000c66 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -16,14 +16,23 @@ package main import ( "flag" + "math/rand" "os" + "os/signal" "runtime/pprof" "strings" + "syscall" + "time" "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" + "github.com/coreos/clair/api" + "github.com/coreos/clair/api/context" "github.com/coreos/clair/config" + "github.com/coreos/clair/database" + "github.com/coreos/clair/notifier" + "github.com/coreos/clair/updater" + "github.com/coreos/clair/utils" // Register database driver. _ "github.com/coreos/clair/database/pgsql" @@ -50,30 +59,10 @@ import ( var log = capnslog.NewPackageLogger("github.com/coreos/clair/cmd/clair", "main") -func main() { - // Parse command-line arguments - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flagConfigPath := flag.String("config", "/etc/clair/config.yaml", "Load configuration from the specified file.") - flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.") - flagLogLevel := flag.String("log-level", "info", "Define the logging level.") - flag.Parse() - // Load configuration - config, err := config.Load(*flagConfigPath) - if err != nil { - log.Fatalf("failed to load configuration: %s", err) - } - - // Initialize logging system - logLevel, err := capnslog.ParseLevel(strings.ToUpper(*flagLogLevel)) - capnslog.SetGlobalLogLevel(logLevel) - capnslog.SetFormatter(capnslog.NewPrettyFormatter(os.Stdout, false)) - - // Enable CPU Profiling if specified - if *flagCPUProfilePath != "" { - defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath)) - } - - clair.Boot(config) +func waitForSignals(signals ...os.Signal) { + interrupts := make(chan os.Signal, 1) + signal.Notify(interrupts, signals...) + <-interrupts } func startCPUProfiling(path string) *os.File { @@ -97,3 +86,62 @@ func stopCPUProfiling(f *os.File) { f.Close() log.Info("stopped CPU profiling") } + +// Boot starts Clair instance with the provided config. +func Boot(config *config.Config) { + rand.Seed(time.Now().UnixNano()) + st := utils.NewStopper() + + // Open database + db, err := database.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Start notifier + st.Begin() + go notifier.Run(config.Notifier, db, st) + + // Start API + st.Begin() + go api.Run(config.API, &context.RouteContext{db, config.API}, st) + st.Begin() + go api.RunHealth(config.API, &context.RouteContext{db, config.API}, st) + + // Start updater + st.Begin() + go updater.Run(config.Updater, db, st) + + // Wait for interruption and shutdown gracefully. + waitForSignals(syscall.SIGINT, syscall.SIGTERM) + log.Info("Received interruption, gracefully stopping ...") + st.Stop() +} + +func main() { + // Parse command-line arguments + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flagConfigPath := flag.String("config", "/etc/clair/config.yaml", "Load configuration from the specified file.") + flagCPUProfilePath := flag.String("cpu-profile", "", "Write a CPU profile to the specified file before exiting.") + flagLogLevel := flag.String("log-level", "info", "Define the logging level.") + flag.Parse() + + // Load configuration + config, err := config.Load(*flagConfigPath) + if err != nil { + log.Fatalf("failed to load configuration: %s", err) + } + + // Initialize logging system + logLevel, err := capnslog.ParseLevel(strings.ToUpper(*flagLogLevel)) + capnslog.SetGlobalLogLevel(logLevel) + capnslog.SetFormatter(capnslog.NewPrettyFormatter(os.Stdout, false)) + + // Enable CPU Profiling if specified + if *flagCPUProfilePath != "" { + defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath)) + } + + Boot(config) +} diff --git a/database/database.go b/database/database.go index 4ca13e42..0e57921f 100644 --- a/database/database.go +++ b/database/database.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,7 +29,8 @@ var ( ErrBackendException = errors.New("database: an error occured when querying the backend") // ErrInconsistent is an error that occurs when a database consistency check - // fails (ie. when an entity which is supposed to be unique is detected twice) + // fails (i.e. when an entity which is supposed to be unique is detected + // twice) ErrInconsistent = errors.New("database: inconsistent database") ) @@ -62,123 +63,151 @@ func Open(cfg config.RegistrableComponentConfig) (Datastore, error) { return driver(cfg) } -// Datastore is the interface that describes a database backend implementation. +// Datastore represents the required operations on a persistent data store for +// a Clair deployment. type Datastore interface { - // # Namespace // ListNamespaces returns the entire list of known Namespaces. ListNamespaces() ([]Namespace, error) - // # Layer // InsertLayer stores a Layer in the database. - // A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory. - // If a Parent is specified, it is expected that it has been retrieved using FindLayer. - // If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher - // than the stored one, the stored Layer should be updated. - // The function has to be idempotent, inserting a layer that already exists shouln'd return an - // error. + // + // A Layer is uniquely identified by its Name. + // The Name and EngineVersion fields are mandatory. + // If a Parent is specified, it is expected that it has been retrieved using + // FindLayer. + // If a Layer that already exists is inserted and the EngineVersion of the + // given Layer is higher than the stored one, the stored Layer should be + // updated. + // The function has to be idempotent, inserting a layer that already exists + // shouldn't return an error. InsertLayer(Layer) error // FindLayer retrieves a Layer from the database. - // withFeatures specifies whether the Features field should be filled. When withVulnerabilities is - // true, the Features field should be filled and their AffectedBy fields should contain every - // vulnerabilities that affect them. + // + // When `withFeatures` is true, the Features field should be filled. + // When `withVulnerabilities` is true, the Features field should be filled + // and their AffectedBy fields should contain every vulnerabilities that + // affect them. FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error) - // DeleteLayer deletes a Layer from the database and every layers that are based on it, - // recursively. + // DeleteLayer deletes a Layer from the database and every layers that are + // based on it, recursively. DeleteLayer(name string) error - // # Vulnerability - // ListVulnerabilities returns the list of vulnerabilies of a certain Namespace. + // ListVulnerabilities returns the list of vulnerabilities of a particular + // Namespace. + // // The Limit and page parameters are used to paginate the return list. - // The first given page should be 0. The function will then return the next available page. - // If there is no more page, -1 has to be returned. + // The first given page should be 0. + // The function should return the next available page. If there are no more + // pages, -1 has to be returned. ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) - // InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if - // necessary. A vulnerability is uniquely identified by its Namespace and its Name. - // The FixedIn field may only contain a partial list of Features that are affected by the - // Vulnerability, along with the version in which the vulnerability is fixed. It is the - // responsibility of the implementation to update the list properly. A version equals to - // types.MinVersion means that the given Feature is not being affected by the Vulnerability at - // all and thus, should be removed from the list. It is important that Features should be unique - // in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as - // a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and - // Features are Namespaced (i.e. specific to one operating system). - // Each vulnerability insertion or update has to create a Notification that will contain the - // old and the updated Vulnerability, unless createNotification equals to true. + // InsertVulnerabilities stores the given Vulnerabilities in the database, + // updating them if necessary. + // + // A vulnerability is uniquely identified by its Namespace and its Name. + // The FixedIn field may only contain a partial list of Features that are + // affected by the Vulnerability, along with the version in which the + // vulnerability is fixed. It is the responsibility of the implementation to + // update the list properly. + // A version equals to versionfmt.MinVersion means that the given Feature is + // not being affected by the Vulnerability at all and thus, should be removed + // from the list. + // It is important that Features should be unique in the FixedIn list. For + // example, it doesn't make sense to have two `openssl` Feature listed as a + // Vulnerability can only be fixed in one Version. This is true because + // Vulnerabilities and Features are namespaced (i.e. specific to one + // operating system). + // Each vulnerability insertion or update has to create a Notification that + // will contain the old and the updated Vulnerability, unless + // createNotification equals to true. InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error - // FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list. + // FindVulnerability retrieves a Vulnerability from the database, including + // the FixedIn list. FindVulnerability(namespaceName, name string) (Vulnerability, error) // DeleteVulnerability removes a Vulnerability from the database. + // // It has to create a Notification that will contain the old Vulnerability. DeleteVulnerability(namespaceName, name string) error - // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to - // the specified Vulnerability in the database. - // It has has to create a Notification that will contain the old and the updated Vulnerability. + // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions + // of existing ones to the specified Vulnerability in the database. + // + // It has has to create a Notification that will contain the old and the + // updated Vulnerability. InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error - // DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the - // database. It can be used to store the fact that a Vulnerability no longer affects the given - // Feature in any Version. + // DeleteVulnerabilityFix removes a FixedIn Feature from the specified + // Vulnerability in the database. It can be used to store the fact that a + // Vulnerability no longer affects the given Feature in any Version. + // // It has has to create a Notification that will contain the old and the updated Vulnerability. DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error - // # Notification - // GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a - // Notification that should be handled. The renotify interval defines how much time after being - // marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should - // be returned again by this function. A Notification for which there is a valid Lock with the - // same Name should not be returned. + // GetAvailableNotification returns the Name, Created, Notified and Deleted + // fields of a Notification that should be handled. + // + // The renotify interval defines how much time after being marked as Notified + // by SetNotificationNotified, a Notification that hasn't been deleted should + // be returned again by this function. + // A Notification for which there is a valid Lock with the same Name should + // not be returned. GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error) - // GetNotification returns a Notification, including its OldVulnerability and NewVulnerability - // fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with - // every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion). - // The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first - // given page should be VulnerabilityNotificationFirstPage. The function will then return the next - // availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned. + // GetNotification returns a Notification, including its OldVulnerability and + // NewVulnerability fields. + // + // On these Vulnerabilities, LayersIntroducingVulnerability should be filled + // with every Layer that introduces the Vulnerability (i.e. adds at least one + // affected FeatureVersion). + // The Limit and page parameters are used to paginate + // LayersIntroducingVulnerability. The first given page should be + // VulnerabilityNotificationFirstPage. The function will then return the next + // available page. If there is no more page, NoVulnerabilityNotificationPage + // has to be returned. GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) - // SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for - // GetAvailableNotification, until the renotify duration is elapsed. + // SetNotificationNotified marks a Notification as notified and thus, makes + // it unavailable for GetAvailableNotification, until the renotify duration + // is elapsed. SetNotificationNotified(name string) error - // DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for - // GetAvailableNotification. + // DeleteNotification marks a Notification as deleted, and thus, makes it + // unavailable for GetAvailableNotification. DeleteNotification(name string) error - // # Key/Value // InsertKeyValue stores or updates a simple key/value pair in the database. InsertKeyValue(key, value string) error // GetKeyValue retrieves a value from the database from the given key. + // // It returns an empty string if there is no such key. GetKeyValue(key string) (string, error) - // # Lock - // Lock creates or renew a Lock in the database with the given name, owner and duration. - // After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus, - // let other users create a Lock with the same name. However, the owner can renew its Lock by - // setting renew to true. Lock should not block, it should instead returns whether the Lock has - // been successfully acquired/renewed. If it's the case, the expiration time of that Lock is - // returned as well. + // Lock creates or renew a Lock in the database with the given name, owner + // and duration. + // + // After the specified duration, the Lock expires by itself if it hasn't been + // unlocked, and thus, let other users create a Lock with the same name. + // However, the owner can renew its Lock by setting renew to true. + // Lock should not block, it should instead returns whether the Lock has been + // successfully acquired/renewed. If it's the case, the expiration time of + // that Lock is returned as well. Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time) // Unlock releases an existing Lock. Unlock(name, owner string) - // FindLock returns the owner of a Lock specified by the name, and its experation time if it - // exists. + // FindLock returns the owner of a Lock specified by the name, and its + // expiration time if it exists. FindLock(name string) (string, time.Time, error) - // # Miscellaneous // Ping returns the health status of the database. Ping() bool - // Close closes the database and free any allocated resource. + // Close closes the database and frees any allocated resource. Close() } diff --git a/database/models.go b/database/models.go index 898d2c9a..86a14096 100644 --- a/database/models.go +++ b/database/models.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ import ( "encoding/json" "time" - "github.com/coreos/clair/utils/types" + "github.com/coreos/clair" ) // ID is only meant to be used by database implementations and should never be used for anything else. @@ -70,7 +70,7 @@ type Vulnerability struct { Description string Link string - Severity types.Priority + Severity clair.Severity Metadata MetadataMap diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index b49903fb..407d8b2a 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,10 +26,10 @@ import ( "github.com/pborman/uuid" "github.com/stretchr/testify/assert" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/utils" - "github.com/coreos/clair/utils/types" ) const ( @@ -93,7 +93,7 @@ func TestRaceAffects(t *testing.T) { Version: strconv.Itoa(version), }, }, - Severity: types.Unknown, + Severity: clair.Unknown, } vulnerabilities[version] = append(vulnerabilities[version], vulnerability) diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index 3aa6e622..ad6ae1d7 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -20,10 +20,10 @@ import ( "github.com/stretchr/testify/assert" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) func TestFindLayer(t *testing.T) { @@ -91,7 +91,7 @@ func TestFindLayer(t *testing.T) { if assert.Len(t, featureVersion.AffectedBy, 1) { assert.Equal(t, "debian:7", featureVersion.AffectedBy[0].Namespace.Name) assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name) - assert.Equal(t, types.High, featureVersion.AffectedBy[0].Severity) + assert.Equal(t, clair.High, featureVersion.AffectedBy[0].Severity) assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description) assert.Equal(t, "http://google.com/#q=CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Link) assert.Equal(t, "2.0", featureVersion.AffectedBy[0].FixedBy) diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 5dab2c22..4e53e217 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -20,11 +20,11 @@ import ( "github.com/stretchr/testify/assert" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) func TestNotification(t *testing.T) { @@ -169,7 +169,7 @@ func TestNotification(t *testing.T) { // Update a vulnerability and ensure that the old/new vulnerabilities are correct. v1b := v1 - v1b.Severity = types.High + v1b.Severity = clair.High v1b.FixedIn = []database.FeatureVersion{ { Feature: f1, diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 17461374..3d791af4 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -17,7 +17,6 @@ package pgsql import ( "database/sql" "encoding/json" - "fmt" "reflect" "time" @@ -197,11 +196,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on if vulnerability.Name == "" || vulnerability.Namespace.Name == "" { return commonerr.NewBadRequestError("insertVulnerability needs at least the Name and the Namespace") } - if !onlyFixedIn && !vulnerability.Severity.IsValid() { - msg := fmt.Sprintf("could not insert a vulnerability that has an invalid Severity: %s", vulnerability.Severity) - log.Warning(msg) - return commonerr.NewBadRequestError(msg) - } + for i := 0; i < len(vulnerability.FixedIn); i++ { fifv := &vulnerability.FixedIn[i] @@ -271,8 +266,9 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on return handleError("removeVulnerability", err) } } else { - // The vulnerability is new, we don't want to have any types.MinVersion as they are only used - // for diffing existing vulnerabilities. + // The vulnerability is new, we don't want to have any + // versionfmt.MinVersion as they are only used for diffing existing + // vulnerabilities. var fixedIn []database.FeatureVersion for _, fv := range vulnerability.FixedIn { if fv.Version != versionfmt.MinVersion { diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index 788856ba..006a7fda 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -20,11 +20,11 @@ import ( "github.com/stretchr/testify/assert" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) func TestFindVulnerability(t *testing.T) { @@ -44,7 +44,7 @@ func TestFindVulnerability(t *testing.T) { Name: "CVE-OPENSSL-1-DEB7", Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", - Severity: types.High, + Severity: clair.High, Namespace: database.Namespace{ Name: "debian:7", VersionFormat: dpkg.ParserName, @@ -74,7 +74,7 @@ func TestFindVulnerability(t *testing.T) { Name: "debian:7", VersionFormat: dpkg.ParserName, }, - Severity: types.Unknown, + Severity: clair.Unknown, } v2f, err := datastore.FindVulnerability("debian:7", "CVE-NOPE") @@ -180,30 +180,24 @@ func TestInsertVulnerability(t *testing.T) { Name: "", Namespace: n1, FixedIn: []database.FeatureVersion{f1}, - Severity: types.Unknown, + Severity: clair.Unknown, }, { Name: "TestInsertVulnerability0", Namespace: database.Namespace{}, FixedIn: []database.FeatureVersion{f1}, - Severity: types.Unknown, + Severity: clair.Unknown, }, { Name: "TestInsertVulnerability0-", Namespace: database.Namespace{}, FixedIn: []database.FeatureVersion{f1}, }, - { - Name: "TestInsertVulnerability0", - Namespace: n1, - FixedIn: []database.FeatureVersion{f1}, - Severity: types.Priority(""), - }, { Name: "TestInsertVulnerability0", Namespace: n1, FixedIn: []database.FeatureVersion{f2}, - Severity: types.Unknown, + Severity: clair.Unknown, }, } { err := datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true) @@ -223,7 +217,7 @@ func TestInsertVulnerability(t *testing.T) { Name: "TestInsertVulnerability1", Namespace: n1, FixedIn: []database.FeatureVersion{f1, f3, f6, f7}, - Severity: types.Low, + Severity: clair.Low, Description: "TestInsertVulnerabilityDescription1", Link: "TestInsertVulnerabilityLink1", Metadata: v1meta, @@ -239,7 +233,7 @@ func TestInsertVulnerability(t *testing.T) { // Update vulnerability. v1.Description = "TestInsertVulnerabilityLink2" v1.Link = "TestInsertVulnerabilityLink2" - v1.Severity = types.High + v1.Severity = clair.High // Update f3 in f4, add fixed in f5, add fixed in f6 which already exists, // removes fixed in f7 by adding f8 which is f7 but with MinVersion, and // add fixed by f5 a second time (duplicated). diff --git a/ext/vulnmdsrc/driver.go b/ext/vulnmdsrc/driver.go index 06d5cb78..4de7f358 100644 --- a/ext/vulnmdsrc/driver.go +++ b/ext/vulnmdsrc/driver.go @@ -19,8 +19,8 @@ package vulnmdsrc import ( "sync" + "github.com/coreos/clair" "github.com/coreos/clair/database" - "github.com/coreos/clair/utils/types" ) var ( @@ -29,7 +29,7 @@ var ( ) // AppendFunc is the type of a callback provided to an Appender. -type AppendFunc func(metadataKey string, metadata interface{}, severity types.Priority) +type AppendFunc func(metadataKey string, metadata interface{}, severity clair.Severity) // Appender represents anything that can fetch vulnerability metadata and // append it to a Vulnerability. diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 1b98f609..1c6b7559 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -28,15 +28,14 @@ import ( "os" "strconv" "strings" - "sync" "time" "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) const ( @@ -52,7 +51,6 @@ type appender struct { localPath string dataFeedHashes map[string]string metadata map[string]NVDMetadata - sync.Mutex } type NVDMetadata struct { @@ -69,9 +67,6 @@ func init() { } func (a *appender) BuildCache(datastore database.Datastore) error { - a.Lock() - defer a.Unlock() - var err error a.metadata = make(map[string]NVDMetadata) @@ -115,27 +110,18 @@ func (a *appender) BuildCache(datastore database.Datastore) error { } func (a *appender) Append(vulnName string, appendFunc vulnmdsrc.AppendFunc) error { - a.Lock() - defer a.Unlock() - if nvdMetadata, ok := a.metadata[vulnName]; ok { - appendFunc(appenderName, nvdMetadata, scoreToPriority(nvdMetadata.CVSSv2.Score)) + appendFunc(appenderName, nvdMetadata, SeverityFromCVSS(nvdMetadata.CVSSv2.Score)) } return nil } func (a *appender) PurgeCache() { - a.Lock() - defer a.Unlock() - a.metadata = nil } func (a *appender) Clean() { - a.Lock() - defer a.Unlock() - os.RemoveAll(a.localPath) } @@ -232,23 +218,23 @@ func getHashFromMetaURL(metaURL string) (string, error) { return "", errors.New("invalid .meta file format") } -// scoreToPriority converts the CVSS Score (0.0 - 10.0) into user-friendy -// types.Priority following the qualitative rating scale available in the -// CVSS v3.0 specification (https://www.first.org/cvss/specification-document), -// Table 14. The Negligible level is set for CVSS scores between [0, 1), -// replacing the specified None level, originally used for a score of 0. -func scoreToPriority(score float64) types.Priority { +// SeverityFromCVSS converts the CVSS Score (0.0 - 10.0) into a clair.Severity +// following the qualitative rating scale available in the CVSS v3.0 +// specification (https://www.first.org/cvss/specification-document), Table 14. +// The Negligible level is set for CVSS scores between [0, 1), replacing the +// specified None level, originally used for a score of 0. +func SeverityFromCVSS(score float64) clair.Severity { switch { case score < 1.0: - return types.Negligible + return clair.Negligible case score < 3.9: - return types.Low + return clair.Low case score < 6.9: - return types.Medium + return clair.Medium case score < 8.9: - return types.High + return clair.High case score <= 10: - return types.Critical + return clair.Critical } - return types.Unknown + return clair.Unknown } diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index 20a59033..1ddf2aa5 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -27,13 +27,13 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - "github.com/coreos/clair/utils/types" ) const ( @@ -225,7 +225,7 @@ func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { vulns = append(vulns, database.Vulnerability{ Name: fix, - Severity: types.Unknown, + Severity: clair.Unknown, Link: nvdURLPrefix + fix, FixedIn: []database.FeatureVersion{ { @@ -279,7 +279,7 @@ func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { for _, vulnStr := range vulnStrs { var vuln database.Vulnerability - vuln.Severity = types.Unknown + vuln.Severity = clair.Unknown vuln.Name = vulnStr vuln.Link = nvdURLPrefix + vulnStr vuln.FixedIn = []database.FeatureVersion{ diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index a0388e72..26860b58 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -27,12 +27,12 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) const ( @@ -158,17 +158,17 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, vulnerability = &database.Vulnerability{ Name: vulnName, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), - Severity: types.Unknown, + Severity: clair.Unknown, Description: vulnNode.Description, } } // Set the priority of the vulnerability. // In the JSON, a vulnerability has one urgency per package it affects. - // The highest urgency should be the one set. - urgency := urgencyToSeverity(releaseNode.Urgency) - if urgency.Compare(vulnerability.Severity) > 0 { - vulnerability.Severity = urgency + severity := SeverityFromUrgency(releaseNode.Urgency) + if severity.Compare(vulnerability.Severity) > 0 { + // The highest urgency should be the one set. + vulnerability.Severity = severity } // Determine the version of the package the vulnerability affects. @@ -219,39 +219,41 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, return } -func urgencyToSeverity(urgency string) types.Priority { +// SeverityFromUrgency converts the urgency scale used by the Debian Security +// Bug Tracker into a clair.Severity. +func SeverityFromUrgency(urgency string) clair.Severity { switch urgency { case "not yet assigned": - return types.Unknown + return clair.Unknown case "end-of-life": fallthrough case "unimportant": - return types.Negligible + return clair.Negligible case "low": fallthrough case "low*": fallthrough case "low**": - return types.Low + return clair.Low case "medium": fallthrough case "medium*": fallthrough case "medium**": - return types.Medium + return clair.Medium case "high": fallthrough case "high*": fallthrough case "high**": - return types.High + return clair.High default: - log.Warningf("could not determine vulnerability priority from: %s", urgency) - return types.Unknown + log.Warningf("could not determine vulnerability severity from: %s", urgency) + return clair.Unknown } } diff --git a/ext/vulnsrc/debian/debian_test.go b/ext/vulnsrc/debian/debian_test.go index 37cb0fe0..3a4c0d70 100644 --- a/ext/vulnsrc/debian/debian_test.go +++ b/ext/vulnsrc/debian/debian_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,10 +20,10 @@ import ( "runtime" "testing" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -37,7 +37,7 @@ func TestDebianParser(t *testing.T) { for _, vulnerability := range response.Vulnerabilities { if vulnerability.Name == "CVE-2015-1323" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2015-1323", vulnerability.Link) - assert.Equal(t, types.Low, vulnerability.Severity) + assert.Equal(t, clair.Low, vulnerability.Severity) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -68,7 +68,7 @@ func TestDebianParser(t *testing.T) { } } else if vulnerability.Name == "CVE-2003-0779" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link) - assert.Equal(t, types.High, vulnerability.Severity) + assert.Equal(t, clair.High, vulnerability.Severity) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -109,7 +109,7 @@ func TestDebianParser(t *testing.T) { } } else if vulnerability.Name == "CVE-2013-2685" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2013-2685", vulnerability.Link) - assert.Equal(t, types.Negligible, vulnerability.Severity) + assert.Equal(t, clair.Negligible, vulnerability.Severity) assert.Equal(t, "Un-affected packages.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index 569752ad..e82d18f9 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -25,13 +25,14 @@ import ( "strconv" "strings" + "github.com/coreos/pkg/capnslog" + + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" - "github.com/coreos/pkg/capnslog" ) const ( @@ -172,7 +173,7 @@ func parseELSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, vulnerability := database.Vulnerability{ Name: name(definition), Link: link(definition), - Severity: priority(definition), + Severity: severity(definition), Description: description(definition), } for _, p := range pkgs { @@ -336,24 +337,20 @@ func link(def definition) (link string) { return } -func priority(def definition) types.Priority { - // Parse the priority. - priority := strings.ToLower(def.Severity) - - // Normalize the priority. - switch priority { +func severity(def definition) clair.Severity { + switch strings.ToLower(def.Severity) { case "n/a": - return types.Negligible + return clair.Negligible case "low": - return types.Low + return clair.Low case "moderate": - return types.Medium + return clair.Medium case "important": - return types.High + return clair.High case "critical": - return types.Critical + return clair.Critical default: - log.Warningf("could not determine vulnerability priority from: %s.", priority) - return types.Unknown + log.Warningf("could not determine vulnerability severity from: %s.", def.Severity) + return clair.Unknown } } diff --git a/ext/vulnsrc/oracle/oracle_test.go b/ext/vulnsrc/oracle/oracle_test.go index d23ef866..2a33207c 100644 --- a/ext/vulnsrc/oracle/oracle_test.go +++ b/ext/vulnsrc/oracle/oracle_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ import ( "runtime" "testing" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -38,7 +38,7 @@ func TestOracleParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "ELSA-2015-1193", vulnerabilities[0].Name) assert.Equal(t, "http://linux.oracle.com/errata/ELSA-2015-1193.html", vulnerabilities[0].Link) - assert.Equal(t, types.Medium, vulnerabilities[0].Severity) + assert.Equal(t, clair.Medium, vulnerabilities[0].Severity) assert.Equal(t, ` [3.1.1-7] Resolves: rhbz#1217104 CVE-2015-0252 `, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -86,7 +86,7 @@ func TestOracleParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "ELSA-2015-1207", vulnerabilities[0].Name) assert.Equal(t, "http://linux.oracle.com/errata/ELSA-2015-1207.html", vulnerabilities[0].Link) - assert.Equal(t, types.Critical, vulnerabilities[0].Severity) + assert.Equal(t, clair.Critical, vulnerabilities[0].Severity) assert.Equal(t, ` [38.1.0-1.0.1.el7_1] - Add firefox-oracle-default-prefs.js and remove the corresponding Red Hat file [38.1.0-1] - Update to 38.1.0 ESR [38.0.1-2] - Fixed rhbz#1222807 by removing preun section `, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ { diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index f1d88986..9dd66bcc 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -27,12 +27,12 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils/types" ) const ( @@ -175,7 +175,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, vulnerability := database.Vulnerability{ Name: name(definition), Link: link(definition), - Severity: priority(definition), + Severity: severity(definition), Description: description(definition), } for _, p := range pkgs { @@ -344,22 +344,18 @@ func link(def definition) (link string) { return } -func priority(def definition) types.Priority { - // Parse the priority. - priority := strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) - - // Normalize the priority. - switch priority { +func severity(def definition) clair.Severity { + switch strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) { case "Low": - return types.Low + return clair.Low case "Moderate": - return types.Medium + return clair.Medium case "Important": - return types.High + return clair.High case "Critical": - return types.Critical + return clair.Critical default: - log.Warning("could not determine vulnerability priority from: %s.", priority) - return types.Unknown + log.Warning("could not determine vulnerability severity from: %s.", def.Title) + return clair.Unknown } } diff --git a/ext/vulnsrc/rhel/rhel_test.go b/ext/vulnsrc/rhel/rhel_test.go index 2b87eec9..7aa13938 100644 --- a/ext/vulnsrc/rhel/rhel_test.go +++ b/ext/vulnsrc/rhel/rhel_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,9 +20,9 @@ import ( "runtime" "testing" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/rpm" - "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -36,7 +36,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].Name) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link) - assert.Equal(t, types.Medium, vulnerabilities[0].Severity) + assert.Equal(t, clair.Medium, vulnerabilities[0].Severity) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -83,7 +83,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].Name) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link) - assert.Equal(t, types.Critical, vulnerabilities[0].Severity) + assert.Equal(t, clair.Critical, vulnerabilities[0].Severity) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index 0a74db4a..d74b6122 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -29,13 +29,13 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/utils" - "github.com/coreos/clair/utils/types" ) const ( @@ -301,7 +301,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability priority = priority[:strings.Index(priority, " ")] } - vulnerability.Severity = ubuntuPriorityToSeverity(priority) + vulnerability.Severity = SeverityFromPriority(priority) continue } @@ -388,28 +388,30 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability // If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown if vulnerability.Severity == "" { - vulnerability.Severity = types.Unknown + vulnerability.Severity = clair.Unknown } return } -func ubuntuPriorityToSeverity(priority string) types.Priority { +// SeverityFromPriority converts an priority from the Ubuntu CVE Tracker into +// a clair.Severity. +func SeverityFromPriority(priority string) clair.Severity { switch priority { case "untriaged": - return types.Unknown + return clair.Unknown case "negligible": - return types.Negligible + return clair.Negligible case "low": - return types.Low + return clair.Low case "medium": - return types.Medium + return clair.Medium case "high": - return types.High + return clair.High case "critical": - return types.Critical + return clair.Critical + default: + log.Warning("could not determine a vulnerability severity from: %s", priority) + return clair.Unknown } - - log.Warning("Could not determine a vulnerability priority from: %s", priority) - return types.Unknown } diff --git a/ext/vulnsrc/ubuntu/ubuntu_test.go b/ext/vulnsrc/ubuntu/ubuntu_test.go index e1c84637..9ebeba7a 100644 --- a/ext/vulnsrc/ubuntu/ubuntu_test.go +++ b/ext/vulnsrc/ubuntu/ubuntu_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,9 +22,9 @@ import ( "github.com/stretchr/testify/assert" + "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" - "github.com/coreos/clair/utils/types" ) func TestUbuntuParser(t *testing.T) { @@ -37,7 +37,7 @@ func TestUbuntuParser(t *testing.T) { vulnerability, unknownReleases, err := parseUbuntuCVE(testData) if assert.Nil(t, err) { assert.Equal(t, "CVE-2015-4471", vulnerability.Name) - assert.Equal(t, types.Medium, vulnerability.Severity) + assert.Equal(t, clair.Medium, vulnerability.Severity) assert.Equal(t, "Off-by-one error in the lzxd_decompress function in lzxd.c in libmspack before 0.5 allows remote attackers to cause a denial of service (buffer under-read and application crash) via a crafted CAB archive.", vulnerability.Description) // Unknown release (line 28) diff --git a/utils/types/priority.go b/severity.go similarity index 56% rename from utils/types/priority.go rename to severity.go index aac56d40..46bb9927 100644 --- a/utils/types/priority.go +++ b/severity.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,80 +12,104 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package types defines useful types that are used in database models. -package types +package clair import ( "database/sql/driver" "errors" - "fmt" + "strings" ) -// Priority defines a vulnerability priority -type Priority string +var ( + // ErrFailedToParseSeverity is the error returned when a severity could not + // be parsed from a string. + ErrFailedToParseSeverity = errors.New("failed to parse Severity from input") +) + +// Severity defines a standard scale for measuring the severity of a +// vulnerability. +type Severity string const ( // Unknown is either a security problem that has not been // assigned to a priority yet or a priority that our system - // did not recognize - Unknown Priority = "Unknown" + // did not recognize. + Unknown Severity = "Unknown" + // Negligible is technically a security problem, but is // only theoretical in nature, requires a very special // situation, has almost no install base, or does no real // damage. These tend not to get backport from upstreams, // and will likely not be included in security updates unless // there is an easy fix and some other issue causes an update. - Negligible Priority = "Negligible" + Negligible Severity = "Negligible" + // Low is a security problem, but is hard to // exploit due to environment, requires a user-assisted // attack, a small install base, or does very little damage. // These tend to be included in security updates only when // higher priority issues require an update, or if many // low priority issues have built up. - Low Priority = "Low" + Low Severity = "Low" + // Medium is a real security problem, and is exploitable // for many people. Includes network daemon denial of service // attacks, cross-site scripting, and gaining user privileges. // Updates should be made soon for this priority of issue. - Medium Priority = "Medium" + Medium Severity = "Medium" + // High is a real problem, exploitable for many people in a default // installation. Includes serious remote denial of services, // local root privilege escalations, or data loss. - High Priority = "High" + High Severity = "High" + // Critical is a world-burning problem, exploitable for nearly all people // in a default installation of Linux. Includes remote root // privilege escalations, or massive data loss. - Critical Priority = "Critical" + Critical Severity = "Critical" + // Defcon1 is a Critical problem which has been manually highlighted by // the team. It requires an immediate attention. - Defcon1 Priority = "Defcon1" + Defcon1 Severity = "Defcon1" ) -// Priorities lists all known priorities, ordered from lower to higher -var Priorities = []Priority{Unknown, Negligible, Low, Medium, High, Critical, Defcon1} +// Severities lists all known severities, ordered from lowest to highest. +var Severities = []Severity{ + Unknown, + Negligible, + Low, + Medium, + High, + Critical, + Defcon1, +} -// IsValid determines if the priority is a valid one -func (p Priority) IsValid() bool { - for _, pp := range Priorities { - if p == pp { - return true +// NewSeverity attempts to parse a string into a standard Severity value. +func NewSeverity(s string) (Severity, error) { + for _, ss := range Severities { + if strings.EqualFold(s, string(ss)) { + return ss, nil } } - return false + return Unknown, ErrFailedToParseSeverity } -// Compare compares two priorities -func (p Priority) Compare(p2 Priority) int { +// Compare determines the equality of two severities. +// +// If the severities are equal, returns 0. +// If the receiever is less, returns -1. +// If the receiver is greater, returns 1. +func (s Severity) Compare(s2 Severity) int { var i1, i2 int - for i1 = 0; i1 < len(Priorities); i1 = i1 + 1 { - if p == Priorities[i1] { + for i1 = 0; i1 < len(Severities); i1 = i1 + 1 { + if s == Severities[i1] { break } } - for i2 = 0; i2 < len(Priorities); i2 = i2 + 1 { - if p2 == Priorities[i2] { + for i2 = 0; i2 < len(Severities); i2 = i2 + 1 { + if s2 == Severities[i2] { break } } @@ -93,18 +117,23 @@ func (p Priority) Compare(p2 Priority) int { return i1 - i2 } -func (p *Priority) Scan(value interface{}) error { +// Scan implements the database/sql.Scanner interface. +func (s *Severity) Scan(value interface{}) error { val, ok := value.([]byte) if !ok { - return errors.New("could not scan a Priority from a non-string input") + return errors.New("could not scan a Severity from a non-string input") } - *p = Priority(string(val)) - if !p.IsValid() { - return fmt.Errorf("could not scan an invalid Priority (%v)", p) + + var err error + *s, err = NewSeverity(string(val)) + if err != nil { + return err } + return nil } -func (p *Priority) Value() (driver.Value, error) { - return string(*p), nil +// Value implements the database/sql/driver.Valuer interface. +func (s Severity) Value() (driver.Value, error) { + return string(s), nil } diff --git a/utils/types/priority_test.go b/severity_test.go similarity index 59% rename from utils/types/priority_test.go rename to severity_test.go index 15558d5a..6bb180ef 100644 --- a/utils/types/priority_test.go +++ b/severity_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package types +package clair import ( "testing" @@ -20,13 +20,16 @@ import ( "github.com/stretchr/testify/assert" ) -func TestComparePriority(t *testing.T) { - assert.Equal(t, Medium.Compare(Medium), 0, "Priority comparison failed") - assert.True(t, Medium.Compare(High) < 0, "Priority comparison failed") - assert.True(t, Critical.Compare(Low) > 0, "Priority comparison failed") +func TestCompareSeverity(t *testing.T) { + assert.Equal(t, Medium.Compare(Medium), 0, "Severity comparison failed") + assert.True(t, Medium.Compare(High) < 0, "Severity comparison failed") + assert.True(t, Critical.Compare(Low) > 0, "Severity comparison failed") } -func TestIsValid(t *testing.T) { - assert.False(t, Priority("Test").IsValid()) - assert.True(t, Unknown.IsValid()) +func TestParseSeverity(t *testing.T) { + _, err := NewSeverity("Test") + assert.Equal(t, ErrFailedToParseSeverity, err) + + _, err = NewSeverity("Unknown") + assert.Nil(t, err) } diff --git a/updater/updater.go b/updater/updater.go index a7b4e558..0ae34970 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -27,12 +27,12 @@ import ( "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" + "github.com/coreos/clair" "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/utils" - "github.com/coreos/clair/utils/types" ) const ( @@ -316,7 +316,7 @@ type lockableVulnerability struct { sync.Mutex } -func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity types.Priority) { +func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity clair.Severity) { lv.Lock() defer lv.Unlock() @@ -329,7 +329,7 @@ func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interfa lv.Metadata[metadataKey] = metadata // If necessary, provide a severity for the vulnerability. - if lv.Severity == "" || lv.Severity == types.Unknown { + if lv.Severity == clair.Unknown { lv.Severity = severity } } From e7f72ef5adca478985545dfafd9d0011e098367c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 20:33:20 -0500 Subject: [PATCH 14/23] utils: rm prometheus.go --- api/context/context.go | 8 +++++--- database/pgsql/pgsql.go | 5 +++-- notifier/notifier.go | 3 ++- utils/prometheus.go | 13 ------------- 4 files changed, 10 insertions(+), 19 deletions(-) delete mode 100644 utils/prometheus.go diff --git a/api/context/context.go b/api/context/context.go index 77fa388b..d7df44f2 100644 --- a/api/context/context.go +++ b/api/context/context.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" - "github.com/coreos/clair/utils" ) var ( @@ -52,7 +51,10 @@ func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle { if status == 0 { statusStr = "???" } - utils.PrometheusObserveTimeMilliseconds(promResponseDurationMilliseconds.WithLabelValues(route, statusStr), start) + + promResponseDurationMilliseconds. + WithLabelValues(route, statusStr). + Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)) log.Infof("%s \"%s %s\" %s (%s)", r.RemoteAddr, r.Method, r.RequestURI, statusStr, time.Since(start)) } diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index f1d8d7d6..4abd29ee 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -35,7 +35,6 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/database/pgsql/migrations" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" ) var ( @@ -300,5 +299,7 @@ func isErrUniqueViolation(err error) bool { } func observeQueryTime(query, subquery string, start time.Time) { - utils.PrometheusObserveTimeMilliseconds(promQueryDurationMilliseconds.WithLabelValues(query, subquery), start) + promQueryDurationMilliseconds. + WithLabelValues(query, subquery). + Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)) } diff --git a/notifier/notifier.go b/notifier/notifier.go index fa80482d..ad3619fa 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -96,8 +96,9 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u go func() { success, interrupted := handleTask(*notification, stopper, config.Attempts) if success { - utils.PrometheusObserveTimeMilliseconds(promNotifierLatencyMilliseconds, notification.Created) datastore.SetNotificationNotified(notification.Name) + + promNotifierLatencyMilliseconds.Observe(float64(time.Since(notification.Created).Nanoseconds()) / float64(time.Millisecond)) } if interrupted { running = false diff --git a/utils/prometheus.go b/utils/prometheus.go deleted file mode 100644 index decd1a9e..00000000 --- a/utils/prometheus.go +++ /dev/null @@ -1,13 +0,0 @@ -package utils - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" -) - -// PrometheusObserveTimeMilliseconds observes the elapsed time since start, in milliseconds, -// on the specified Prometheus Histogram. -func PrometheusObserveTimeMilliseconds(h prometheus.Histogram, start time.Time) { - h.Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)) -} From c2f4a4406812f85658615305f52329ec688975e5 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 21:22:20 -0500 Subject: [PATCH 15/23] utils: rm exec.go This change also adds a dependency check at startup, rather than runtime. --- cmd/clair/main.go | 9 +++++++++ ext/featurefmt/rpm/rpm.go | 8 +++----- ext/vulnsrc/alpine/alpine.go | 17 ++++++++++------ ext/vulnsrc/ubuntu/ubuntu.go | 20 +++++++++++++----- utils/exec.go | 39 ------------------------------------ 5 files changed, 38 insertions(+), 55 deletions(-) delete mode 100644 utils/exec.go diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 22000c66..750c5fc6 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -18,6 +18,7 @@ import ( "flag" "math/rand" "os" + "os/exec" "os/signal" "runtime/pprof" "strings" @@ -127,6 +128,14 @@ func main() { flagLogLevel := flag.String("log-level", "info", "Define the logging level.") flag.Parse() + // Check for dependencies. + for _, bin := range []string{"git", "bzr", "rpm", "xz"} { + _, err := exec.LookPath(bin) + if err != nil { + log.Fatalf("failed to find dependency: %s", bin) + } + } + // Load configuration config, err := config.Load(*flagConfigPath) if err != nil { diff --git a/ext/featurefmt/rpm/rpm.go b/ext/featurefmt/rpm/rpm.go index ffcdaaf7..b1e9b6e8 100644 --- a/ext/featurefmt/rpm/rpm.go +++ b/ext/featurefmt/rpm/rpm.go @@ -19,6 +19,7 @@ import ( "bufio" "io/ioutil" "os" + "os/exec" "strings" "github.com/coreos/pkg/capnslog" @@ -29,7 +30,6 @@ import ( "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" - "github.com/coreos/clair/utils" ) var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/featurefmt/rpm") @@ -63,10 +63,8 @@ func (l lister) ListFeatures(files tarutil.FilesMap) ([]database.FeatureVersion, return []database.FeatureVersion{}, commonerr.ErrFilesystem } - // Query RPM - // We actually extract binary package names instead of source package names here because RHSA refers to package names - // In the dpkg system, we extract the source instead - out, err := utils.Exec(tmpDir, "rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n") + // Extract binary package names because RHSA refers to binary package names. + out, err := exec.Command("rpm", "--dbpath", tmpDir, "-qa", "--qf", "%{NAME} %{EPOCH}:%{VERSION}-%{RELEASE}\n").CombinedOutput() if err != nil { log.Errorf("could not query RPM: %s. output: %s", err, string(out)) // Do not bubble up because we probably won't be able to fix it, diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index 1ddf2aa5..b5391ae5 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -21,6 +21,7 @@ import ( "io" "io/ioutil" "os" + "os/exec" "strings" "gopkg.in/yaml.v2" @@ -33,7 +34,6 @@ import ( "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" ) const ( @@ -169,20 +169,25 @@ func (u *updater) pullRepository() (commit string, err error) { return "", vulnsrc.ErrFilesystem } - if out, err := utils.Exec(u.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil { + cmd := exec.Command("git", "clone", secdbGitURL, ".") + cmd.Dir = u.repositoryLocalPath + if out, err := cmd.CombinedOutput(); err != nil { u.Clean() log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out) return "", commonerr.ErrCouldNotDownload } } else { - // The repository exists and it needs to be refreshed via a pull. - _, err := utils.Exec(u.repositoryLocalPath, "git", "pull") - if err != nil { + // The repository already exists and it needs to be refreshed via a pull. + cmd := exec.Command("git", "pull") + cmd.Dir = u.repositoryLocalPath + if _, err := cmd.CombinedOutput(); err != nil { return "", vulnsrc.ErrGitFailure } } - out, err := utils.Exec(u.repositoryLocalPath, "git", "rev-parse", "HEAD") + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = u.repositoryLocalPath + out, err := cmd.CombinedOutput() if err != nil { return "", vulnsrc.ErrGitFailure } diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index d74b6122..38e73ab1 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -23,6 +23,7 @@ import ( "io" "io/ioutil" "os" + "os/exec" "regexp" "strconv" "strings" @@ -35,7 +36,6 @@ import ( "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" ) const ( @@ -174,7 +174,9 @@ func (u *updater) pullRepository() (err error) { } // Branch repository. - if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "branch", "--use-existing-dir", trackerRepository, "."); err != nil { + cmd := exec.Command("bzr", "branch", "--use-existing-dir", trackerRepository, ".") + cmd.Dir = u.repositoryLocalPath + if out, err := cmd.CombinedOutput(); err != nil { log.Errorf("could not branch Ubuntu repository: %s. output: %s", err, out) return commonerr.ErrCouldNotDownload } @@ -183,7 +185,9 @@ func (u *updater) pullRepository() (err error) { } // Pull repository. - if out, err := utils.Exec(u.repositoryLocalPath, "bzr", "pull", "--overwrite"); err != nil { + cmd := exec.Command("bzr", "pull", "--overwrite") + cmd.Dir = u.repositoryLocalPath + if out, err := cmd.CombinedOutput(); err != nil { os.RemoveAll(u.repositoryLocalPath) log.Errorf("could not pull Ubuntu repository: %s. output: %s", err, out) @@ -194,16 +198,20 @@ func (u *updater) pullRepository() (err error) { } func getRevisionNumber(pathToRepo string) (int, error) { - out, err := utils.Exec(pathToRepo, "bzr", "revno") + cmd := exec.Command("bzr", "revno") + cmd.Dir = pathToRepo + out, err := cmd.CombinedOutput() if err != nil { log.Errorf("could not get Ubuntu repository's revision number: %s. output: %s", err, out) return 0, commonerr.ErrCouldNotDownload } + revno, err := strconv.Atoi(strings.TrimSpace(string(out))) if err != nil { log.Errorf("could not parse Ubuntu repository's revision number: %s. output: %s", err, out) return 0, commonerr.ErrCouldNotDownload } + return revno, nil } @@ -252,7 +260,9 @@ func collectModifiedVulnerabilities(revision int, dbRevision, repositoryLocalPat } // Handle a database that needs upgrading. - out, err := utils.Exec(repositoryLocalPath, "bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0") + cmd := exec.Command("bzr", "log", "--verbose", "-r"+strconv.Itoa(dbRevisionInt+1)+"..", "-n0") + cmd.Dir = repositoryLocalPath + out, err := cmd.CombinedOutput() if err != nil { log.Errorf("could not get Ubuntu vulnerabilities repository logs: %s. output: %s", err, out) return nil, commonerr.ErrCouldNotDownload diff --git a/utils/exec.go b/utils/exec.go deleted file mode 100644 index 6d5e7cfc..00000000 --- a/utils/exec.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package utils simply defines utility functions and types. -package utils - -import ( - "bytes" - "os/exec" -) - -// Exec runs the given binary with arguments -func Exec(dir string, bin string, args ...string) ([]byte, error) { - _, err := exec.LookPath(bin) - if err != nil { - return nil, err - } - - cmd := exec.Command(bin, args...) - cmd.Dir = dir - - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - - err = cmd.Run() - return buf.Bytes(), err -} From 3e4dc3834f539d67829d83cf42c4c978611ab83e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 21:40:59 -0500 Subject: [PATCH 16/23] utils: remove string.go --- database/pgsql/complex_test.go | 5 +- database/pgsql/layer.go | 5 +- database/pgsql/vulnerability.go | 44 ++++++++++++++-- database/pgsql/vulnerability_test.go | 13 +++++ utils/string.go | 75 ---------------------------- utils/utils_test.go | 62 ----------------------- worker/worker.go | 14 ++++-- 7 files changed, 69 insertions(+), 149 deletions(-) delete mode 100644 utils/string.go delete mode 100644 utils/utils_test.go diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 407d8b2a..9e7e06e0 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -29,7 +29,6 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" - "github.com/coreos/clair/utils" ) const ( @@ -157,7 +156,7 @@ func TestRaceAffects(t *testing.T) { } } - assert.Len(t, utils.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0) - assert.Len(t, utils.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0) + assert.Len(t, compareStringLists(expectedAffectedNames, actualAffectedNames), 0) + assert.Len(t, compareStringLists(actualAffectedNames, expectedAffectedNames), 0) } } diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 5b50efd3..32f1cc29 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -23,7 +23,6 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" ) func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) { @@ -362,8 +361,8 @@ func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer * parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features) // Calculate the added and deleted FeatureVersions name:version. - addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV) - delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV) + addNV := compareStringLists(layerFeaturesNV, parentLayerFeaturesNV) + delNV := compareStringLists(parentLayerFeaturesNV, layerFeaturesNV) // Fill the structures containing the added and deleted FeatureVersions. for _, nv := range addNV { diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 3d791af4..4f67a380 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -23,10 +23,48 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" "github.com/guregu/null/zero" ) +// compareStringLists returns the strings that are present in X but not in Y. +func compareStringLists(X, Y []string) []string { + m := make(map[string]bool) + + for _, y := range Y { + m[y] = true + } + + diff := []string{} + for _, x := range X { + if m[x] { + continue + } + + diff = append(diff, x) + m[x] = true + } + + return diff +} + +func compareStringListsInBoth(X, Y []string) []string { + m := make(map[string]struct{}) + + for _, y := range Y { + m[y] = struct{}{} + } + + diff := []string{} + for _, x := range X { + if _, e := m[x]; e { + diff = append(diff, x) + delete(m, x) + } + } + + return diff +} + func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) { defer observeQueryTime("listVulnerabilities", "all", time.Now()) @@ -341,8 +379,8 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F currentMap, currentNames := createFeatureVersionNameMap(currentList) diffMap, diffNames := createFeatureVersionNameMap(diff) - addedNames := utils.CompareStringLists(diffNames, currentNames) - inBothNames := utils.CompareStringListsInBoth(diffNames, currentNames) + addedNames := compareStringLists(diffNames, currentNames) + inBothNames := compareStringListsInBoth(diffNames, currentNames) different := false diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index 006a7fda..53eaeaad 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -282,3 +282,16 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { } } } + +func TestStringComparison(t *testing.T) { + cmp := compareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"}) + assert.Len(t, cmp, 1) + assert.NotContains(t, cmp, "a") + assert.Contains(t, cmp, "b") + + cmp = compareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) + assert.Len(t, cmp, 2) + assert.NotContains(t, cmp, "b") + assert.Contains(t, cmp, "a") + assert.Contains(t, cmp, "c") +} diff --git a/utils/string.go b/utils/string.go deleted file mode 100644 index 78f7b7a9..00000000 --- a/utils/string.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import "regexp" - -var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) - -// CleanURL removes all parameters from an URL. -func CleanURL(str string) string { - return urlParametersRegexp.ReplaceAllString(str, "") -} - -// Contains looks for a string into an array of strings and returns whether -// the string exists. -func Contains(needle string, haystack []string) bool { - for _, h := range haystack { - if h == needle { - return true - } - } - return false -} - -// CompareStringLists returns the strings that are present in X but not in Y. -func CompareStringLists(X, Y []string) []string { - m := make(map[string]bool) - - for _, y := range Y { - m[y] = true - } - - diff := []string{} - for _, x := range X { - if m[x] { - continue - } - - diff = append(diff, x) - m[x] = true - } - - return diff -} - -// CompareStringListsInBoth returns the strings that are present in both X and Y. -func CompareStringListsInBoth(X, Y []string) []string { - m := make(map[string]struct{}) - - for _, y := range Y { - m[y] = struct{}{} - } - - diff := []string{} - for _, x := range X { - if _, e := m[x]; e { - diff = append(diff, x) - delete(m, x) - } - } - - return diff -} diff --git a/utils/utils_test.go b/utils/utils_test.go deleted file mode 100644 index 83cc6cce..00000000 --- a/utils/utils_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import ( - "testing" - - "github.com/pborman/uuid" - "github.com/stretchr/testify/assert" -) - -const fileToDownload = "http://www.google.com/robots.txt" - -// TestDiff tests the diff.go source file -func TestDiff(t *testing.T) { - cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"}) - assert.Len(t, cmp, 1) - assert.NotContains(t, cmp, "a") - assert.Contains(t, cmp, "b") - - cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) - assert.Len(t, cmp, 2) - assert.NotContains(t, cmp, "b") - assert.Contains(t, cmp, "a") - assert.Contains(t, cmp, "c") -} - -// TestExec tests the exec.go source file -func TestExec(t *testing.T) { - _, err := Exec(uuid.New(), "touch", uuid.New()) - assert.Error(t, err, "Exec should not be able to run in a not existing directory") - - o, err := Exec("/tmp", "echo", "test") - assert.Nil(t, err, "Could not exec echo") - assert.Equal(t, "test\n", string(o), "Could not exec echo") - - _, err = Exec("/tmp", uuid.New()) - assert.Error(t, err, "An invalid command should return an error") -} - -// TestString tests the string.go file -func TestString(t *testing.T) { - assert.False(t, Contains("", []string{})) - assert.True(t, Contains("a", []string{"a", "b"})) - assert.False(t, Contains("c", []string{"a", "b"})) -} - -func TestCleanURL(t *testing.T) { - assert.Equal(t, "Test http://test.cn/test Test", CleanURL("Test http://test.cn/test?foo=bar&bar=foo Test")) -} diff --git a/worker/worker.go b/worker/worker.go index e6f00932..97bcd9f1 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -17,6 +17,8 @@ package worker import ( + "regexp" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" @@ -25,7 +27,6 @@ import ( "github.com/coreos/clair/ext/imagefmt" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" - "github.com/coreos/clair/utils" ) const ( @@ -44,8 +45,15 @@ var ( // ErrParentUnknown is the error that should be raised when a parent layer // has yet to be processed for the current layer. ErrParentUnknown = commonerr.NewBadRequestError("worker: parent layer is unknown, it must be processed first") + + urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) ) +// cleanURL removes all parameters from an URL. +func cleanURL(str string) string { + return urlParametersRegexp.ReplaceAllString(str, "") +} + // Process detects the Namespace of a layer, the features it adds/removes, and // then stores everything in the database. // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an @@ -65,7 +73,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s } log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)", - name, utils.CleanURL(path), Version, parentName, imageFormat) + name, cleanURL(path), Version, parentName, imageFormat) // Check to see if the layer is already in the database. layer, err := datastore.FindLayer(name, false, false) @@ -118,7 +126,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...) files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles) if err != nil { - log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) + log.Errorf("layer %s: failed to extract data from %s: %s", name, cleanURL(path), err) return } From 00e4f7097241574277d4ed77d4c017f0c158a4e0 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 22:18:03 -0500 Subject: [PATCH 17/23] pkg/stopper: init from utils.Stopper --- api/api.go | 12 ++++++------ cmd/clair/main.go | 4 ++-- notifier/notifier.go | 8 ++++---- {utils => pkg/stopper}/stopper.go | 4 ++-- updater/updater.go | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) rename {utils => pkg/stopper}/stopper.go (97%) diff --git a/api/api.go b/api/api.go index 9528fe72..efc83a7c 100644 --- a/api/api.go +++ b/api/api.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ import ( "github.com/coreos/clair/api/context" "github.com/coreos/clair/config" - "github.com/coreos/clair/utils" + "github.com/coreos/clair/pkg/stopper" "github.com/coreos/pkg/capnslog" ) @@ -35,7 +35,7 @@ const timeoutResponse = `{"Error":{"Message":"Clair failed to respond within the var log = capnslog.NewPackageLogger("github.com/coreos/clair", "api") -func Run(config *config.APIConfig, ctx *context.RouteContext, st *utils.Stopper) { +func Run(config *config.APIConfig, ctx *context.RouteContext, st *stopper.Stopper) { defer st.End() // Do not run the API service if there is no config. @@ -68,7 +68,7 @@ func Run(config *config.APIConfig, ctx *context.RouteContext, st *utils.Stopper) log.Info("main API stopped") } -func RunHealth(config *config.APIConfig, ctx *context.RouteContext, st *utils.Stopper) { +func RunHealth(config *config.APIConfig, ctx *context.RouteContext, st *stopper.Stopper) { defer st.End() // Do not run the API service if there is no config. @@ -94,8 +94,8 @@ func RunHealth(config *config.APIConfig, ctx *context.RouteContext, st *utils.St // listenAndServeWithStopper wraps graceful.Server's // ListenAndServe/ListenAndServeTLS and adds the ability to interrupt them with -// the provided utils.Stopper -func listenAndServeWithStopper(srv *graceful.Server, st *utils.Stopper, certFile, keyFile string) { +// the provided stopper.Stopper. +func listenAndServeWithStopper(srv *graceful.Server, st *stopper.Stopper, certFile, keyFile string) { go func() { <-st.Chan() srv.Stop(0) diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 750c5fc6..3fa21556 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -32,8 +32,8 @@ import ( "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/notifier" + "github.com/coreos/clair/pkg/stopper" "github.com/coreos/clair/updater" - "github.com/coreos/clair/utils" // Register database driver. _ "github.com/coreos/clair/database/pgsql" @@ -91,7 +91,7 @@ func stopCPUProfiling(f *os.File) { // Boot starts Clair instance with the provided config. func Boot(config *config.Config) { rand.Seed(time.Now().UnixNano()) - st := utils.NewStopper() + st := stopper.NewStopper() // Open database db, err := database.Open(config.Database) diff --git a/notifier/notifier.go b/notifier/notifier.go index ad3619fa..07ec8b7c 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -29,7 +29,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/pkg/commonerr" - "github.com/coreos/clair/utils" + "github.com/coreos/clair/pkg/stopper" ) const ( @@ -59,7 +59,7 @@ func init() { } // Run starts the Notifier service. -func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) { +func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *stopper.Stopper) { defer stopper.End() // Configure registered notifiers. @@ -122,7 +122,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u log.Info("notifier service stopped") } -func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *database.VulnerabilityNotification { +func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoAmI string, stopper *stopper.Stopper) *database.VulnerabilityNotification { for { // Find a notification to send. notification, err := datastore.GetAvailableNotification(renotifyInterval) @@ -148,7 +148,7 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } } -func handleTask(n database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { +func handleTask(n database.VulnerabilityNotification, st *stopper.Stopper, maxAttempts int) (bool, bool) { // Send notification. for senderName, sender := range notification.Senders() { var attempts int diff --git a/utils/stopper.go b/pkg/stopper/stopper.go similarity index 97% rename from utils/stopper.go rename to pkg/stopper/stopper.go index e2ed6322..e939aa2a 100644 --- a/utils/stopper.go +++ b/pkg/stopper/stopper.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package stopper import ( "sync" diff --git a/updater/updater.go b/updater/updater.go index 0ae34970..267ea7fe 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -32,7 +32,7 @@ import ( "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/ext/vulnsrc" - "github.com/coreos/clair/utils" + "github.com/coreos/clair/pkg/stopper" ) const ( @@ -70,7 +70,7 @@ func init() { } // Run updates the vulnerability database at regular intervals. -func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) { +func Run(config *config.UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. From 346c22fe28db34f87959e5f7a4931940fb926a08 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 22:20:31 -0500 Subject: [PATCH 18/23] README: s/Namespace/Feature Namespace --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 64273f24..41e7da10 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,8 @@ Documentation for the current branch can be found [inside the Documentation dire - *Image* - a tarball of the contents of a container - *Layer* - an *appc* or *Docker* image that may or maybe not be dependent on another image -- *Detector* - a Go package that identifies the content, *namespaces* and *features* from a *layer* -- *Namespace* - a context around *features* and *vulnerabilities* (e.g. an operating system) - *Feature* - anything that when present could be an indication of a *vulnerability* (e.g. the presence of a file or an installed software package) +- *Feature Namespace* - a context around *features* and *vulnerabilities* (e.g. an operating system) - *Vulnerability Updater* - a Go package that tracks upstream vulnerability data and imports them into Clair - *Vulnerability Metadata Appender* - a Go package that tracks upstream vulnerability metadata and appends them into vulnerabilities managed by Clair From 6e8e6ad26b0d6ce7a9c34dde7a7c80926aae3a48 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Jan 2017 22:38:29 -0500 Subject: [PATCH 19/23] docs: fix broken link --- Documentation/api_v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/api_v1.md b/Documentation/api_v1.md index 5f9e7108..7797d72b 100644 --- a/Documentation/api_v1.md +++ b/Documentation/api_v1.md @@ -19,7 +19,7 @@ - [DELETE](#delete-namespacesnsnamevulnerabilitiesvulnnamefixesfeaturename) - [Notifications](#notifications) - [GET](#get-notificationsname) - - [DELETE](#delete-notificationname) + - [DELETE](#delete-notificationsname) ## Error Handling From 9c63a639440d5669bbc318f85aa672d4ce9fa10f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 19 Jan 2017 13:42:37 -0500 Subject: [PATCH 20/23] clair: mv updater clair and mv severity to db --- api/v1/models.go | 3 +- cmd/clair/main.go | 4 +- database/database.go | 6 +- database/models.go | 4 +- database/pgsql/complex_test.go | 3 +- database/pgsql/layer_test.go | 3 +- database/pgsql/notification_test.go | 3 +- database/pgsql/vulnerability_test.go | 15 +- database/severity.go | 134 +++++++++++++++++ severity_test.go => database/severity_test.go | 8 +- ext/vulnmdsrc/driver.go | 3 +- ext/vulnmdsrc/nvd/nvd.go | 23 +-- ext/vulnsrc/alpine/alpine.go | 9 +- ext/vulnsrc/debian/debian.go | 19 ++- ext/vulnsrc/debian/debian_test.go | 7 +- ext/vulnsrc/oracle/oracle.go | 15 +- ext/vulnsrc/oracle/oracle_test.go | 5 +- ext/vulnsrc/rhel/rhel.go | 13 +- ext/vulnsrc/rhel/rhel_test.go | 5 +- ext/vulnsrc/ubuntu/ubuntu.go | 21 ++- ext/vulnsrc/ubuntu/ubuntu_test.go | 3 +- severity.go | 139 ------------------ updater/updater.go => updater.go | 47 +++--- updater/updater_test.go => updater_test.go | 4 +- 24 files changed, 235 insertions(+), 261 deletions(-) create mode 100644 database/severity.go rename severity_test.go => database/severity_test.go (75%) delete mode 100644 severity.go rename updater/updater.go => updater.go (88%) rename updater/updater_test.go => updater_test.go (97%) diff --git a/api/v1/models.go b/api/v1/models.go index e3b1e1fb..93d048b9 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -24,7 +24,6 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/fernet/fernet-go" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" ) @@ -109,7 +108,7 @@ type Vulnerability struct { } func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { - severity, err := clair.NewSeverity(v.Severity) + severity, err := database.NewSeverity(v.Severity) if err != nil { return database.Vulnerability{}, err } diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 3fa21556..b342edc2 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -27,13 +27,13 @@ import ( "github.com/coreos/pkg/capnslog" + "github.com/coreos/clair" "github.com/coreos/clair/api" "github.com/coreos/clair/api/context" "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/notifier" "github.com/coreos/clair/pkg/stopper" - "github.com/coreos/clair/updater" // Register database driver. _ "github.com/coreos/clair/database/pgsql" @@ -112,7 +112,7 @@ func Boot(config *config.Config) { // Start updater st.Begin() - go updater.Run(config.Updater, db, st) + go clair.RunUpdater(config.Updater, db, st) // Wait for interruption and shutdown gracefully. waitForSignals(syscall.SIGINT, syscall.SIGTERM) diff --git a/database/database.go b/database/database.go index 0e57921f..f8ce18dd 100644 --- a/database/database.go +++ b/database/database.go @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package database defines the Clair's models and a common interface for database implementations. +// Package database defines the Clair's models and a common interface for +// database implementations. package database import ( @@ -144,7 +145,8 @@ type Datastore interface { // Vulnerability in the database. It can be used to store the fact that a // Vulnerability no longer affects the given Feature in any Version. // - // It has has to create a Notification that will contain the old and the updated Vulnerability. + // It has has to create a Notification that will contain the old and the + // updated Vulnerability. DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error // GetAvailableNotification returns the Name, Created, Notified and Deleted diff --git a/database/models.go b/database/models.go index 86a14096..2d645e6c 100644 --- a/database/models.go +++ b/database/models.go @@ -18,8 +18,6 @@ import ( "database/sql/driver" "encoding/json" "time" - - "github.com/coreos/clair" ) // ID is only meant to be used by database implementations and should never be used for anything else. @@ -70,7 +68,7 @@ type Vulnerability struct { Description string Link string - Severity clair.Severity + Severity Severity Metadata MetadataMap diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 9e7e06e0..ed038b4e 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -26,7 +26,6 @@ import ( "github.com/pborman/uuid" "github.com/stretchr/testify/assert" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" ) @@ -92,7 +91,7 @@ func TestRaceAffects(t *testing.T) { Version: strconv.Itoa(version), }, }, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, } vulnerabilities[version] = append(vulnerabilities[version], vulnerability) diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index ad6ae1d7..72ede4d8 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/dpkg" "github.com/coreos/clair/pkg/commonerr" @@ -91,7 +90,7 @@ func TestFindLayer(t *testing.T) { if assert.Len(t, featureVersion.AffectedBy, 1) { assert.Equal(t, "debian:7", featureVersion.AffectedBy[0].Namespace.Name) assert.Equal(t, "CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Name) - assert.Equal(t, clair.High, featureVersion.AffectedBy[0].Severity) + assert.Equal(t, database.HighSeverity, featureVersion.AffectedBy[0].Severity) assert.Equal(t, "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", featureVersion.AffectedBy[0].Description) assert.Equal(t, "http://google.com/#q=CVE-OPENSSL-1-DEB7", featureVersion.AffectedBy[0].Link) assert.Equal(t, "2.0", featureVersion.AffectedBy[0].FixedBy) diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 4e53e217..24e79246 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -169,7 +168,7 @@ func TestNotification(t *testing.T) { // Update a vulnerability and ensure that the old/new vulnerabilities are correct. v1b := v1 - v1b.Severity = clair.High + v1b.Severity = database.HighSeverity v1b.FixedIn = []database.FeatureVersion{ { Feature: f1, diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index 53eaeaad..61d835bb 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -44,7 +43,7 @@ func TestFindVulnerability(t *testing.T) { Name: "CVE-OPENSSL-1-DEB7", Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", - Severity: clair.High, + Severity: database.HighSeverity, Namespace: database.Namespace{ Name: "debian:7", VersionFormat: dpkg.ParserName, @@ -74,7 +73,7 @@ func TestFindVulnerability(t *testing.T) { Name: "debian:7", VersionFormat: dpkg.ParserName, }, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, } v2f, err := datastore.FindVulnerability("debian:7", "CVE-NOPE") @@ -180,13 +179,13 @@ func TestInsertVulnerability(t *testing.T) { Name: "", Namespace: n1, FixedIn: []database.FeatureVersion{f1}, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, }, { Name: "TestInsertVulnerability0", Namespace: database.Namespace{}, FixedIn: []database.FeatureVersion{f1}, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, }, { Name: "TestInsertVulnerability0-", @@ -197,7 +196,7 @@ func TestInsertVulnerability(t *testing.T) { Name: "TestInsertVulnerability0", Namespace: n1, FixedIn: []database.FeatureVersion{f2}, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, }, } { err := datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true) @@ -217,7 +216,7 @@ func TestInsertVulnerability(t *testing.T) { Name: "TestInsertVulnerability1", Namespace: n1, FixedIn: []database.FeatureVersion{f1, f3, f6, f7}, - Severity: clair.Low, + Severity: database.LowSeverity, Description: "TestInsertVulnerabilityDescription1", Link: "TestInsertVulnerabilityLink1", Metadata: v1meta, @@ -233,7 +232,7 @@ func TestInsertVulnerability(t *testing.T) { // Update vulnerability. v1.Description = "TestInsertVulnerabilityLink2" v1.Link = "TestInsertVulnerabilityLink2" - v1.Severity = clair.High + v1.Severity = database.HighSeverity // Update f3 in f4, add fixed in f5, add fixed in f6 which already exists, // removes fixed in f7 by adding f8 which is f7 but with MinVersion, and // add fixed by f5 a second time (duplicated). diff --git a/database/severity.go b/database/severity.go new file mode 100644 index 00000000..58084d64 --- /dev/null +++ b/database/severity.go @@ -0,0 +1,134 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "database/sql/driver" + "errors" + "strings" +) + +// ErrFailedToParseSeverity is the error returned when a severity could not +// be parsed from a string. +var ErrFailedToParseSeverity = errors.New("failed to parse Severity from input") + +// Severity defines a standard scale for measuring the severity of a +// vulnerability. +type Severity string + +const ( + // UnknownSeverity is either a security problem that has not been assigned to + // a priority yet or a priority that our system did not recognize. + UnknownSeverity Severity = "Unknown" + + // NegligibleSeverity is technically a security problem, but is only + // theoretical in nature, requires a very special situation, has almost no + // install base, or does no real damage. These tend not to get backport from + // upstreams, and will likely not be included in security updates unless + // there is an easy fix and some other issue causes an update. + NegligibleSeverity Severity = "Negligible" + + // LowSeverity is a security problem, but is hard to exploit due to + // environment, requires a user-assisted attack, a small install base, or + // does very little damage. These tend to be included in security updates + // only when higher priority issues require an update, or if many low + // priority issues have built up. + LowSeverity Severity = "Low" + + // MediumSeverity is a real security problem, and is exploitable for many + // people. Includes network daemon denial of service attacks, cross-site + // scripting, and gaining user privileges. Updates should be made soon for + // this priority of issue. + MediumSeverity Severity = "Medium" + + // HighSeverity is a real problem, exploitable for many people in a default + // installation. Includes serious remote denial of services, local root + // privilege escalations, or data loss. + HighSeverity Severity = "High" + + // CriticalSeverity is a world-burning problem, exploitable for nearly all + // people in a default installation of Linux. Includes remote root privilege + // escalations, or massive data loss. + CriticalSeverity Severity = "Critical" + + // Defcon1Severity is a Critical problem which has been manually highlighted + // by the team. It requires an immediate attention. + Defcon1Severity Severity = "Defcon1" +) + +// Severities lists all known severities, ordered from lowest to highest. +var Severities = []Severity{ + UnknownSeverity, + NegligibleSeverity, + LowSeverity, + MediumSeverity, + HighSeverity, + CriticalSeverity, + Defcon1Severity, +} + +// NewSeverity attempts to parse a string into a standard Severity value. +func NewSeverity(s string) (Severity, error) { + for _, ss := range Severities { + if strings.EqualFold(s, string(ss)) { + return ss, nil + } + } + + return UnknownSeverity, ErrFailedToParseSeverity +} + +// Compare determines the equality of two severities. +// +// If the severities are equal, returns 0. +// If the receiever is less, returns -1. +// If the receiver is greater, returns 1. +func (s Severity) Compare(s2 Severity) int { + var i1, i2 int + + for i1 = 0; i1 < len(Severities); i1 = i1 + 1 { + if s == Severities[i1] { + break + } + } + for i2 = 0; i2 < len(Severities); i2 = i2 + 1 { + if s2 == Severities[i2] { + break + } + } + + return i1 - i2 +} + +// Scan implements the database/sql.Scanner interface. +func (s *Severity) Scan(value interface{}) error { + val, ok := value.([]byte) + if !ok { + return errors.New("could not scan a Severity from a non-string input") + } + + var err error + *s, err = NewSeverity(string(val)) + if err != nil { + return err + } + + return nil +} + +// Value implements the database/sql/driver.Valuer interface. +func (s Severity) Value() (driver.Value, error) { + return string(s), nil +} diff --git a/severity_test.go b/database/severity_test.go similarity index 75% rename from severity_test.go rename to database/severity_test.go index 6bb180ef..d255c373 100644 --- a/severity_test.go +++ b/database/severity_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package clair +package database import ( "testing" @@ -21,9 +21,9 @@ import ( ) func TestCompareSeverity(t *testing.T) { - assert.Equal(t, Medium.Compare(Medium), 0, "Severity comparison failed") - assert.True(t, Medium.Compare(High) < 0, "Severity comparison failed") - assert.True(t, Critical.Compare(Low) > 0, "Severity comparison failed") + assert.Equal(t, MediumSeverity.Compare(MediumSeverity), 0, "Severity comparison failed") + assert.True(t, MediumSeverity.Compare(HighSeverity) < 0, "Severity comparison failed") + assert.True(t, CriticalSeverity.Compare(LowSeverity) > 0, "Severity comparison failed") } func TestParseSeverity(t *testing.T) { diff --git a/ext/vulnmdsrc/driver.go b/ext/vulnmdsrc/driver.go index 4de7f358..d1da7c31 100644 --- a/ext/vulnmdsrc/driver.go +++ b/ext/vulnmdsrc/driver.go @@ -19,7 +19,6 @@ package vulnmdsrc import ( "sync" - "github.com/coreos/clair" "github.com/coreos/clair/database" ) @@ -29,7 +28,7 @@ var ( ) // AppendFunc is the type of a callback provided to an Appender. -type AppendFunc func(metadataKey string, metadata interface{}, severity clair.Severity) +type AppendFunc func(metadataKey string, metadata interface{}, severity database.Severity) // Appender represents anything that can fetch vulnerability metadata and // append it to a Vulnerability. diff --git a/ext/vulnmdsrc/nvd/nvd.go b/ext/vulnmdsrc/nvd/nvd.go index 1c6b7559..c8b7698e 100644 --- a/ext/vulnmdsrc/nvd/nvd.go +++ b/ext/vulnmdsrc/nvd/nvd.go @@ -32,7 +32,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/pkg/commonerr" @@ -218,23 +217,25 @@ func getHashFromMetaURL(metaURL string) (string, error) { return "", errors.New("invalid .meta file format") } -// SeverityFromCVSS converts the CVSS Score (0.0 - 10.0) into a clair.Severity -// following the qualitative rating scale available in the CVSS v3.0 -// specification (https://www.first.org/cvss/specification-document), Table 14. +// SeverityFromCVSS converts the CVSS Score (0.0 - 10.0) into a +// database.Severity following the qualitative rating scale available in the +// CVSS v3.0 specification (https://www.first.org/cvss/specification-document), +// Table 14. +// // The Negligible level is set for CVSS scores between [0, 1), replacing the // specified None level, originally used for a score of 0. -func SeverityFromCVSS(score float64) clair.Severity { +func SeverityFromCVSS(score float64) database.Severity { switch { case score < 1.0: - return clair.Negligible + return database.NegligibleSeverity case score < 3.9: - return clair.Low + return database.LowSeverity case score < 6.9: - return clair.Medium + return database.MediumSeverity case score < 8.9: - return clair.High + return database.HighSeverity case score <= 10: - return clair.Critical + return database.CriticalSeverity } - return clair.Unknown + return database.UnknownSeverity } diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index b5391ae5..5f3acb22 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -28,7 +28,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -43,9 +42,7 @@ const ( nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" ) -var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine") -) +var log = capnslog.NewPackageLogger("github.com/coreos/clair", "ext/vulnsrc/alpine") func init() { vulnsrc.RegisterUpdater("alpine", &updater{}) @@ -230,7 +227,7 @@ func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) { vulns = append(vulns, database.Vulnerability{ Name: fix, - Severity: clair.Unknown, + Severity: database.UnknownSeverity, Link: nvdURLPrefix + fix, FixedIn: []database.FeatureVersion{ { @@ -284,7 +281,7 @@ func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) { for _, vulnStr := range vulnStrs { var vuln database.Vulnerability - vuln.Severity = clair.Unknown + vuln.Severity = database.UnknownSeverity vuln.Name = vulnStr vuln.Link = nvdURLPrefix + vulnStr vuln.FixedIn = []database.FeatureVersion{ diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index 26860b58..ec917fe3 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -158,7 +157,7 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, vulnerability = &database.Vulnerability{ Name: vulnName, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), - Severity: clair.Unknown, + Severity: database.UnknownSeverity, Description: vulnNode.Description, } } @@ -220,40 +219,40 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, } // SeverityFromUrgency converts the urgency scale used by the Debian Security -// Bug Tracker into a clair.Severity. -func SeverityFromUrgency(urgency string) clair.Severity { +// Bug Tracker into a database.Severity. +func SeverityFromUrgency(urgency string) database.Severity { switch urgency { case "not yet assigned": - return clair.Unknown + return database.UnknownSeverity case "end-of-life": fallthrough case "unimportant": - return clair.Negligible + return database.NegligibleSeverity case "low": fallthrough case "low*": fallthrough case "low**": - return clair.Low + return database.LowSeverity case "medium": fallthrough case "medium*": fallthrough case "medium**": - return clair.Medium + return database.MediumSeverity case "high": fallthrough case "high*": fallthrough case "high**": - return clair.High + return database.HighSeverity default: log.Warningf("could not determine vulnerability severity from: %s", urgency) - return clair.Unknown + return database.UnknownSeverity } } diff --git a/ext/vulnsrc/debian/debian_test.go b/ext/vulnsrc/debian/debian_test.go index 3a4c0d70..1c62500c 100644 --- a/ext/vulnsrc/debian/debian_test.go +++ b/ext/vulnsrc/debian/debian_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -37,7 +36,7 @@ func TestDebianParser(t *testing.T) { for _, vulnerability := range response.Vulnerabilities { if vulnerability.Name == "CVE-2015-1323" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2015-1323", vulnerability.Link) - assert.Equal(t, clair.Low, vulnerability.Severity) + assert.Equal(t, database.LowSeverity, vulnerability.Severity) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -68,7 +67,7 @@ func TestDebianParser(t *testing.T) { } } else if vulnerability.Name == "CVE-2003-0779" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2003-0779", vulnerability.Link) - assert.Equal(t, clair.High, vulnerability.Severity) + assert.Equal(t, database.HighSeverity, vulnerability.Severity) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -109,7 +108,7 @@ func TestDebianParser(t *testing.T) { } } else if vulnerability.Name == "CVE-2013-2685" { assert.Equal(t, "https://security-tracker.debian.org/tracker/CVE-2013-2685", vulnerability.Link) - assert.Equal(t, clair.Negligible, vulnerability.Severity) + assert.Equal(t, database.NegligibleSeverity, vulnerability.Severity) assert.Equal(t, "Un-affected packages.", vulnerability.Description) expectedFeatureVersions := []database.FeatureVersion{ diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index e82d18f9..40704096 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" @@ -337,20 +336,20 @@ func link(def definition) (link string) { return } -func severity(def definition) clair.Severity { +func severity(def definition) database.Severity { switch strings.ToLower(def.Severity) { case "n/a": - return clair.Negligible + return database.NegligibleSeverity case "low": - return clair.Low + return database.LowSeverity case "moderate": - return clair.Medium + return database.MediumSeverity case "important": - return clair.High + return database.HighSeverity case "critical": - return clair.Critical + return database.CriticalSeverity default: log.Warningf("could not determine vulnerability severity from: %s.", def.Severity) - return clair.Unknown + return database.UnknownSeverity } } diff --git a/ext/vulnsrc/oracle/oracle_test.go b/ext/vulnsrc/oracle/oracle_test.go index 2a33207c..63d3b7ee 100644 --- a/ext/vulnsrc/oracle/oracle_test.go +++ b/ext/vulnsrc/oracle/oracle_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/stretchr/testify/assert" @@ -38,7 +37,7 @@ func TestOracleParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "ELSA-2015-1193", vulnerabilities[0].Name) assert.Equal(t, "http://linux.oracle.com/errata/ELSA-2015-1193.html", vulnerabilities[0].Link) - assert.Equal(t, clair.Medium, vulnerabilities[0].Severity) + assert.Equal(t, database.MediumSeverity, vulnerabilities[0].Severity) assert.Equal(t, ` [3.1.1-7] Resolves: rhbz#1217104 CVE-2015-0252 `, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -86,7 +85,7 @@ func TestOracleParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "ELSA-2015-1207", vulnerabilities[0].Name) assert.Equal(t, "http://linux.oracle.com/errata/ELSA-2015-1207.html", vulnerabilities[0].Link) - assert.Equal(t, clair.Critical, vulnerabilities[0].Severity) + assert.Equal(t, database.CriticalSeverity, vulnerabilities[0].Severity) assert.Equal(t, ` [38.1.0-1.0.1.el7_1] - Add firefox-oracle-default-prefs.js and remove the corresponding Red Hat file [38.1.0-1] - Update to 38.1.0 ESR [38.0.1-2] - Fixed rhbz#1222807 by removing preun section `, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ { diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index 9dd66bcc..a31aa86c 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/rpm" @@ -344,18 +343,18 @@ func link(def definition) (link string) { return } -func severity(def definition) clair.Severity { +func severity(def definition) database.Severity { switch strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) { case "Low": - return clair.Low + return database.LowSeverity case "Moderate": - return clair.Medium + return database.MediumSeverity case "Important": - return clair.High + return database.HighSeverity case "Critical": - return clair.Critical + return database.CriticalSeverity default: log.Warning("could not determine vulnerability severity from: %s.", def.Title) - return clair.Unknown + return database.UnknownSeverity } } diff --git a/ext/vulnsrc/rhel/rhel_test.go b/ext/vulnsrc/rhel/rhel_test.go index 7aa13938..db762610 100644 --- a/ext/vulnsrc/rhel/rhel_test.go +++ b/ext/vulnsrc/rhel/rhel_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt/rpm" "github.com/stretchr/testify/assert" @@ -36,7 +35,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1193", vulnerabilities[0].Name) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1193.html", vulnerabilities[0].Link) - assert.Equal(t, clair.Medium, vulnerabilities[0].Severity) + assert.Equal(t, database.MediumSeverity, vulnerabilities[0].Severity) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ @@ -83,7 +82,7 @@ func TestRHELParser(t *testing.T) { if assert.Nil(t, err) && assert.Len(t, vulnerabilities, 1) { assert.Equal(t, "RHSA-2015:1207", vulnerabilities[0].Name) assert.Equal(t, "https://rhn.redhat.com/errata/RHSA-2015-1207.html", vulnerabilities[0].Link) - assert.Equal(t, clair.Critical, vulnerabilities[0].Severity) + assert.Equal(t, database.CriticalSeverity, vulnerabilities[0].Severity) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) expectedFeatureVersions := []database.FeatureVersion{ diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index 38e73ab1..70602e6c 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -30,7 +30,6 @@ import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" "github.com/coreos/clair/ext/versionfmt/dpkg" @@ -398,30 +397,30 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability // If no priority has been provided (CVE-2007-0667 for instance), set the priority to Unknown if vulnerability.Severity == "" { - vulnerability.Severity = clair.Unknown + vulnerability.Severity = database.UnknownSeverity } return } // SeverityFromPriority converts an priority from the Ubuntu CVE Tracker into -// a clair.Severity. -func SeverityFromPriority(priority string) clair.Severity { +// a database.Severity. +func SeverityFromPriority(priority string) database.Severity { switch priority { case "untriaged": - return clair.Unknown + return database.UnknownSeverity case "negligible": - return clair.Negligible + return database.NegligibleSeverity case "low": - return clair.Low + return database.LowSeverity case "medium": - return clair.Medium + return database.MediumSeverity case "high": - return clair.High + return database.HighSeverity case "critical": - return clair.Critical + return database.CriticalSeverity default: log.Warning("could not determine a vulnerability severity from: %s", priority) - return clair.Unknown + return database.UnknownSeverity } } diff --git a/ext/vulnsrc/ubuntu/ubuntu_test.go b/ext/vulnsrc/ubuntu/ubuntu_test.go index 9ebeba7a..3b9fd2ae 100644 --- a/ext/vulnsrc/ubuntu/ubuntu_test.go +++ b/ext/vulnsrc/ubuntu/ubuntu_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/versionfmt" ) @@ -37,7 +36,7 @@ func TestUbuntuParser(t *testing.T) { vulnerability, unknownReleases, err := parseUbuntuCVE(testData) if assert.Nil(t, err) { assert.Equal(t, "CVE-2015-4471", vulnerability.Name) - assert.Equal(t, clair.Medium, vulnerability.Severity) + assert.Equal(t, database.MediumSeverity, vulnerability.Severity) assert.Equal(t, "Off-by-one error in the lzxd_decompress function in lzxd.c in libmspack before 0.5 allows remote attackers to cause a denial of service (buffer under-read and application crash) via a crafted CAB archive.", vulnerability.Description) // Unknown release (line 28) diff --git a/severity.go b/severity.go deleted file mode 100644 index 46bb9927..00000000 --- a/severity.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package clair - -import ( - "database/sql/driver" - "errors" - "strings" -) - -var ( - // ErrFailedToParseSeverity is the error returned when a severity could not - // be parsed from a string. - ErrFailedToParseSeverity = errors.New("failed to parse Severity from input") -) - -// Severity defines a standard scale for measuring the severity of a -// vulnerability. -type Severity string - -const ( - // Unknown is either a security problem that has not been - // assigned to a priority yet or a priority that our system - // did not recognize. - Unknown Severity = "Unknown" - - // Negligible is technically a security problem, but is - // only theoretical in nature, requires a very special - // situation, has almost no install base, or does no real - // damage. These tend not to get backport from upstreams, - // and will likely not be included in security updates unless - // there is an easy fix and some other issue causes an update. - Negligible Severity = "Negligible" - - // Low is a security problem, but is hard to - // exploit due to environment, requires a user-assisted - // attack, a small install base, or does very little damage. - // These tend to be included in security updates only when - // higher priority issues require an update, or if many - // low priority issues have built up. - Low Severity = "Low" - - // Medium is a real security problem, and is exploitable - // for many people. Includes network daemon denial of service - // attacks, cross-site scripting, and gaining user privileges. - // Updates should be made soon for this priority of issue. - Medium Severity = "Medium" - - // High is a real problem, exploitable for many people in a default - // installation. Includes serious remote denial of services, - // local root privilege escalations, or data loss. - High Severity = "High" - - // Critical is a world-burning problem, exploitable for nearly all people - // in a default installation of Linux. Includes remote root - // privilege escalations, or massive data loss. - Critical Severity = "Critical" - - // Defcon1 is a Critical problem which has been manually highlighted by - // the team. It requires an immediate attention. - Defcon1 Severity = "Defcon1" -) - -// Severities lists all known severities, ordered from lowest to highest. -var Severities = []Severity{ - Unknown, - Negligible, - Low, - Medium, - High, - Critical, - Defcon1, -} - -// NewSeverity attempts to parse a string into a standard Severity value. -func NewSeverity(s string) (Severity, error) { - for _, ss := range Severities { - if strings.EqualFold(s, string(ss)) { - return ss, nil - } - } - - return Unknown, ErrFailedToParseSeverity -} - -// Compare determines the equality of two severities. -// -// If the severities are equal, returns 0. -// If the receiever is less, returns -1. -// If the receiver is greater, returns 1. -func (s Severity) Compare(s2 Severity) int { - var i1, i2 int - - for i1 = 0; i1 < len(Severities); i1 = i1 + 1 { - if s == Severities[i1] { - break - } - } - for i2 = 0; i2 < len(Severities); i2 = i2 + 1 { - if s2 == Severities[i2] { - break - } - } - - return i1 - i2 -} - -// Scan implements the database/sql.Scanner interface. -func (s *Severity) Scan(value interface{}) error { - val, ok := value.([]byte) - if !ok { - return errors.New("could not scan a Severity from a non-string input") - } - - var err error - *s, err = NewSeverity(string(val)) - if err != nil { - return err - } - - return nil -} - -// Value implements the database/sql/driver.Valuer interface. -func (s Severity) Value() (driver.Value, error) { - return string(s), nil -} diff --git a/updater/updater.go b/updater.go similarity index 88% rename from updater/updater.go rename to updater.go index 267ea7fe..41eb8128 100644 --- a/updater/updater.go +++ b/updater.go @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package updater updates the vulnerability database periodically using the -// registered vulnerability source updaters and vulnerability metadata -// appenders. -package updater +package clair import ( "math/rand" @@ -27,7 +24,6 @@ import ( "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" - "github.com/coreos/clair" "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" @@ -36,16 +32,14 @@ import ( ) const ( - flagName = "updater/last" - notesFlagName = "updater/notes" - - lockName = "updater" - lockDuration = refreshLockDuration + time.Minute*2 - refreshLockDuration = time.Minute * 8 + updaterLastFlagName = "updater/last" + updaterLockName = "updater" + updaterLockDuration = updaterLockRefreshDuration + time.Minute*2 + updaterLockRefreshDuration = time.Minute * 8 ) var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater") + log = capnslog.NewPackageLogger("github.com/coreos/clair", "clair") promUpdaterErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ Name: "clair_updater_errors_total", @@ -69,8 +63,9 @@ func init() { prometheus.MustRegister(promUpdaterNotesTotal) } -// Run updates the vulnerability database at regular intervals. -func Run(config *config.UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { +// RunUpdater begins a process that updates the vulnerability database at +// regular intervals. +func RunUpdater(config *config.UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. @@ -100,12 +95,12 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *stopper if nextUpdate.Before(time.Now().UTC()) { // Attempt to get a lock on the the update. log.Debug("attempting to obtain update lock") - hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false) + hasLock, hasLockUntil := datastore.Lock(updaterLockName, whoAmI, updaterLockDuration, false) if hasLock { // Launch update in a new go routine. doneC := make(chan bool, 1) go func() { - Update(datastore, firstUpdate) + update(datastore, firstUpdate) doneC <- true }() @@ -113,23 +108,23 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *stopper select { case <-doneC: done = true - case <-time.After(refreshLockDuration): + case <-time.After(updaterLockRefreshDuration): // Refresh the lock until the update is done. - datastore.Lock(lockName, whoAmI, lockDuration, true) + datastore.Lock(updaterLockName, whoAmI, updaterLockDuration, true) case <-st.Chan(): stop = true } } // Unlock the update. - datastore.Unlock(lockName, whoAmI) + datastore.Unlock(updaterLockName, whoAmI) if stop { break } continue } else { - lockOwner, lockExpiration, err := datastore.FindLock(lockName) + lockOwner, lockExpiration, err := datastore.FindLock(updaterLockName) if err != nil { log.Debug("update lock is already taken") nextUpdate = hasLockUntil @@ -162,9 +157,9 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *stopper log.Info("updater service stopped") } -// Update fetches all the vulnerabilities from the registered fetchers, upserts +// update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. -func Update(datastore database.Datastore, firstUpdate bool) { +func update(datastore database.Datastore, firstUpdate bool) { defer setUpdaterDuration(time.Now()) log.Info("updating vulnerabilities") @@ -195,7 +190,7 @@ func Update(datastore database.Datastore, firstUpdate bool) { // Update last successful update if every fetchers worked properly. if status { - datastore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) + datastore.InsertKeyValue(updaterLastFlagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) } log.Info("update finished") @@ -293,7 +288,7 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner } func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) { - lastUpdateTSS, err := datastore.GetKeyValue(flagName) + lastUpdateTSS, err := datastore.GetKeyValue(updaterLastFlagName) if err != nil { return time.Time{}, false, err } @@ -316,7 +311,7 @@ type lockableVulnerability struct { sync.Mutex } -func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity clair.Severity) { +func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interface{}, severity database.Severity) { lv.Lock() defer lv.Unlock() @@ -329,7 +324,7 @@ func (lv *lockableVulnerability) appendFunc(metadataKey string, metadata interfa lv.Metadata[metadataKey] = metadata // If necessary, provide a severity for the vulnerability. - if lv.Severity == clair.Unknown { + if lv.Severity == database.UnknownSeverity { lv.Severity = severity } } diff --git a/updater/updater_test.go b/updater_test.go similarity index 97% rename from updater/updater_test.go rename to updater_test.go index c3672571..380ff277 100644 --- a/updater/updater_test.go +++ b/updater_test.go @@ -1,4 +1,4 @@ -// Copyright 2016 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package updater +package clair import ( "fmt" From e5c567f3f98b68d990e2b41c3a7f1f0261dcf060 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 26 Jan 2017 18:19:32 -0500 Subject: [PATCH 21/23] clair: mv notifier to top level --- cmd/clair/main.go | 3 +-- notifier/notifier.go => notifier.go | 31 ++++++++++++----------------- 2 files changed, 14 insertions(+), 20 deletions(-) rename notifier/notifier.go => notifier.go (83%) diff --git a/cmd/clair/main.go b/cmd/clair/main.go index b342edc2..75a4124e 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -32,7 +32,6 @@ import ( "github.com/coreos/clair/api/context" "github.com/coreos/clair/config" "github.com/coreos/clair/database" - "github.com/coreos/clair/notifier" "github.com/coreos/clair/pkg/stopper" // Register database driver. @@ -102,7 +101,7 @@ func Boot(config *config.Config) { // Start notifier st.Begin() - go notifier.Run(config.Notifier, db, st) + go clair.RunNotifier(config.Notifier, db, st) // Start API st.Begin() diff --git a/notifier/notifier.go b/notifier.go similarity index 83% rename from notifier/notifier.go rename to notifier.go index 07ec8b7c..50158cdd 100644 --- a/notifier/notifier.go +++ b/notifier.go @@ -12,15 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package notifier fetches notifications from the database and informs the -// specified remote handler about their existences, inviting the third party to -// actively query the API about it. -package notifier +package clair import ( "time" - "github.com/coreos/pkg/capnslog" "github.com/coreos/pkg/timeutil" "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" @@ -33,15 +29,13 @@ import ( ) const ( - checkInterval = 5 * time.Minute - refreshLockDuration = time.Minute * 2 - lockDuration = time.Minute*8 + refreshLockDuration - maxBackOff = 15 * time.Minute + notifierCheckInterval = 5 * time.Minute + notifierMaxBackOff = 15 * time.Minute + notifierLockRefreshDuration = time.Minute * 2 + notifierLockDuration = time.Minute*8 + notifierLockRefreshDuration ) var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "notifier") - promNotifierLatencyMilliseconds = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "clair_notifier_latency_milliseconds", Help: "Time it takes to send a notification after it's been created.", @@ -58,8 +52,9 @@ func init() { prometheus.MustRegister(promNotifierBackendErrorsTotal) } -// Run starts the Notifier service. -func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *stopper.Stopper) { +// RunNotifier begins a process that checks for new notifications that should +// be sent out to third parties. +func RunNotifier(config *config.NotifierConfig, datastore database.Datastore, stopper *stopper.Stopper) { defer stopper.End() // Configure registered notifiers. @@ -113,8 +108,8 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *s select { case <-done: break outer - case <-time.After(refreshLockDuration): - datastore.Lock(notification.Name, whoAmI, lockDuration, true) + case <-time.After(notifierLockRefreshDuration): + datastore.Lock(notification.Name, whoAmI, notifierLockDuration, true) } } } @@ -133,7 +128,7 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } // Wait. - if !stopper.Sleep(checkInterval) { + if !stopper.Sleep(notifierCheckInterval) { return nil } @@ -141,7 +136,7 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } // Lock the notification. - if hasLock, _ := datastore.Lock(notification.Name, whoAmI, lockDuration, false); hasLock { + if hasLock, _ := datastore.Lock(notification.Name, whoAmI, notifierLockDuration, false); hasLock { log.Infof("found and locked a notification: %s", notification.Name) return ¬ification } @@ -173,7 +168,7 @@ func handleTask(n database.VulnerabilityNotification, st *stopper.Stopper, maxAt // Send failed; increase attempts/backoff and retry. promNotifierBackendErrorsTotal.WithLabelValues(senderName).Inc() log.Errorf("could not send notification '%s' via notifier '%s': %v", n.Name, senderName, err) - backOff = timeutil.ExpBackoff(backOff, maxBackOff) + backOff = timeutil.ExpBackoff(backOff, notifierMaxBackOff) attempts++ continue } From 889615276af2c1d5ac04971a237e09d2e9fa6bda Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 26 Jan 2017 18:24:04 -0500 Subject: [PATCH 22/23] clair: move worker to top level package --- api/v1/routes.go | 8 ++++---- .../DistUpgrade/blank.tar.gz | Bin .../DistUpgrade/jessie.tar.gz | Bin .../DistUpgrade/wheezy.tar.gz | Bin worker/worker.go => worker.go | 19 +++++++----------- worker/worker_test.go => worker_test.go | 2 +- 6 files changed, 12 insertions(+), 17 deletions(-) rename {worker/testdata => testdata}/DistUpgrade/blank.tar.gz (100%) rename {worker/testdata => testdata}/DistUpgrade/jessie.tar.gz (100%) rename {worker/testdata => testdata}/DistUpgrade/wheezy.tar.gz (100%) rename worker/worker.go => worker.go (92%) rename worker/worker_test.go => worker_test.go (99%) diff --git a/api/v1/routes.go b/api/v1/routes.go index bde7560d..97048eec 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -25,11 +25,11 @@ import ( "github.com/julienschmidt/httprouter" "github.com/prometheus/client_golang/prometheus" + "github.com/coreos/clair" "github.com/coreos/clair/api/context" "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" - "github.com/coreos/clair/worker" ) const ( @@ -109,11 +109,11 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return postLayerRoute, http.StatusBadRequest } - err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) + err = clair.ProcessLayer(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) if err != nil { if err == tarutil.ErrCouldNotExtract || err == tarutil.ErrExtractedFileTooBig || - err == worker.ErrUnsupported { + err == clair.ErrUnsupported { writeResponse(w, r, statusUnprocessableEntity, LayerEnvelope{Error: &Error{err.Error()}}) return postLayerRoute, statusUnprocessableEntity } @@ -133,7 +133,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx Path: request.Layer.Path, Headers: request.Layer.Headers, Format: request.Layer.Format, - IndexedByVersion: worker.Version, + IndexedByVersion: clair.Version, }}) return postLayerRoute, http.StatusCreated } diff --git a/worker/testdata/DistUpgrade/blank.tar.gz b/testdata/DistUpgrade/blank.tar.gz similarity index 100% rename from worker/testdata/DistUpgrade/blank.tar.gz rename to testdata/DistUpgrade/blank.tar.gz diff --git a/worker/testdata/DistUpgrade/jessie.tar.gz b/testdata/DistUpgrade/jessie.tar.gz similarity index 100% rename from worker/testdata/DistUpgrade/jessie.tar.gz rename to testdata/DistUpgrade/jessie.tar.gz diff --git a/worker/testdata/DistUpgrade/wheezy.tar.gz b/testdata/DistUpgrade/wheezy.tar.gz similarity index 100% rename from worker/testdata/DistUpgrade/wheezy.tar.gz rename to testdata/DistUpgrade/wheezy.tar.gz diff --git a/worker/worker.go b/worker.go similarity index 92% rename from worker/worker.go rename to worker.go index 97bcd9f1..92aeea2f 100644 --- a/worker/worker.go +++ b/worker.go @@ -12,15 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package worker implements the logic to extract useful informations from a -// container layer and store it in the database. -package worker +package clair import ( "regexp" - "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair/database" "github.com/coreos/clair/ext/featurefmt" "github.com/coreos/clair/ext/featurens" @@ -36,8 +32,6 @@ const ( ) var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker") - // ErrUnsupported is the error that should be raised when an OS or package // manager is not supported. ErrUnsupported = commonerr.NewBadRequestError("worker: OS and/or package manager are not supported") @@ -54,11 +48,12 @@ func cleanURL(str string) string { return urlParametersRegexp.ReplaceAllString(str, "") } -// Process detects the Namespace of a layer, the features it adds/removes, and -// then stores everything in the database. -// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an -// older engine version and that processes them. -func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error { +// ProcessLayer detects the Namespace of a layer, the features it adds/removes, +// and then stores everything in the database. +// +// TODO(Quentin-M): We could have a goroutine that looks for layers that have +// been analyzed with an older engine version and that processes them. +func ProcessLayer(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error { // Verify parameters. if name == "" { return commonerr.NewBadRequestError("could not process a layer which does not have a name") diff --git a/worker/worker_test.go b/worker_test.go similarity index 99% rename from worker/worker_test.go rename to worker_test.go index 26e104fa..34d6f039 100644 --- a/worker/worker_test.go +++ b/worker_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package worker +package clair import ( "path/filepath" From 6a569fd945dc0e54255e62d9c87f924551fe0fc7 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 26 Jan 2017 20:14:44 -0500 Subject: [PATCH 23/23] move config to main / decentralize config This puts config in its relevant location and moves functions around loading config files into the main package. As a side effect of removing cyclic imports for the API config, the context library is no longer used. --- api/api.go | 38 ++++++++------ api/context/context.go | 66 ------------------------ api/router.go | 31 ++++++------ api/v1/router.go | 78 ++++++++++++++++++++++------- api/v1/routes.go | 41 ++++++++------- {config => cmd/clair}/config.go | 64 ++++++++--------------- cmd/clair/main.go | 10 ++-- database/database.go | 14 ++++-- database/pgsql/pgsql.go | 13 ++--- database/pgsql/pgsql_test.go | 7 +-- ext/notification/driver.go | 12 ++++- ext/notification/webhook/webhook.go | 3 +- notifier.go | 3 +- updater.go | 8 ++- worker_test.go | 6 +-- 15 files changed, 186 insertions(+), 208 deletions(-) delete mode 100644 api/context/context.go rename {config => cmd/clair}/config.go (61%) diff --git a/api/api.go b/api/api.go index efc83a7c..9da5073a 100644 --- a/api/api.go +++ b/api/api.go @@ -23,29 +23,37 @@ import ( "strconv" "time" + "github.com/coreos/pkg/capnslog" "github.com/tylerb/graceful" - "github.com/coreos/clair/api/context" - "github.com/coreos/clair/config" + "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/stopper" - "github.com/coreos/pkg/capnslog" ) const timeoutResponse = `{"Error":{"Message":"Clair failed to respond within the configured timeout window.","Type":"Timeout"}}` var log = capnslog.NewPackageLogger("github.com/coreos/clair", "api") -func Run(config *config.APIConfig, ctx *context.RouteContext, st *stopper.Stopper) { +// Config is the configuration for the API service. +type Config struct { + Port int + HealthPort int + Timeout time.Duration + PaginationKey string + CertFile, KeyFile, CAFile string +} + +func Run(cfg *Config, store database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the API service if there is no config. - if config == nil { + if cfg == nil { log.Infof("main API service is disabled.") return } - log.Infof("starting main API on port %d.", config.Port) + log.Infof("starting main API on port %d.", cfg.Port) - tlsConfig, err := tlsClientConfig(config.CAFile) + tlsConfig, err := tlsClientConfig(cfg.CAFile) if err != nil { log.Fatalf("could not initialize client cert authentication: %s\n", err) } @@ -57,33 +65,33 @@ func Run(config *config.APIConfig, ctx *context.RouteContext, st *stopper.Stoppe Timeout: 0, // Already handled by our TimeOut middleware NoSignalHandling: true, // We want to use our own Stopper Server: &http.Server{ - Addr: ":" + strconv.Itoa(config.Port), + Addr: ":" + strconv.Itoa(cfg.Port), TLSConfig: tlsConfig, - Handler: http.TimeoutHandler(newAPIHandler(ctx), config.Timeout, timeoutResponse), + Handler: http.TimeoutHandler(newAPIHandler(cfg, store), cfg.Timeout, timeoutResponse), }, } - listenAndServeWithStopper(srv, st, config.CertFile, config.KeyFile) + listenAndServeWithStopper(srv, st, cfg.CertFile, cfg.KeyFile) log.Info("main API stopped") } -func RunHealth(config *config.APIConfig, ctx *context.RouteContext, st *stopper.Stopper) { +func RunHealth(cfg *Config, store database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the API service if there is no config. - if config == nil { + if cfg == nil { log.Infof("health API service is disabled.") return } - log.Infof("starting health API on port %d.", config.HealthPort) + log.Infof("starting health API on port %d.", cfg.HealthPort) srv := &graceful.Server{ Timeout: 10 * time.Second, // Interrupt health checks when stopping NoSignalHandling: true, // We want to use our own Stopper Server: &http.Server{ - Addr: ":" + strconv.Itoa(config.HealthPort), - Handler: http.TimeoutHandler(newHealthHandler(ctx), config.Timeout, timeoutResponse), + Addr: ":" + strconv.Itoa(cfg.HealthPort), + Handler: http.TimeoutHandler(newHealthHandler(store), cfg.Timeout, timeoutResponse), }, } diff --git a/api/context/context.go b/api/context/context.go deleted file mode 100644 index d7df44f2..00000000 --- a/api/context/context.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package context - -import ( - "net/http" - "strconv" - "time" - - "github.com/coreos/pkg/capnslog" - "github.com/julienschmidt/httprouter" - "github.com/prometheus/client_golang/prometheus" - - "github.com/coreos/clair/config" - "github.com/coreos/clair/database" -) - -var ( - log = capnslog.NewPackageLogger("github.com/coreos/clair", "api") - - promResponseDurationMilliseconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Name: "clair_api_response_duration_milliseconds", - Help: "The duration of time it takes to receieve and write a response to an API request", - Buckets: prometheus.ExponentialBuckets(9.375, 2, 10), - }, []string{"route", "code"}) -) - -func init() { - prometheus.MustRegister(promResponseDurationMilliseconds) -} - -type Handler func(http.ResponseWriter, *http.Request, httprouter.Params, *RouteContext) (route string, status int) - -func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle { - return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { - start := time.Now() - route, status := handler(w, r, p, ctx) - statusStr := strconv.Itoa(status) - if status == 0 { - statusStr = "???" - } - - promResponseDurationMilliseconds. - WithLabelValues(route, statusStr). - Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)) - - log.Infof("%s \"%s %s\" %s (%s)", r.RemoteAddr, r.Method, r.RequestURI, statusStr, time.Since(start)) - } -} - -type RouteContext struct { - Store database.Datastore - Config *config.APIConfig -} diff --git a/api/router.go b/api/router.go index 808ca5f7..76d9b827 100644 --- a/api/router.go +++ b/api/router.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ import ( "github.com/julienschmidt/httprouter" - "github.com/coreos/clair/api/context" "github.com/coreos/clair/api/v1" + "github.com/coreos/clair/database" ) // router is an HTTP router that forwards requests to the appropriate sub-router @@ -31,9 +31,9 @@ type router map[string]*httprouter.Router // Let's hope we never have more than 99 API versions. const apiVersionLength = len("v99") -func newAPIHandler(ctx *context.RouteContext) http.Handler { +func newAPIHandler(cfg *Config, store database.Datastore) http.Handler { router := make(router) - router["/v1"] = v1.NewRouter(ctx) + router["/v1"] = v1.NewRouter(store, cfg.PaginationKey) return router } @@ -56,21 +56,22 @@ func (rtr router) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) } -func newHealthHandler(ctx *context.RouteContext) http.Handler { +func newHealthHandler(store database.Datastore) http.Handler { router := httprouter.New() - router.GET("/health", context.HTTPHandler(getHealth, ctx)) + router.GET("/health", healthHandler(store)) return router } -func getHealth(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - header := w.Header() - header.Set("Server", "clair") +func healthHandler(store database.Datastore) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + header := w.Header() + header.Set("Server", "clair") - status := http.StatusInternalServerError - if ctx.Store.Ping() { - status = http.StatusOK + status := http.StatusInternalServerError + if store.Ping() { + status = http.StatusOK + } + + w.WriteHeader(status) } - - w.WriteHeader(status) - return "health", status } diff --git a/api/v1/router.go b/api/v1/router.go index 5a3d640e..44608949 100644 --- a/api/v1/router.go +++ b/api/v1/router.go @@ -16,41 +16,83 @@ package v1 import ( - "github.com/julienschmidt/httprouter" + "net/http" + "strconv" + "time" - "github.com/coreos/clair/api/context" + "github.com/julienschmidt/httprouter" + "github.com/prometheus/client_golang/prometheus" + + "github.com/coreos/clair/database" ) +var ( + promResponseDurationMilliseconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "clair_api_response_duration_milliseconds", + Help: "The duration of time it takes to receieve and write a response to an API request", + Buckets: prometheus.ExponentialBuckets(9.375, 2, 10), + }, []string{"route", "code"}) +) + +func init() { + prometheus.MustRegister(promResponseDurationMilliseconds) +} + +type handler func(http.ResponseWriter, *http.Request, httprouter.Params, *context) (route string, status int) + +func httpHandler(h handler, ctx *context) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + start := time.Now() + route, status := h(w, r, p, ctx) + statusStr := strconv.Itoa(status) + if status == 0 { + statusStr = "???" + } + + promResponseDurationMilliseconds. + WithLabelValues(route, statusStr). + Observe(float64(time.Since(start).Nanoseconds()) / float64(time.Millisecond)) + + log.Infof("%s \"%s %s\" %s (%s)", r.RemoteAddr, r.Method, r.RequestURI, statusStr, time.Since(start)) + } +} + +type context struct { + Store database.Datastore + PaginationKey string +} + // NewRouter creates an HTTP router for version 1 of the Clair API. -func NewRouter(ctx *context.RouteContext) *httprouter.Router { +func NewRouter(store database.Datastore, paginationKey string) *httprouter.Router { router := httprouter.New() + ctx := &context{store, paginationKey} // Layers - router.POST("/layers", context.HTTPHandler(postLayer, ctx)) - router.GET("/layers/:layerName", context.HTTPHandler(getLayer, ctx)) - router.DELETE("/layers/:layerName", context.HTTPHandler(deleteLayer, ctx)) + router.POST("/layers", httpHandler(postLayer, ctx)) + router.GET("/layers/:layerName", httpHandler(getLayer, ctx)) + router.DELETE("/layers/:layerName", httpHandler(deleteLayer, ctx)) // Namespaces - router.GET("/namespaces", context.HTTPHandler(getNamespaces, ctx)) + router.GET("/namespaces", httpHandler(getNamespaces, ctx)) // Vulnerabilities - router.GET("/namespaces/:namespaceName/vulnerabilities", context.HTTPHandler(getVulnerabilities, ctx)) - router.POST("/namespaces/:namespaceName/vulnerabilities", context.HTTPHandler(postVulnerability, ctx)) - router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(getVulnerability, ctx)) - router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(putVulnerability, ctx)) - router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", context.HTTPHandler(deleteVulnerability, ctx)) + router.GET("/namespaces/:namespaceName/vulnerabilities", httpHandler(getVulnerabilities, ctx)) + router.POST("/namespaces/:namespaceName/vulnerabilities", httpHandler(postVulnerability, ctx)) + router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", httpHandler(getVulnerability, ctx)) + router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", httpHandler(putVulnerability, ctx)) + router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName", httpHandler(deleteVulnerability, ctx)) // Fixes - router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes", context.HTTPHandler(getFixes, ctx)) - router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", context.HTTPHandler(putFix, ctx)) - router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", context.HTTPHandler(deleteFix, ctx)) + router.GET("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes", httpHandler(getFixes, ctx)) + router.PUT("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", httpHandler(putFix, ctx)) + router.DELETE("/namespaces/:namespaceName/vulnerabilities/:vulnerabilityName/fixes/:fixName", httpHandler(deleteFix, ctx)) // Notifications - router.GET("/notifications/:notificationName", context.HTTPHandler(getNotification, ctx)) - router.DELETE("/notifications/:notificationName", context.HTTPHandler(deleteNotification, ctx)) + router.GET("/notifications/:notificationName", httpHandler(getNotification, ctx)) + router.DELETE("/notifications/:notificationName", httpHandler(deleteNotification, ctx)) // Metrics - router.GET("/metrics", context.HTTPHandler(getMetrics, ctx)) + router.GET("/metrics", httpHandler(getMetrics, ctx)) return router } diff --git a/api/v1/routes.go b/api/v1/routes.go index 97048eec..eab3115c 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -26,7 +26,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/coreos/clair" - "github.com/coreos/clair/api/context" "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/commonerr" "github.com/coreos/clair/pkg/tarutil" @@ -96,7 +95,7 @@ func writeResponse(w http.ResponseWriter, r *http.Request, status int, resp inte } } -func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { request := LayerEnvelope{} err := decodeJSON(r, &request) if err != nil { @@ -138,7 +137,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return postLayerRoute, http.StatusCreated } -func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { _, withFeatures := r.URL.Query()["features"] _, withVulnerabilities := r.URL.Query()["vulnerabilities"] @@ -157,7 +156,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * return getLayerRoute, http.StatusOK } -func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { err := ctx.Store.DeleteLayer(p.ByName("layerName")) if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}}) @@ -171,7 +170,7 @@ func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ct return deleteLayerRoute, http.StatusOK } -func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { dbNamespaces, err := ctx.Store.ListNamespaces() if err != nil { writeResponse(w, r, http.StatusInternalServerError, NamespaceEnvelope{Error: &Error{err.Error()}}) @@ -189,7 +188,7 @@ func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, return getNamespacesRoute, http.StatusOK } -func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { query := r.URL.Query() limitStrs, limitExists := query["limit"] @@ -209,7 +208,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par page := 0 pageStrs, pageExists := query["page"] if pageExists { - err = tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page) + err = tokenUnmarshal(pageStrs[0], ctx.PaginationKey, &page) if err != nil { writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"invalid page format: " + err.Error()}}) return getNotificationRoute, http.StatusBadRequest @@ -239,7 +238,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par var nextPageStr string if nextPage != -1 { - nextPageBytes, err := tokenMarshal(nextPage, ctx.Config.PaginationKey) + nextPageBytes, err := tokenMarshal(nextPage, ctx.PaginationKey) if err != nil { writeResponse(w, r, http.StatusBadRequest, VulnerabilityEnvelope{Error: &Error{"failed to marshal token: " + err.Error()}}) return getNotificationRoute, http.StatusBadRequest @@ -251,7 +250,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par return getVulnerabilitiesRoute, http.StatusOK } -func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { request := VulnerabilityEnvelope{} err := decodeJSON(r, &request) if err != nil { @@ -286,7 +285,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para return postVulnerabilityRoute, http.StatusCreated } -func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { _, withFixedIn := r.URL.Query()["fixedIn"] dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) @@ -304,7 +303,7 @@ func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param return getVulnerabilityRoute, http.StatusOK } -func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { request := VulnerabilityEnvelope{} err := decodeJSON(r, &request) if err != nil { @@ -347,7 +346,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param return putVulnerabilityRoute, http.StatusOK } -func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) @@ -361,7 +360,7 @@ func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Pa return deleteVulnerabilityRoute, http.StatusOK } -func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) @@ -376,7 +375,7 @@ func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * return getFixesRoute, http.StatusOK } -func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { request := FeatureEnvelope{} err := decodeJSON(r, &request) if err != nil { @@ -420,7 +419,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co return putFixRoute, http.StatusOK } -func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName")) if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) @@ -434,7 +433,7 @@ func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return deleteFixRoute, http.StatusOK } -func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { query := r.URL.Query() limitStrs, limitExists := query["limit"] @@ -452,14 +451,14 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params page := database.VulnerabilityNotificationFirstPage pageStrs, pageExists := query["page"] if pageExists { - err := tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page) + err := tokenUnmarshal(pageStrs[0], ctx.PaginationKey, &page) if err != nil { writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"invalid page format: " + err.Error()}}) return getNotificationRoute, http.StatusBadRequest } pageToken = pageStrs[0] } else { - pageTokenBytes, err := tokenMarshal(page, ctx.Config.PaginationKey) + pageTokenBytes, err := tokenMarshal(page, ctx.PaginationKey) if err != nil { writeResponse(w, r, http.StatusBadRequest, NotificationEnvelope{Error: &Error{"failed to marshal token: " + err.Error()}}) return getNotificationRoute, http.StatusBadRequest @@ -476,13 +475,13 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params return getNotificationRoute, http.StatusInternalServerError } - notification := NotificationFromDatabaseModel(dbNotification, limit, pageToken, nextPage, ctx.Config.PaginationKey) + notification := NotificationFromDatabaseModel(dbNotification, limit, pageToken, nextPage, ctx.PaginationKey) writeResponse(w, r, http.StatusOK, NotificationEnvelope{Notification: ¬ification}) return getNotificationRoute, http.StatusOK } -func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { err := ctx.Store.DeleteNotification(p.ByName("notificationName")) if err == commonerr.ErrNotFound { writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}}) @@ -496,7 +495,7 @@ func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Par return deleteNotificationRoute, http.StatusOK } -func getMetrics(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { +func getMetrics(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context) (string, int) { prometheus.Handler().ServeHTTP(w, r) return getMetricsRoute, 0 } diff --git a/config/config.go b/cmd/clair/config.go similarity index 61% rename from config/config.go rename to cmd/clair/config.go index 12572e6c..c11e971f 100644 --- a/config/config.go +++ b/cmd/clair/config.go @@ -1,4 +1,4 @@ -// Copyright 2015 clair authors +// Copyright 2017 clair authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package config +package main import ( "errors" @@ -20,21 +20,19 @@ import ( "os" "time" - "github.com/fernet/fernet-go" "gopkg.in/yaml.v2" + + "github.com/coreos/clair" + "github.com/coreos/clair/api" + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/notification" + "github.com/fernet/fernet-go" ) -// ErrDatasourceNotLoaded is returned when the datasource variable in the configuration file is not loaded properly +// ErrDatasourceNotLoaded is returned when the datasource variable in the +// configuration file is not loaded properly var ErrDatasourceNotLoaded = errors.New("could not load configuration: no database source specified") -// RegistrableComponentConfig is a configuration block that can be used to -// determine which registrable component should be initialized and pass -// custom configuration to it. -type RegistrableComponentConfig struct { - Type string - Options map[string]interface{} -} - // File represents a YAML configuration file that namespaces all Clair // configuration under the top-level "clair" key. type File struct { @@ -43,57 +41,37 @@ type File struct { // Config is the global configuration for an instance of Clair. type Config struct { - Database RegistrableComponentConfig - Updater *UpdaterConfig - Notifier *NotifierConfig - API *APIConfig -} - -// UpdaterConfig is the configuration for the Updater service. -type UpdaterConfig struct { - Interval time.Duration -} - -// NotifierConfig is the configuration for the Notifier service and its registered notifiers. -type NotifierConfig struct { - Attempts int - RenotifyInterval time.Duration - Params map[string]interface{} `yaml:",inline"` -} - -// APIConfig is the configuration for the API service. -type APIConfig struct { - Port int - HealthPort int - Timeout time.Duration - PaginationKey string - CertFile, KeyFile, CAFile string + Database database.RegistrableComponentConfig + Updater *clair.UpdaterConfig + Notifier *notification.Config + API *api.Config } // DefaultConfig is a configuration that can be used as a fallback value. func DefaultConfig() Config { return Config{ - Database: RegistrableComponentConfig{ + Database: database.RegistrableComponentConfig{ Type: "pgsql", }, - Updater: &UpdaterConfig{ + Updater: &clair.UpdaterConfig{ Interval: 1 * time.Hour, }, - API: &APIConfig{ + API: &api.Config{ Port: 6060, HealthPort: 6061, Timeout: 900 * time.Second, }, - Notifier: &NotifierConfig{ + Notifier: ¬ification.Config{ Attempts: 5, RenotifyInterval: 2 * time.Hour, }, } } -// Load is a shortcut to open a file, read it, and generate a Config. +// LoadConfig is a shortcut to open a file, read it, and generate a Config. +// // It supports relative and absolute paths. Given "", it returns DefaultConfig. -func Load(path string) (config *Config, err error) { +func LoadConfig(path string) (config *Config, err error) { var cfgFile File cfgFile.Clair = DefaultConfig() if path == "" { diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 75a4124e..754e54e5 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -29,8 +29,6 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/api" - "github.com/coreos/clair/api/context" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/pkg/stopper" @@ -88,7 +86,7 @@ func stopCPUProfiling(f *os.File) { } // Boot starts Clair instance with the provided config. -func Boot(config *config.Config) { +func Boot(config *Config) { rand.Seed(time.Now().UnixNano()) st := stopper.NewStopper() @@ -105,9 +103,9 @@ func Boot(config *config.Config) { // Start API st.Begin() - go api.Run(config.API, &context.RouteContext{db, config.API}, st) + go api.Run(config.API, db, st) st.Begin() - go api.RunHealth(config.API, &context.RouteContext{db, config.API}, st) + go api.RunHealth(config.API, db, st) // Start updater st.Begin() @@ -136,7 +134,7 @@ func main() { } // Load configuration - config, err := config.Load(*flagConfigPath) + config, err := LoadConfig(*flagConfigPath) if err != nil { log.Fatalf("failed to load configuration: %s", err) } diff --git a/database/database.go b/database/database.go index f8ce18dd..d3f7fc0f 100644 --- a/database/database.go +++ b/database/database.go @@ -20,8 +20,6 @@ import ( "errors" "fmt" "time" - - "github.com/coreos/clair/config" ) var ( @@ -35,11 +33,19 @@ var ( ErrInconsistent = errors.New("database: inconsistent database") ) +// RegistrableComponentConfig is a configuration block that can be used to +// determine which registrable component should be initialized and pass custom +// configuration to it. +type RegistrableComponentConfig struct { + Type string + Options map[string]interface{} +} + var drivers = make(map[string]Driver) // Driver is a function that opens a Datastore specified by its database driver type and specific // configuration. -type Driver func(config.RegistrableComponentConfig) (Datastore, error) +type Driver func(RegistrableComponentConfig) (Datastore, error) // Register makes a Constructor available by the provided name. // @@ -56,7 +62,7 @@ func Register(name string, driver Driver) { } // Open opens a Datastore specified by a configuration. -func Open(cfg config.RegistrableComponentConfig) (Datastore, error) { +func Open(cfg RegistrableComponentConfig) (Datastore, error) { driver, ok := drivers[cfg.Type] if !ok { return nil, fmt.Errorf("database: unknown Driver %q (forgotten configuration or import?)", cfg.Type) diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index 4abd29ee..3353ddcc 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -31,7 +31,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/remind101/migrate" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/database/pgsql/migrations" "github.com/coreos/clair/pkg/commonerr" @@ -114,11 +113,13 @@ type Config struct { FixturePath string } -// openDatabase opens a PostgresSQL-backed Datastore using the given configuration. -// It immediately every necessary migrations. If ManageDatabaseLifecycle is specified, -// the database will be created first. If FixturePath is specified, every SQL queries that are -// present insides will be executed. -func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) (database.Datastore, error) { +// openDatabase opens a PostgresSQL-backed Datastore using the given +// configuration. +// +// It immediately runs all necessary migrations. If ManageDatabaseLifecycle is +// specified, the database will be created first. If FixturePath is specified, +// every SQL queries that are present insides will be executed. +func openDatabase(registrableComponentConfig database.RegistrableComponentConfig) (database.Datastore, error) { var pg pgSQL var err error diff --git a/database/pgsql/pgsql_test.go b/database/pgsql/pgsql_test.go index b9ee97af..93f53144 100644 --- a/database/pgsql/pgsql_test.go +++ b/database/pgsql/pgsql_test.go @@ -21,8 +21,9 @@ import ( "runtime" "strings" - "github.com/coreos/clair/config" "github.com/pborman/uuid" + + "github.com/coreos/clair/database" ) func openDatabaseForTest(testName string, loadFixture bool) (*pgSQL, error) { @@ -34,7 +35,7 @@ func openDatabaseForTest(testName string, loadFixture bool) (*pgSQL, error) { return datastore, nil } -func generateTestConfig(testName string, loadFixture bool) config.RegistrableComponentConfig { +func generateTestConfig(testName string, loadFixture bool) database.RegistrableComponentConfig { dbName := "test_" + strings.ToLower(testName) + "_" + strings.Replace(uuid.New(), "-", "_", -1) var fixturePath string @@ -48,7 +49,7 @@ func generateTestConfig(testName string, loadFixture bool) config.RegistrableCom source = fmt.Sprintf(sourceEnv, dbName) } - return config.RegistrableComponentConfig{ + return database.RegistrableComponentConfig{ Options: map[string]interface{}{ "source": source, "cachesize": 0, diff --git a/ext/notification/driver.go b/ext/notification/driver.go index acd4d4f1..ad1dcc1e 100644 --- a/ext/notification/driver.go +++ b/ext/notification/driver.go @@ -22,10 +22,10 @@ package notification import ( "sync" + "time" "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" ) @@ -36,11 +36,19 @@ var ( senders = make(map[string]Sender) ) +// Config is the configuration for the Notifier service and its registered +// notifiers. +type Config struct { + Attempts int + RenotifyInterval time.Duration + Params map[string]interface{} `yaml:",inline"` +} + // Sender represents anything that can transmit notifications. type Sender interface { // Configure attempts to initialize the notifier with the provided configuration. // It returns whether the notifier is enabled or not. - Configure(*config.NotifierConfig) (bool, error) + Configure(*Config) (bool, error) // Send informs the existence of the specified notification. Send(notification database.VulnerabilityNotification) error diff --git a/ext/notification/webhook/webhook.go b/ext/notification/webhook/webhook.go index 6a991cf4..d54b588b 100644 --- a/ext/notification/webhook/webhook.go +++ b/ext/notification/webhook/webhook.go @@ -29,7 +29,6 @@ import ( "gopkg.in/yaml.v2" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/notification" ) @@ -55,7 +54,7 @@ func init() { notification.RegisterSender("webhook", &sender{}) } -func (s *sender) Configure(config *config.NotifierConfig) (bool, error) { +func (s *sender) Configure(config *notification.Config) (bool, error) { // Get configuration var httpConfig Config if config == nil { diff --git a/notifier.go b/notifier.go index 50158cdd..0586f019 100644 --- a/notifier.go +++ b/notifier.go @@ -21,7 +21,6 @@ import ( "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/notification" "github.com/coreos/clair/pkg/commonerr" @@ -54,7 +53,7 @@ func init() { // RunNotifier begins a process that checks for new notifications that should // be sent out to third parties. -func RunNotifier(config *config.NotifierConfig, datastore database.Datastore, stopper *stopper.Stopper) { +func RunNotifier(config *notification.Config, datastore database.Datastore, stopper *stopper.Stopper) { defer stopper.End() // Configure registered notifiers. diff --git a/updater.go b/updater.go index 41eb8128..d3379cdd 100644 --- a/updater.go +++ b/updater.go @@ -24,7 +24,6 @@ import ( "github.com/pborman/uuid" "github.com/prometheus/client_golang/prometheus" - "github.com/coreos/clair/config" "github.com/coreos/clair/database" "github.com/coreos/clair/ext/vulnmdsrc" "github.com/coreos/clair/ext/vulnsrc" @@ -63,9 +62,14 @@ func init() { prometheus.MustRegister(promUpdaterNotesTotal) } +// UpdaterConfig is the configuration for the Updater service. +type UpdaterConfig struct { + Interval time.Duration +} + // RunUpdater begins a process that updates the vulnerability database at // regular intervals. -func RunUpdater(config *config.UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { +func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. diff --git a/worker_test.go b/worker_test.go index 34d6f039..b0f14dbc 100644 --- a/worker_test.go +++ b/worker_test.go @@ -78,9 +78,9 @@ func TestProcessWithDistUpgrade(t *testing.T) { // wheezy.tar: FROM debian:wheezy // jessie.tar: RUN sed -i "s/precise/trusty/" /etc/apt/sources.list && apt-get update && // apt-get -y dist-upgrade - assert.Nil(t, Process(datastore, "Docker", "blank", "", testDataPath+"blank.tar.gz", nil)) - assert.Nil(t, Process(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil)) - assert.Nil(t, Process(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil)) + assert.Nil(t, ProcessLayer(datastore, "Docker", "blank", "", testDataPath+"blank.tar.gz", nil)) + assert.Nil(t, ProcessLayer(datastore, "Docker", "wheezy", "blank", testDataPath+"wheezy.tar.gz", nil)) + assert.Nil(t, ProcessLayer(datastore, "Docker", "jessie", "wheezy", testDataPath+"jessie.tar.gz", nil)) // Ensure that the 'wheezy' layer has the expected namespace and features. wheezy, ok := datastore.layers["wheezy"]