From ee37c47a4b83b91b8e3ab8c71e45aa4a4aef4788 Mon Sep 17 00:00:00 2001 From: Avi Miller Date: Tue, 21 Mar 2017 15:16:45 +1100 Subject: [PATCH] Implements #221 This adds a configuration section to config.yaml which allows users to enable/disable each fetcher at runtime. Signed-off-by: Avi Miller --- cmd/clair/config.go | 3 +++ cmd/clair/main.go | 2 +- config.example.yaml | 16 ++++++++++++++++ ext/vulnsrc/alpine/alpine.go | 30 ++++++++++++++++++++++++++++++ ext/vulnsrc/debian/debian.go | 32 ++++++++++++++++++++++++++++++++ ext/vulnsrc/driver.go | 14 ++++++++++++++ ext/vulnsrc/oracle/oracle.go | 31 +++++++++++++++++++++++++++++++ ext/vulnsrc/rhel/rhel.go | 32 ++++++++++++++++++++++++++++++++ ext/vulnsrc/ubuntu/ubuntu.go | 32 ++++++++++++++++++++++++++++++++ updater.go | 16 +++++++++++++++- 10 files changed, 206 insertions(+), 2 deletions(-) diff --git a/cmd/clair/config.go b/cmd/clair/config.go index c11e971f..e1ac46e4 100644 --- a/cmd/clair/config.go +++ b/cmd/clair/config.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/clair" "github.com/coreos/clair/api" "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/vulnsrc" "github.com/coreos/clair/ext/notification" "github.com/fernet/fernet-go" ) @@ -43,6 +44,7 @@ type File struct { type Config struct { Database database.RegistrableComponentConfig Updater *clair.UpdaterConfig + Fetcher *vulnsrc.Config Notifier *notification.Config API *api.Config } @@ -61,6 +63,7 @@ func DefaultConfig() Config { HealthPort: 6061, Timeout: 900 * time.Second, }, + Fetcher: &vulnsrc.Config {}, Notifier: ¬ification.Config{ Attempts: 5, RenotifyInterval: 2 * time.Hour, diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 11f0de4c..983a638e 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -110,7 +110,7 @@ func Boot(config *Config) { // Start updater st.Begin() - go clair.RunUpdater(config.Updater, db, st) + go clair.RunUpdater(config.Updater, config.Fetcher, db, st) // Wait for interruption and shutdown gracefully. waitForSignals(syscall.SIGINT, syscall.SIGTERM) diff --git a/config.example.yaml b/config.example.yaml index 423068a3..658b5da0 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -56,6 +56,22 @@ clair: # The value 0 disables the updater entirely. interval: 2h + fetcher: + # You can enable or disable individual fetchers in this section. This is useful for reducing + # overall update times by disabling the distributions you will never scan + # The default if a distribution is unspecified/unconfigured is true/enabled. + alpine: + enabled: true + debian: + enabled: true + oracle: + enabled: true + rhel: + enabled: true + ubuntu: + enabled: true + + notifier: # Number of attempts before the notification is marked as failed to be sent attempts: 3 diff --git a/ext/vulnsrc/alpine/alpine.go b/ext/vulnsrc/alpine/alpine.go index e99926ba..4fe758a9 100644 --- a/ext/vulnsrc/alpine/alpine.go +++ b/ext/vulnsrc/alpine/alpine.go @@ -19,6 +19,7 @@ package alpine import ( "io" "io/ioutil" + "errors" "os" "os/exec" "path/filepath" @@ -48,10 +49,39 @@ func init() { vulnsrc.RegisterUpdater("alpine", &updater{}) } +type Config struct { + Enabled bool +} + type updater struct { repositoryLocalPath string } +func (u *updater) Configure(config *vulnsrc.Config) (bool, error) { + var fetcherConfig Config + + // If no configuration for this fetcher, assume enabled + if _, ok := config.Params["alpine"]; !ok { + return true, nil + } + yamlConfig, err := yaml.Marshal(config.Params["alpine"]) + if err != nil { + return false, errors.New("Invalid configuration for Alpine Linux fetcher.") + } + err = yaml.Unmarshal(yamlConfig, &fetcherConfig) + if err != nil { + return false, errors.New("Invalid configuration for Alpine Linux fetcher.") + } + + if fetcherConfig.Enabled == true { + return true, nil + } else { + log.Infof("Alpine Linux fetcher disabled.") + return false, nil + } +} + + func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Alpine vulnerabilities") diff --git a/ext/vulnsrc/debian/debian.go b/ext/vulnsrc/debian/debian.go index ec917fe3..d17de2c4 100644 --- a/ext/vulnsrc/debian/debian.go +++ b/ext/vulnsrc/debian/debian.go @@ -20,11 +20,14 @@ import ( "crypto/sha1" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" "strings" + "gopkg.in/yaml.v2" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" @@ -55,12 +58,41 @@ type jsonRel struct { Urgency string `json:"urgency"` } +type Config struct { + Enabled bool +} + type updater struct{} func init() { vulnsrc.RegisterUpdater("debian", &updater{}) } +func (u *updater) Configure(config *vulnsrc.Config) (bool, error) { + var fetcherConfig Config + + // If no configuration for this fetcher, assume enabled + if _, ok := config.Params["debian"]; !ok { + return true, nil + } + yamlConfig, err := yaml.Marshal(config.Params["debian"]) + if err != nil { + return false, errors.New("Invalid configuration for Debian fetcher.") + } + err = yaml.Unmarshal(yamlConfig, &fetcherConfig) + if err != nil { + return false, errors.New("Invalid configuration for Debian fetcher.") + } + + if fetcherConfig.Enabled == true { + return true, nil + } else { + log.Infof("Debian fetcher disabled.") + return false, nil + } +} + + func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Debian vulnerabilities") diff --git a/ext/vulnsrc/driver.go b/ext/vulnsrc/driver.go index fd442416..048c9204 100644 --- a/ext/vulnsrc/driver.go +++ b/ext/vulnsrc/driver.go @@ -42,9 +42,16 @@ type UpdateResponse struct { Vulnerabilities []database.Vulnerability } +type Config struct { + Params map[string]interface{} `yaml:",inline"` +} + // Updater represents anything that can fetch vulnerabilities and insert them // into a Clair datastore. type Updater interface { + // Check configuration for Updater + Configure(*Config) (bool, error) + // Update gets vulnerability updates. Update(database.Datastore) (UpdateResponse, error) @@ -76,6 +83,13 @@ func RegisterUpdater(name string, u Updater) { updaters[name] = u } +func UnregisterUpdater(name string) { + updatersM.Lock() + defer updatersM.Unlock() + + delete(updaters, name) +} + // Updaters returns the list of the registered Updaters. func Updaters() map[string]Updater { updatersM.RLock() diff --git a/ext/vulnsrc/oracle/oracle.go b/ext/vulnsrc/oracle/oracle.go index 40704096..e4bbda7f 100644 --- a/ext/vulnsrc/oracle/oracle.go +++ b/ext/vulnsrc/oracle/oracle.go @@ -19,12 +19,15 @@ package oracle import ( "bufio" "encoding/xml" + "errors" "io" "net/http" "regexp" "strconv" "strings" + "gopkg.in/yaml.v2" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" @@ -79,12 +82,40 @@ type criterion struct { Comment string `xml:"comment,attr"` } +type Config struct { + Enabled bool +} + type updater struct{} func init() { vulnsrc.RegisterUpdater("oracle", &updater{}) } +func (u *updater) Configure(config *vulnsrc.Config) (bool, error) { + var fetcherConfig Config + + // If no configuration for this fetcher, assume enabled + if _, ok := config.Params["oracle"]; !ok { + return true, nil + } + yamlConfig, err := yaml.Marshal(config.Params["oracle"]) + if err != nil { + return false, errors.New("Invalid configuration for Oracle Linux fetcher.") + } + err = yaml.Unmarshal(yamlConfig, &fetcherConfig) + if err != nil { + return false, errors.New("Invalid configuration for Oracle Linux fetcher.") + } + + if fetcherConfig.Enabled == true { + return true, nil + } else { + log.Infof("Oracle Linux fetcher disabled.") + return false, nil + } +} + func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Oracle Linux vulnerabilities") diff --git a/ext/vulnsrc/rhel/rhel.go b/ext/vulnsrc/rhel/rhel.go index ea1e9c3d..72973220 100644 --- a/ext/vulnsrc/rhel/rhel.go +++ b/ext/vulnsrc/rhel/rhel.go @@ -19,12 +19,15 @@ package rhel import ( "bufio" "encoding/xml" + "errors" "io" "net/http" "regexp" "strconv" "strings" + "gopkg.in/yaml.v2" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" @@ -83,12 +86,41 @@ type criterion struct { Comment string `xml:"comment,attr"` } +type Config struct { + Enabled bool +} + type updater struct{} func init() { vulnsrc.RegisterUpdater("rhel", &updater{}) } +func (u *updater) Configure(config *vulnsrc.Config) (bool, error) { + var fetcherConfig Config + + // If no configuration for this fetcher, assume enabled + if _, ok := config.Params["rhel"]; !ok { + return true, nil + } + yamlConfig, err := yaml.Marshal(config.Params["rhel"]) + if err != nil { + return false, errors.New("Invalid configuration for RHEL fetcher.") + } + err = yaml.Unmarshal(yamlConfig, &fetcherConfig) + if err != nil { + return false, errors.New("Invalid configuration for RHEL fetcher.") + } + + if fetcherConfig.Enabled == true { + return true, nil + } else { + log.Infof("RHEL fetcher disabled.") + return false, nil + } +} + + func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching RHEL vulnerabilities") diff --git a/ext/vulnsrc/ubuntu/ubuntu.go b/ext/vulnsrc/ubuntu/ubuntu.go index 6ef3633a..0451327c 100644 --- a/ext/vulnsrc/ubuntu/ubuntu.go +++ b/ext/vulnsrc/ubuntu/ubuntu.go @@ -19,6 +19,7 @@ package ubuntu import ( "bufio" "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -28,6 +29,8 @@ import ( "strconv" "strings" + "gopkg.in/yaml.v2" + "github.com/coreos/pkg/capnslog" "github.com/coreos/clair/database" @@ -82,10 +85,39 @@ type updater struct { repositoryLocalPath string } +type Config struct { + Enabled bool +} + func init() { vulnsrc.RegisterUpdater("ubuntu", &updater{}) } +func (u *updater) Configure(config *vulnsrc.Config) (bool, error) { + var fetcherConfig Config + + // If no configuration for this fetcher, assume enabled + if _, ok := config.Params["ubuntu"]; !ok { + return true, nil + } + yamlConfig, err := yaml.Marshal(config.Params["ubuntu"]) + if err != nil { + return false, errors.New("Invalid configuration for Ubuntu fetcher.") + } + err = yaml.Unmarshal(yamlConfig, &fetcherConfig) + if err != nil { + return false, errors.New("Invalid configuration for Ubuntu fetcher.") + } + + if fetcherConfig.Enabled == true { + return true, nil + } else { + log.Infof("Ubuntu fetcher disabled.") + return false, nil + } +} + + func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) { log.Info("fetching Ubuntu vulnerabilities") diff --git a/updater.go b/updater.go index a404dceb..a6e1f2fc 100644 --- a/updater.go +++ b/updater.go @@ -69,7 +69,7 @@ type UpdaterConfig struct { // RunUpdater begins a process that updates the vulnerability database at // regular intervals. -func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) { +func RunUpdater(config *UpdaterConfig, fetcherConfig *vulnsrc.Config, datastore database.Datastore, st *stopper.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. @@ -81,6 +81,20 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper whoAmI := uuid.New() log.Infof("updater service started. lock identifier: %s", whoAmI) + for updaterName, updater := range vulnsrc.Updaters() { + if configured, err := updater.Configure(fetcherConfig); !configured { + vulnsrc.UnregisterUpdater(updaterName) + if err != nil { + log.Errorf("could not configure updater '%s': %s'", updaterName, err) + } + } + } + + if len(vulnsrc.Updaters()) == 0 { + log.Infof("updater service is disabled") + return + } + for { var stop bool