Implements #221

This adds a configuration section to config.yaml which allows users
to enable/disable each fetcher at runtime.

Signed-off-by: Avi Miller <avi.miller@oracle.com>

cmd/clair/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/api"
 	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/ext/notification"
 	"github.com/fernet/fernet-go"
 )
@@ -43,6 +44,7 @@ type File struct {
 type Config struct {
 	Database database.RegistrableComponentConfig
 	Updater  *clair.UpdaterConfig
+	Fetcher  *vulnsrc.Config
 	Notifier *notification.Config
 	API      *api.Config
 }
@@ -61,6 +63,7 @@ func DefaultConfig() Config {
 			HealthPort: 6061,
 			Timeout:    900 * time.Second,
 		},
+		Fetcher: &vulnsrc.Config {},
 		Notifier: &notification.Config{
 			Attempts:         5,
 			RenotifyInterval: 2 * time.Hour,

cmd/clair/main.go

@@ -110,7 +110,7 @@ func Boot(config *Config) {
 
 	// Start updater
 	st.Begin()
-	go clair.RunUpdater(config.Updater, db, st)
+	go clair.RunUpdater(config.Updater, config.Fetcher, db, st)
 
 	// Wait for interruption and shutdown gracefully.
 	waitForSignals(syscall.SIGINT, syscall.SIGTERM)

config.example.yaml

@@ -56,6 +56,22 @@ clair:
     # The value 0 disables the updater entirely.
     interval: 2h
 
+  fetcher:
+    # You can enable or disable individual fetchers in this section. This is useful for reducing
+    # overall update times by disabling the distributions you will never scan
+    # The default if a distribution is unspecified/unconfigured is true/enabled.
+    alpine:
+      enabled: true
+    debian:
+      enabled: true
+    oracle:
+      enabled: true
+    rhel:
+      enabled: true
+    ubuntu:
+      enabled: true
+
+
   notifier:
     # Number of attempts before the notification is marked as failed to be sent
     attempts: 3

ext/vulnsrc/alpine/alpine.go

@@ -19,6 +19,7 @@ package alpine
 import (
 	"io"
 	"io/ioutil"
+	"errors"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -48,10 +49,39 @@ func init() {
 	vulnsrc.RegisterUpdater("alpine", &updater{})
 }
 
+type Config struct {
+	Enabled bool
+}
+
 type updater struct {
 	repositoryLocalPath string
 }
 
+func (u *updater) Configure(config *vulnsrc.Config) (bool, error) {
+        var fetcherConfig Config
+
+        // If no configuration for this fetcher, assume enabled
+        if _, ok := config.Params["alpine"]; !ok {
+                return true, nil
+        }
+        yamlConfig, err := yaml.Marshal(config.Params["alpine"])
+        if err != nil {
+                return false, errors.New("Invalid configuration for Alpine Linux fetcher.")
+        }
+        err = yaml.Unmarshal(yamlConfig, &fetcherConfig)
+        if err != nil {
+                return false, errors.New("Invalid configuration for Alpine Linux fetcher.")
+        }
+
+        if fetcherConfig.Enabled == true {
+                return true, nil
+        } else {
+                log.Infof("Alpine Linux fetcher disabled.")
+                return false, nil
+        }
+}
+
+
 func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching Alpine vulnerabilities")
 

ext/vulnsrc/debian/debian.go

@@ -20,11 +20,14 @@ import (
 	"crypto/sha1"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
@@ -55,12 +58,41 @@ type jsonRel struct {
 	Urgency      string `json:"urgency"`
 }
 
+type Config struct {
+	Enabled bool
+}
+
 type updater struct{}
 
 func init() {
 	vulnsrc.RegisterUpdater("debian", &updater{})
 }
 
+func (u *updater) Configure(config *vulnsrc.Config) (bool, error) {
+        var fetcherConfig Config
+
+        // If no configuration for this fetcher, assume enabled
+        if _, ok := config.Params["debian"]; !ok {
+                return true, nil
+        }
+        yamlConfig, err := yaml.Marshal(config.Params["debian"])
+        if err != nil {
+                return false, errors.New("Invalid configuration for Debian fetcher.")
+        }
+        err = yaml.Unmarshal(yamlConfig, &fetcherConfig)
+        if err != nil {
+                return false, errors.New("Invalid configuration for Debian fetcher.")
+        }
+
+        if fetcherConfig.Enabled == true {
+                return true, nil
+        } else {
+                log.Infof("Debian fetcher disabled.")
+                return false, nil
+        }
+}
+
+
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching Debian vulnerabilities")
 

ext/vulnsrc/driver.go

@@ -42,9 +42,16 @@ type UpdateResponse struct {
 	Vulnerabilities []database.Vulnerability
 }
 
+type Config struct {
+	Params   map[string]interface{} `yaml:",inline"`
+}
+
 // Updater represents anything that can fetch vulnerabilities and insert them
 // into a Clair datastore.
 type Updater interface {
+	// Check configuration for Updater
+	Configure(*Config) (bool, error)
+
 	// Update gets vulnerability updates.
 	Update(database.Datastore) (UpdateResponse, error)
 
@@ -76,6 +83,13 @@ func RegisterUpdater(name string, u Updater) {
 	updaters[name] = u
 }
 
+func UnregisterUpdater(name string) {
+	updatersM.Lock()
+	defer updatersM.Unlock()
+
+	delete(updaters, name)
+}
+
 // Updaters returns the list of the registered Updaters.
 func Updaters() map[string]Updater {
 	updatersM.RLock()

ext/vulnsrc/oracle/oracle.go

@@ -19,12 +19,15 @@ package oracle
 import (
 	"bufio"
 	"encoding/xml"
+	"errors"
 	"io"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
@@ -79,12 +82,40 @@ type criterion struct {
 	Comment string `xml:"comment,attr"`
 }
 
+type Config struct {
+	Enabled bool
+}
+
 type updater struct{}
 
 func init() {
 	vulnsrc.RegisterUpdater("oracle", &updater{})
 }
 
+func (u *updater) Configure(config *vulnsrc.Config) (bool, error) {
+	var fetcherConfig Config
+
+	// If no configuration for this fetcher, assume enabled
+	if _, ok := config.Params["oracle"]; !ok {
+		return true, nil
+	}
+	yamlConfig, err := yaml.Marshal(config.Params["oracle"])
+	if err != nil {
+		return false, errors.New("Invalid configuration for Oracle Linux fetcher.")
+	}
+	err = yaml.Unmarshal(yamlConfig, &fetcherConfig)
+	if err != nil {
+		return false, errors.New("Invalid configuration for Oracle Linux fetcher.")
+	}
+
+	if fetcherConfig.Enabled == true {
+		return true, nil
+	} else {
+		log.Infof("Oracle Linux fetcher disabled.")
+		return false, nil
+	}
+}
+
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching Oracle Linux vulnerabilities")
 

ext/vulnsrc/rhel/rhel.go

@@ -19,12 +19,15 @@ package rhel
 import (
 	"bufio"
 	"encoding/xml"
+	"errors"
 	"io"
 	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
@@ -83,12 +86,41 @@ type criterion struct {
 	Comment string `xml:"comment,attr"`
 }
 
+type Config struct {
+	Enabled bool
+}
+
 type updater struct{}
 
 func init() {
 	vulnsrc.RegisterUpdater("rhel", &updater{})
 }
 
+func (u *updater) Configure(config *vulnsrc.Config) (bool, error) {
+        var fetcherConfig Config
+
+        // If no configuration for this fetcher, assume enabled
+        if _, ok := config.Params["rhel"]; !ok {
+                return true, nil
+        }
+        yamlConfig, err := yaml.Marshal(config.Params["rhel"])
+        if err != nil {
+                return false, errors.New("Invalid configuration for RHEL fetcher.")
+        }
+        err = yaml.Unmarshal(yamlConfig, &fetcherConfig)
+        if err != nil {
+                return false, errors.New("Invalid configuration for RHEL fetcher.")
+        }
+
+        if fetcherConfig.Enabled == true {
+                return true, nil
+        } else {
+                log.Infof("RHEL fetcher disabled.")
+                return false, nil
+        }
+}
+
+
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching RHEL vulnerabilities")
 

ext/vulnsrc/ubuntu/ubuntu.go

@@ -19,6 +19,7 @@ package ubuntu
 import (
 	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -28,6 +29,8 @@ import (
 	"strconv"
 	"strings"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
@@ -82,10 +85,39 @@ type updater struct {
 	repositoryLocalPath string
 }
 
+type Config struct {
+	Enabled bool
+}
+
 func init() {
 	vulnsrc.RegisterUpdater("ubuntu", &updater{})
 }
 
+func (u *updater) Configure(config *vulnsrc.Config) (bool, error) {
+        var fetcherConfig Config
+
+        // If no configuration for this fetcher, assume enabled
+        if _, ok := config.Params["ubuntu"]; !ok {
+                return true, nil
+        }
+        yamlConfig, err := yaml.Marshal(config.Params["ubuntu"])
+        if err != nil {
+                return false, errors.New("Invalid configuration for Ubuntu fetcher.")
+        }
+        err = yaml.Unmarshal(yamlConfig, &fetcherConfig)
+        if err != nil {
+                return false, errors.New("Invalid configuration for Ubuntu fetcher.")
+        }
+
+        if fetcherConfig.Enabled == true {
+                return true, nil
+        } else {
+                log.Infof("Ubuntu fetcher disabled.")
+                return false, nil
+        }
+}
+
+
 func (u *updater) Update(datastore database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.Info("fetching Ubuntu vulnerabilities")
 

updater.go

@@ -69,7 +69,7 @@ type UpdaterConfig struct {
 
 // RunUpdater begins a process that updates the vulnerability database at
 // regular intervals.
-func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) {
+func RunUpdater(config *UpdaterConfig, fetcherConfig *vulnsrc.Config, datastore database.Datastore, st *stopper.Stopper) {
 	defer st.End()
 
 	// Do not run the updater if there is no config or if the interval is 0.
@@ -81,6 +81,20 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 	whoAmI := uuid.New()
 	log.Infof("updater service started. lock identifier: %s", whoAmI)
 
+	for updaterName, updater := range vulnsrc.Updaters() {
+		if configured, err := updater.Configure(fetcherConfig); !configured {
+			vulnsrc.UnregisterUpdater(updaterName)
+			if err != nil {
+				log.Errorf("could not configure updater '%s': %s'", updaterName, err)
+			}
+		}
+	}
+
+	if len(vulnsrc.Updaters()) == 0 {
+		log.Infof("updater service is disabled")
+		return
+	}
+
 	for {
 		var stop bool