Add experimental export of vulnerabilities to Grafeas server

cmd/clair/config.go

@@ -31,6 +31,7 @@ import (
 	"github.com/coreos/clair/ext/featurens"
 	"github.com/coreos/clair/ext/notification"
 	"github.com/coreos/clair/ext/vulnsrc"
+	"github.com/coreos/clair/grafeas"
 )
 
 // ErrDatasourceNotLoaded is returned when the datasource variable in the
@@ -50,6 +51,7 @@ type Config struct {
 	Worker   *clair.WorkerConfig
 	Notifier *notification.Config
 	API      *api.Config
+	Grafeas  *grafeas.Config
 }
 
 // DefaultConfig is a configuration that can be used as a fallback value.
@@ -75,6 +77,11 @@ func DefaultConfig() Config {
 			Attempts:         5,
 			RenotifyInterval: 2 * time.Hour,
 		},
+		Grafeas: &grafeas.Config{
+			Enabled:   false,
+			Addr:      "0.0.0.0:8080",
+			ProjectId: "vuln-scanner",
+		},
 	}
 }
 

cmd/clair/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/coreos/clair/ext/featurens"
 	"github.com/coreos/clair/ext/imagefmt"
 	"github.com/coreos/clair/ext/vulnsrc"
+	"github.com/coreos/clair/grafeas"
 	"github.com/coreos/clair/pkg/formatter"
 	"github.com/coreos/clair/pkg/stopper"
 	"github.com/coreos/clair/pkg/strutil"
@@ -133,6 +134,8 @@ func Boot(config *Config) {
 	rand.Seed(time.Now().UnixNano())
 	st := stopper.NewStopper()
 
+	g := grafeas.NewGrafeas(config.Grafeas)
+
 	// Open database
 	var db database.Datastore
 	var dbError error
@@ -162,7 +165,7 @@ func Boot(config *Config) {
 
 	// Start updater
 	st.Begin()
-	go clair.RunUpdater(config.Updater, db, st)
+	go clair.RunUpdater(config.Updater, db, st, g)
 
 	// Wait for interruption and shutdown gracefully.
 	waitForSignals(syscall.SIGINT, syscall.SIGTERM)

config.yaml.sample

@@ -97,3 +97,8 @@ clair:
 
       # Optional HTTP Proxy: must be a valid URL (including the scheme).
       proxy:
+
+  grafeas:
+    enabled: false
+    addr: "http://grafeas.example.com"
+    projectId: "vuln-scanner"

database/database.go

@@ -148,6 +148,7 @@ type Session interface {
 	// already in the database.
 	InsertVulnerabilities([]VulnerabilityWithAffected) error
 
+	ListVulnerabilities() ([]NullableVulnerability, error)
 	// FindVulnerability retrieves a set of Vulnerabilities with affected
 	// features.
 	FindVulnerabilities([]VulnerabilityID) ([]NullableVulnerability, error)

database/pgsql/queries.go

@@ -100,6 +100,13 @@ const (
 	removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP`
 
 	// vulnerability.go
+	listVulnerabilities = `
+		SELECT v.id, v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format
+		FROM vulnerability AS v, namespace AS n
+		WHERE v.namespace_id = n.id
+		AND v.deleted_at IS NULL
+		`
+
 	searchVulnerability = `
 		SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format 
 		FROM vulnerability AS v, namespace AS n

database/pgsql/vulnerability.go

@@ -46,6 +46,93 @@ type affectedFeatureRows struct {
 	rows map[int64]database.AffectedFeature
 }
 
+func (tx *pgSession) ListVulnerabilities() ([]database.NullableVulnerability, error) {
+	defer observeQueryTime("listVulnerabilities", "", time.Now())
+	vulnIDMap := map[int64][]*database.NullableVulnerability{}
+
+	stmt, err := tx.Prepare(listVulnerabilities)
+	if err != nil {
+		return nil, err
+	}
+
+	rows, err := stmt.Query()
+
+	if err != nil && err != sql.ErrNoRows {
+		stmt.Close()
+		return nil, handleError("listVulnerabilities", err)
+	}
+	defer rows.Close()
+	// load vulnerabilities
+	for rows.Next() {
+		var (
+			id   sql.NullInt64
+			vuln = database.NullableVulnerability{}
+		)
+
+		err := rows.Scan(
+			&id,
+			&vuln.Name,
+			&vuln.Description,
+			&vuln.Link,
+			&vuln.Severity,
+			&vuln.Metadata,
+			&vuln.Namespace.Name,
+			&vuln.Namespace.VersionFormat,
+		)
+
+		if err != nil && err != sql.ErrNoRows {
+			stmt.Close()
+			return nil, handleError("searchVulnerability", err)
+		}
+
+		vuln.Valid = id.Valid
+		if id.Valid {
+			vulnIDMap[id.Int64] = append(vulnIDMap[id.Int64], &vuln)
+		}
+	}
+
+	if err := stmt.Close(); err != nil {
+		return nil, handleError("listVulnerabilities", err)
+	}
+
+	toQuery := make([]int64, 0, len(vulnIDMap))
+	for id := range vulnIDMap {
+		toQuery = append(toQuery, id)
+	}
+
+	// load vulnerability affected features
+	rows, err = tx.Query(searchVulnerabilityAffected, pq.Array(toQuery))
+	if err != nil {
+		return nil, handleError("searchVulnerabilityAffected", err)
+	}
+
+	for rows.Next() {
+		var (
+			id int64
+			f  database.AffectedFeature
+		)
+
+		err := rows.Scan(&id, &f.FeatureName, &f.AffectedVersion, &f.FixedInVersion)
+		if err != nil {
+			return nil, handleError("searchVulnerabilityAffected", err)
+		}
+
+		for _, vuln := range vulnIDMap[id] {
+			f.Namespace = vuln.Namespace
+			vuln.Affected = append(vuln.Affected, f)
+		}
+	}
+
+	var resultVuln []database.NullableVulnerability
+	for _, vulns := range vulnIDMap {
+		for _, vuln := range vulns {
+			resultVuln = append(resultVuln, *vuln)
+		}
+	}
+
+	return resultVuln, nil
+}
+
 func (tx *pgSession) FindVulnerabilities(vulnerabilities []database.VulnerabilityID) ([]database.NullableVulnerability, error) {
 	defer observeQueryTime("findVulnerabilities", "", time.Now())
 	resultVuln := make([]database.NullableVulnerability, len(vulnerabilities))

grafeas/grafeas.go

@@ -0,0 +1,198 @@
+package grafeas
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/coreos/clair/database"
+	"github.com/grafeas/client-go/v1alpha1"
+	log "github.com/sirupsen/logrus"
+)
+
+type Config struct {
+	Enabled   bool
+	Addr      string
+	ProjectId string
+}
+
+type Grafeas struct {
+	Config *Config
+}
+
+func NewGrafeas(config *Config) Grafeas {
+	return Grafeas{config}
+}
+
+func (g *Grafeas) Export(datastore database.Datastore) error {
+	if !g.Config.Enabled {
+		return nil
+	}
+
+	log.Info("exporting vulnerabilities to Grafeas")
+
+	tx, err := datastore.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer tx.Rollback()
+	vulnerabilities, err := tx.ListVulnerabilities()
+	if err != nil {
+		return err
+	}
+
+	pID := g.Config.ProjectId
+	client := v1alpha1.NewGrafeasApiWithBasePath(g.Config.Addr)
+	for _, vuln := range vulnerabilities {
+		nID := vuln.Name
+		score, nistVectors := extractMetadata(vuln.Metadata)
+		note, _, err := client.GetNote(pID, nID)
+		createNewNote := false
+		if err != nil {
+			n := noteWithoutDetails(pID, nID, vuln.Description, string(vuln.Severity), nistVectors, score)
+			note = &n
+			createNewNote = true
+		}
+
+		containsUpdatedDetail := false
+		for _, affected := range vuln.Affected {
+			cpeUri := createCpeUri(affected.Namespace.Name)
+			detail := detail(cpeUri, affected.FeatureName, vuln.Description, string(vuln.Severity), affected.FixedInVersion)
+			index := findDetail(note.VulnerabilityType.Details, detail)
+			if index == -1 {
+				note.VulnerabilityType.Details = append(note.VulnerabilityType.Details, detail)
+				containsUpdatedDetail = true
+			} else if !reflect.DeepEqual(note.VulnerabilityType.Details[index], detail) {
+				note.VulnerabilityType.Details[index] = detail
+				containsUpdatedDetail = true
+			}
+		}
+
+		if createNewNote {
+			_, _, err = client.CreateNote(pID, nID, *note)
+		} else if containsUpdatedDetail {
+			_, _, err = client.UpdateNote(pID, nID, *note)
+		}
+
+		if err != nil {
+			log.Warn("Error creating note %v", err)
+		}
+	}
+
+	log.Info("export done")
+
+	return nil
+}
+
+func extractMetadata(metadata map[string]interface{}) (score float32, vectors string) {
+	if nvd, ok := metadata["NVD"].(map[string]interface{}); ok {
+		if cvss, ok := nvd["CVSSv2"].(map[string]interface{}); ok {
+			score = float32(cvss["Score"].(float64))
+			vectors = cvss["Vectors"].(string)
+		}
+	}
+
+	return score, vectors
+}
+
+// Clair does not report cpe uri:s so we'll have to create one from the namespace.
+func createCpeUri(namespaceName string) string {
+	ss := strings.Split(namespaceName, ":")
+	if len(ss) != 2 {
+		return "CPE_UNSPECIFIED"
+	}
+	os := ss[0]
+	ver := ss[1]
+
+	switch os {
+	case "alpine":
+		return "cpe:/o:alpine:alpine_linux:" + ver
+	case "debian":
+		return "cpe:/o:debian:debian_linux:" + ver
+	case "ubuntu":
+		return "cpe:/o:canonical:ubuntu_linux:" + ver
+	case "centos":
+		return "cpe:/o:centos:centos:" + ver
+	case "rhel":
+		return "cpe:/o:redhat:enterprise_linux:" + ver
+	case "fedora":
+		return "cpe:/o:fedoraproject:fedora:" + ver
+	}
+
+	return "CPE_UNSPECIFIED"
+}
+
+func findDetail(details []v1alpha1.Detail, detail v1alpha1.Detail) int {
+	for i, d := range details {
+		if d.CpeUri == detail.CpeUri && d.Package_ == detail.Package_ {
+			return i
+		}
+	}
+	return -1
+}
+
+func fixedLocation(fixedBy, cpeUri, pkg string) v1alpha1.VulnerabilityLocation {
+	var version v1alpha1.Version
+	if fixedBy == "" {
+		version = v1alpha1.Version{
+			Kind: "MAXIMUM",
+		}
+	} else {
+		// EVR: Epoch:Version.Revision
+		evrRegexp := regexp.MustCompile(`(?:(\d*):)?([\w~]+[\w.~]*)-(~?\w+[\w.]*)`)
+		matches := evrRegexp.FindStringSubmatch(fixedBy)
+		if len(matches) != 0 {
+			versionEpoch, _ := strconv.ParseInt(matches[1], 10, 32)
+			versionName := matches[2]
+			versionRev := matches[3]
+			version = v1alpha1.Version{
+				Epoch:    int32(versionEpoch),
+				Name:     versionName,
+				Revision: versionRev,
+			}
+		} else {
+			version = v1alpha1.Version{
+				Name: fixedBy,
+			}
+		}
+	}
+	return v1alpha1.VulnerabilityLocation{
+		CpeUri:   cpeUri,
+		Package_: pkg,
+		Version:  version,
+	}
+}
+
+func detail(cpeUri, packageName, description, severity, fixedBy string) v1alpha1.Detail {
+	return v1alpha1.Detail{
+		CpeUri:      cpeUri,
+		Package_:    packageName,
+		Description: description,
+		MinAffectedVersion: v1alpha1.Version{
+			Kind: "MINIMUM",
+		},
+		SeverityName:  severity,
+		FixedLocation: fixedLocation(fixedBy, cpeUri, packageName),
+	}
+}
+
+func noteWithoutDetails(pID, name, description, severity, nistVectors string, score float32) v1alpha1.Note {
+	var longDescription string
+	if nistVectors != "" {
+		longDescription = fmt.Sprintf("NIST vectors: %v", nistVectors)
+	}
+	return v1alpha1.Note{
+		Name:             fmt.Sprintf("projects/%v/notes/%v", pID, name),
+		ShortDescription: name,
+		LongDescription:  longDescription,
+		Kind:             "PACKAGE_VULNERABILITY",
+		VulnerabilityType: v1alpha1.VulnerabilityType{
+			CvssScore: score,
+			Severity:  severity,
+			Details:   []v1alpha1.Detail{},
+		},
+	}
+}

updater.go

@@ -28,6 +28,7 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/vulnmdsrc"
 	"github.com/coreos/clair/ext/vulnsrc"
+	"github.com/coreos/clair/grafeas"
 	"github.com/coreos/clair/pkg/stopper"
 )
 
@@ -78,7 +79,7 @@ type vulnerabilityChange struct {
 
 // RunUpdater begins a process that updates the vulnerability database at
 // regular intervals.
-func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) {
+func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper, g grafeas.Grafeas) {
 	defer st.End()
 
 	// Do not run the updater if there is no config or if the interval is 0.
@@ -129,6 +130,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
 					}
 				}
 
+				g.Export(datastore)
+
 				// Unlock the updater.
 				unlock(datastore, updaterLockName, whoAmI)