Add experimental export of vulnerabilities to Grafeas server

pull/501/head
Tobias Furuholm 7 years ago
parent db6379bc9e
commit 8397359296

@ -31,6 +31,7 @@ import (
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/notification"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/grafeas"
)
// ErrDatasourceNotLoaded is returned when the datasource variable in the
@ -50,6 +51,7 @@ type Config struct {
Worker *clair.WorkerConfig
Notifier *notification.Config
API *api.Config
Grafeas *grafeas.Config
}
// DefaultConfig is a configuration that can be used as a fallback value.
@ -75,6 +77,11 @@ func DefaultConfig() Config {
Attempts: 5,
RenotifyInterval: 2 * time.Hour,
},
Grafeas: &grafeas.Config{
Enabled: false,
Addr: "0.0.0.0:8080",
ProjectId: "vuln-scanner",
},
}
}

@ -34,6 +34,7 @@ import (
"github.com/coreos/clair/ext/featurens"
"github.com/coreos/clair/ext/imagefmt"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/grafeas"
"github.com/coreos/clair/pkg/formatter"
"github.com/coreos/clair/pkg/stopper"
"github.com/coreos/clair/pkg/strutil"
@ -133,6 +134,8 @@ func Boot(config *Config) {
rand.Seed(time.Now().UnixNano())
st := stopper.NewStopper()
g := grafeas.NewGrafeas(config.Grafeas)
// Open database
var db database.Datastore
var dbError error
@ -162,7 +165,7 @@ func Boot(config *Config) {
// Start updater
st.Begin()
go clair.RunUpdater(config.Updater, db, st)
go clair.RunUpdater(config.Updater, db, st, g)
// Wait for interruption and shutdown gracefully.
waitForSignals(syscall.SIGINT, syscall.SIGTERM)

@ -97,3 +97,8 @@ clair:
# Optional HTTP Proxy: must be a valid URL (including the scheme).
proxy:
grafeas:
enabled: false
addr: "http://grafeas.example.com"
projectId: "vuln-scanner"

@ -148,6 +148,7 @@ type Session interface {
// already in the database.
InsertVulnerabilities([]VulnerabilityWithAffected) error
ListVulnerabilities() ([]NullableVulnerability, error)
// FindVulnerability retrieves a set of Vulnerabilities with affected
// features.
FindVulnerabilities([]VulnerabilityID) ([]NullableVulnerability, error)

@ -100,6 +100,13 @@ const (
removeLockExpired = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP`
// vulnerability.go
listVulnerabilities = `
SELECT v.id, v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format
FROM vulnerability AS v, namespace AS n
WHERE v.namespace_id = n.id
AND v.deleted_at IS NULL
`
searchVulnerability = `
SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format
FROM vulnerability AS v, namespace AS n

@ -46,6 +46,93 @@ type affectedFeatureRows struct {
rows map[int64]database.AffectedFeature
}
func (tx *pgSession) ListVulnerabilities() ([]database.NullableVulnerability, error) {
defer observeQueryTime("listVulnerabilities", "", time.Now())
vulnIDMap := map[int64][]*database.NullableVulnerability{}
stmt, err := tx.Prepare(listVulnerabilities)
if err != nil {
return nil, err
}
rows, err := stmt.Query()
if err != nil && err != sql.ErrNoRows {
stmt.Close()
return nil, handleError("listVulnerabilities", err)
}
defer rows.Close()
// load vulnerabilities
for rows.Next() {
var (
id sql.NullInt64
vuln = database.NullableVulnerability{}
)
err := rows.Scan(
&id,
&vuln.Name,
&vuln.Description,
&vuln.Link,
&vuln.Severity,
&vuln.Metadata,
&vuln.Namespace.Name,
&vuln.Namespace.VersionFormat,
)
if err != nil && err != sql.ErrNoRows {
stmt.Close()
return nil, handleError("searchVulnerability", err)
}
vuln.Valid = id.Valid
if id.Valid {
vulnIDMap[id.Int64] = append(vulnIDMap[id.Int64], &vuln)
}
}
if err := stmt.Close(); err != nil {
return nil, handleError("listVulnerabilities", err)
}
toQuery := make([]int64, 0, len(vulnIDMap))
for id := range vulnIDMap {
toQuery = append(toQuery, id)
}
// load vulnerability affected features
rows, err = tx.Query(searchVulnerabilityAffected, pq.Array(toQuery))
if err != nil {
return nil, handleError("searchVulnerabilityAffected", err)
}
for rows.Next() {
var (
id int64
f database.AffectedFeature
)
err := rows.Scan(&id, &f.FeatureName, &f.AffectedVersion, &f.FixedInVersion)
if err != nil {
return nil, handleError("searchVulnerabilityAffected", err)
}
for _, vuln := range vulnIDMap[id] {
f.Namespace = vuln.Namespace
vuln.Affected = append(vuln.Affected, f)
}
}
var resultVuln []database.NullableVulnerability
for _, vulns := range vulnIDMap {
for _, vuln := range vulns {
resultVuln = append(resultVuln, *vuln)
}
}
return resultVuln, nil
}
func (tx *pgSession) FindVulnerabilities(vulnerabilities []database.VulnerabilityID) ([]database.NullableVulnerability, error) {
defer observeQueryTime("findVulnerabilities", "", time.Now())
resultVuln := make([]database.NullableVulnerability, len(vulnerabilities))

@ -0,0 +1,198 @@
package grafeas
import (
"fmt"
"reflect"
"regexp"
"strconv"
"strings"
"github.com/coreos/clair/database"
"github.com/grafeas/client-go/v1alpha1"
log "github.com/sirupsen/logrus"
)
type Config struct {
Enabled bool
Addr string
ProjectId string
}
type Grafeas struct {
Config *Config
}
func NewGrafeas(config *Config) Grafeas {
return Grafeas{config}
}
func (g *Grafeas) Export(datastore database.Datastore) error {
if !g.Config.Enabled {
return nil
}
log.Info("exporting vulnerabilities to Grafeas")
tx, err := datastore.Begin()
if err != nil {
return err
}
defer tx.Rollback()
vulnerabilities, err := tx.ListVulnerabilities()
if err != nil {
return err
}
pID := g.Config.ProjectId
client := v1alpha1.NewGrafeasApiWithBasePath(g.Config.Addr)
for _, vuln := range vulnerabilities {
nID := vuln.Name
score, nistVectors := extractMetadata(vuln.Metadata)
note, _, err := client.GetNote(pID, nID)
createNewNote := false
if err != nil {
n := noteWithoutDetails(pID, nID, vuln.Description, string(vuln.Severity), nistVectors, score)
note = &n
createNewNote = true
}
containsUpdatedDetail := false
for _, affected := range vuln.Affected {
cpeUri := createCpeUri(affected.Namespace.Name)
detail := detail(cpeUri, affected.FeatureName, vuln.Description, string(vuln.Severity), affected.FixedInVersion)
index := findDetail(note.VulnerabilityType.Details, detail)
if index == -1 {
note.VulnerabilityType.Details = append(note.VulnerabilityType.Details, detail)
containsUpdatedDetail = true
} else if !reflect.DeepEqual(note.VulnerabilityType.Details[index], detail) {
note.VulnerabilityType.Details[index] = detail
containsUpdatedDetail = true
}
}
if createNewNote {
_, _, err = client.CreateNote(pID, nID, *note)
} else if containsUpdatedDetail {
_, _, err = client.UpdateNote(pID, nID, *note)
}
if err != nil {
log.Warn("Error creating note %v", err)
}
}
log.Info("export done")
return nil
}
func extractMetadata(metadata map[string]interface{}) (score float32, vectors string) {
if nvd, ok := metadata["NVD"].(map[string]interface{}); ok {
if cvss, ok := nvd["CVSSv2"].(map[string]interface{}); ok {
score = float32(cvss["Score"].(float64))
vectors = cvss["Vectors"].(string)
}
}
return score, vectors
}
// Clair does not report cpe uri:s so we'll have to create one from the namespace.
func createCpeUri(namespaceName string) string {
ss := strings.Split(namespaceName, ":")
if len(ss) != 2 {
return "CPE_UNSPECIFIED"
}
os := ss[0]
ver := ss[1]
switch os {
case "alpine":
return "cpe:/o:alpine:alpine_linux:" + ver
case "debian":
return "cpe:/o:debian:debian_linux:" + ver
case "ubuntu":
return "cpe:/o:canonical:ubuntu_linux:" + ver
case "centos":
return "cpe:/o:centos:centos:" + ver
case "rhel":
return "cpe:/o:redhat:enterprise_linux:" + ver
case "fedora":
return "cpe:/o:fedoraproject:fedora:" + ver
}
return "CPE_UNSPECIFIED"
}
func findDetail(details []v1alpha1.Detail, detail v1alpha1.Detail) int {
for i, d := range details {
if d.CpeUri == detail.CpeUri && d.Package_ == detail.Package_ {
return i
}
}
return -1
}
func fixedLocation(fixedBy, cpeUri, pkg string) v1alpha1.VulnerabilityLocation {
var version v1alpha1.Version
if fixedBy == "" {
version = v1alpha1.Version{
Kind: "MAXIMUM",
}
} else {
// EVR: Epoch:Version.Revision
evrRegexp := regexp.MustCompile(`(?:(\d*):)?([\w~]+[\w.~]*)-(~?\w+[\w.]*)`)
matches := evrRegexp.FindStringSubmatch(fixedBy)
if len(matches) != 0 {
versionEpoch, _ := strconv.ParseInt(matches[1], 10, 32)
versionName := matches[2]
versionRev := matches[3]
version = v1alpha1.Version{
Epoch: int32(versionEpoch),
Name: versionName,
Revision: versionRev,
}
} else {
version = v1alpha1.Version{
Name: fixedBy,
}
}
}
return v1alpha1.VulnerabilityLocation{
CpeUri: cpeUri,
Package_: pkg,
Version: version,
}
}
func detail(cpeUri, packageName, description, severity, fixedBy string) v1alpha1.Detail {
return v1alpha1.Detail{
CpeUri: cpeUri,
Package_: packageName,
Description: description,
MinAffectedVersion: v1alpha1.Version{
Kind: "MINIMUM",
},
SeverityName: severity,
FixedLocation: fixedLocation(fixedBy, cpeUri, packageName),
}
}
func noteWithoutDetails(pID, name, description, severity, nistVectors string, score float32) v1alpha1.Note {
var longDescription string
if nistVectors != "" {
longDescription = fmt.Sprintf("NIST vectors: %v", nistVectors)
}
return v1alpha1.Note{
Name: fmt.Sprintf("projects/%v/notes/%v", pID, name),
ShortDescription: name,
LongDescription: longDescription,
Kind: "PACKAGE_VULNERABILITY",
VulnerabilityType: v1alpha1.VulnerabilityType{
CvssScore: score,
Severity: severity,
Details: []v1alpha1.Detail{},
},
}
}

@ -28,6 +28,7 @@ import (
"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/vulnmdsrc"
"github.com/coreos/clair/ext/vulnsrc"
"github.com/coreos/clair/grafeas"
"github.com/coreos/clair/pkg/stopper"
)
@ -78,7 +79,7 @@ type vulnerabilityChange struct {
// RunUpdater begins a process that updates the vulnerability database at
// regular intervals.
func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper) {
func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper.Stopper, g grafeas.Grafeas) {
defer st.End()
// Do not run the updater if there is no config or if the interval is 0.
@ -129,6 +130,8 @@ func RunUpdater(config *UpdaterConfig, datastore database.Datastore, st *stopper
}
}
g.Export(datastore)
// Unlock the updater.
unlock(datastore, updaterLockName, whoAmI)

Loading…
Cancel
Save