arno/clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge 057f1afae7 into efa0d855f4

Browse Source
This commit is contained in:
mattmoor 2016-06-07 16:58:26 +00:00
commit 9cfd75a3fb
63 changed files with 1137 additions and 696 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.travis.yml
View File

@ -17,6 +17,9 @@ install:
- echo 'nop'
script:
- mkdir -p $HOME/gopath/src/github.com/coreos/clair/
- rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/coreos/clair/
- cd $HOME/gopath/src/github.com/coreos/clair/
- go test -v $(go list ./... | grep -v /vendor/)
services:

36
Dockerfile.test Normal file
View File

@ -0,0 +1,36 @@
# Copyright 2015 clair authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM golang:1.6
MAINTAINER Quentin Machu <quentin.machu@coreos.com>
RUN apt-get update && \
apt-get install -y bzr rpm xz-utils && \
apt-get autoremove -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # 18MAR2016
VOLUME /config
EXPOSE 6060 6061
ADD . /go/src/github.com/coreos/clair/
WORKDIR /go/src/github.com/coreos/clair/
RUN go install -v github.com/coreos/clair/cmd/clair
RUN go test $(go list ./... | grep -v /vendor/)
ENTRYPOINT ["clair"]

16
api/context/context.go
View File

@ -24,7 +24,12 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/layers"
"github.com/coreos/clair/services/locks"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/services/notifications"
"github.com/coreos/clair/services/vulnerabilities"
"github.com/coreos/clair/utils"
)
@ -59,6 +64,11 @@ func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle {
}
type RouteContext struct {
Store database.Datastore
Config *config.APIConfig
LockService locks.Service
KeyValueStore keyvalue.Service
VulnerabilityStore vulnerabilities.Service
LayerService layers.Service
NamespaceStore namespaces.Service
NotificationState notifications.Service
Config *config.APIConfig
}

18
api/router.go
View File

@ -66,9 +66,21 @@ func getHealth(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
header := w.Header()
header.Set("Server", "clair")
status := http.StatusInternalServerError
if ctx.Store.Ping() {
status = http.StatusOK
status := http.StatusOK
if !ctx.LockService.Ping() {
status = http.StatusInternalServerError
}
if !ctx.KeyValueStore.Ping() {
status = http.StatusInternalServerError
}
if !ctx.VulnerabilityStore.Ping() {
status = http.StatusInternalServerError
}
if !ctx.NotificationState.Ping() {
status = http.StatusInternalServerError
}
if !ctx.LayerService.Ping() {
status = http.StatusInternalServerError
}
w.WriteHeader(status)

36
api/v1/models.go
View File

@ -21,7 +21,7 @@ import (
"fmt"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/coreos/pkg/capnslog"
"github.com/fernet/fernet-go"
@ -44,7 +44,7 @@ type Layer struct {
Features []Feature `json:"Features,omitempty"`
}
func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer {
func LayerFromDatabaseModel(dbLayer services.Layer, withFeatures, withVulnerabilities bool) Layer {
layer := Layer{
Name: dbLayer.Name,
IndexedByVersion: dbLayer.EngineVersion,
@ -104,25 +104,25 @@ type Vulnerability struct {
FixedIn []Feature `json:"FixedIn,omitempty"`
}
func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
func (v Vulnerability) DatabaseModel() (services.Vulnerability, error) {
severity := types.Priority(v.Severity)
if !severity.IsValid() {
return database.Vulnerability{}, errors.New("Invalid severity")
return services.Vulnerability{}, errors.New("Invalid severity")
}
var dbFeatures []database.FeatureVersion
var dbFeatures []services.FeatureVersion
for _, feature := range v.FixedIn {
dbFeature, err := feature.DatabaseModel()
if err != nil {
return database.Vulnerability{}, err
return services.Vulnerability{}, err
}
dbFeatures = append(dbFeatures, dbFeature)
}
return database.Vulnerability{
return services.Vulnerability{
Name: v.Name,
Namespace: database.Namespace{Name: v.NamespaceName},
Namespace: services.Namespace{Name: v.NamespaceName},
Description: v.Description,
Link: v.Link,
Severity: severity,
@ -131,7 +131,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
}, nil
}
func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn bool) Vulnerability {
func VulnerabilityFromDatabaseModel(dbVuln services.Vulnerability, withFixedIn bool) Vulnerability {
vuln := Vulnerability{
Name: dbVuln.Name,
NamespaceName: dbVuln.Namespace.Name,
@ -158,7 +158,7 @@ type Feature struct {
AddedBy string `json:"AddedBy,omitempty"`
}
func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature {
func FeatureFromDatabaseModel(dbFeatureVersion services.FeatureVersion) Feature {
versionStr := dbFeatureVersion.Version.String()
if versionStr == types.MaxVersion.String() {
versionStr = "None"
@ -172,7 +172,7 @@ func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature
}
}
func (f Feature) DatabaseModel() (database.FeatureVersion, error) {
func (f Feature) DatabaseModel() (services.FeatureVersion, error) {
var version types.Version
if f.Version == "None" {
version = types.MaxVersion
@ -180,14 +180,14 @@ func (f Feature) DatabaseModel() (database.FeatureVersion, error) {
var err error
version, err = types.NewVersion(f.Version)
if err != nil {
return database.FeatureVersion{}, err
return services.FeatureVersion{}, err
}
}
return database.FeatureVersion{
Feature: database.Feature{
return services.FeatureVersion{
Feature: services.Feature{
Name: f.Name,
Namespace: database.Namespace{Name: f.NamespaceName},
Namespace: services.Namespace{Name: f.NamespaceName},
},
Version: version,
}, nil
@ -205,7 +205,7 @@ type Notification struct {
New *VulnerabilityWithLayers `json:"New,omitempty"`
}
func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotification, limit int, pageToken string, nextPage database.VulnerabilityNotificationPageNumber, key string) Notification {
func NotificationFromDatabaseModel(dbNotification services.VulnerabilityNotification, limit int, pageToken string, nextPage services.VulnerabilityNotificationPageNumber, key string) Notification {
var oldVuln *VulnerabilityWithLayers
if dbNotification.OldVulnerability != nil {
v := VulnerabilityWithLayersFromDatabaseModel(*dbNotification.OldVulnerability)
@ -219,7 +219,7 @@ func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotifica
}
var nextPageStr string
if nextPage != database.NoVulnerabilityNotificationPage {
if nextPage != services.NoVulnerabilityNotificationPage {
nextPageBytes, _ := tokenMarshal(nextPage, key)
nextPageStr = string(nextPageBytes)
}
@ -255,7 +255,7 @@ type VulnerabilityWithLayers struct {
LayersIntroducingVulnerability []string `json:"LayersIntroducingVulnerability,omitempty"`
}
func VulnerabilityWithLayersFromDatabaseModel(dbVuln database.Vulnerability) VulnerabilityWithLayers {
func VulnerabilityWithLayersFromDatabaseModel(dbVuln services.Vulnerability) VulnerabilityWithLayers {
vuln := VulnerabilityFromDatabaseModel(dbVuln, true)
var layers []string

32
api/v1/routes.go
View File

@ -26,7 +26,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/api/context"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker"
@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
return postLayerRoute, http.StatusBadRequest
}
err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
err = worker.Process(ctx.LayerService, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
if err != nil {
if err == utils.ErrCouldNotExtract ||
err == utils.ErrExtractedFileTooBig ||
@ -142,7 +142,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *
_, withFeatures := r.URL.Query()["features"]
_, withVulnerabilities := r.URL.Query()["vulnerabilities"]
dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
dbLayer, err := ctx.LayerService.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
return getLayerRoute, http.StatusNotFound
@ -158,7 +158,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *
}
func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteLayer(p.ByName("layerName"))
err := ctx.LayerService.DeleteLayer(p.ByName("layerName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
return deleteLayerRoute, http.StatusNotFound
@ -172,7 +172,7 @@ func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ct
}
func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
dbNamespaces, err := ctx.Store.ListNamespaces()
dbNamespaces, err := ctx.NamespaceStore.ListNamespaces()
if err != nil {
writeResponse(w, r, http.StatusInternalServerError, NamespaceEnvelope{Error: &Error{err.Error()}})
return getNamespacesRoute, http.StatusInternalServerError
@ -219,7 +219,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par
return getNotificationRoute, http.StatusBadRequest
}
dbVulns, nextPage, err := ctx.Store.ListVulnerabilities(namespace, limit, page)
dbVulns, nextPage, err := ctx.VulnerabilityStore.ListVulnerabilities(namespace, limit, page)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilityRoute, http.StatusNotFound
@ -267,7 +267,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para
return postVulnerabilityRoute, http.StatusBadRequest
}
err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true)
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
@ -286,7 +286,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para
func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
_, withFixedIn := r.URL.Query()["fixedIn"]
dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return getVulnerabilityRoute, http.StatusNotFound
@ -328,7 +328,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param
vuln.Namespace.Name = p.ByName("namespaceName")
vuln.Name = p.ByName("vulnerabilityName")
err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true)
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
@ -345,7 +345,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param
}
func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
err := ctx.VulnerabilityStore.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
return deleteVulnerabilityRoute, http.StatusNotFound
@ -359,7 +359,7 @@ func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Pa
}
func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
return getFixesRoute, http.StatusNotFound
@ -397,7 +397,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co
return putFixRoute, http.StatusBadRequest
}
err = ctx.Store.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []database.FeatureVersion{dbFix})
err = ctx.VulnerabilityStore.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []services.FeatureVersion{dbFix})
if err != nil {
switch err.(type) {
case *cerrors.ErrBadRequest:
@ -418,7 +418,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co
}
func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName"))
err := ctx.VulnerabilityStore.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
return deleteFixRoute, http.StatusNotFound
@ -446,7 +446,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
}
var pageToken string
page := database.VulnerabilityNotificationFirstPage
page := services.VulnerabilityNotificationFirstPage
pageStrs, pageExists := query["page"]
if pageExists {
err := tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page)
@ -464,7 +464,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
pageToken = string(pageTokenBytes)
}
dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page)
dbNotification, nextPage, err := ctx.NotificationState.GetNotification(p.ByName("notificationName"), limit, page)
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
return deleteNotificationRoute, http.StatusNotFound
@ -480,7 +480,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
}
func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
err := ctx.Store.DeleteNotification(p.ByName("notificationName"))
err := ctx.NotificationState.DeleteNotification(p.ByName("notificationName"))
if err == cerrors.ErrNotFound {
writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
return deleteNotificationRoute, http.StatusNotFound

52
clair.go
View File

@ -26,8 +26,13 @@ import (
"github.com/coreos/clair/api"
"github.com/coreos/clair/api/context"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/notifier"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/layers"
"github.com/coreos/clair/services/locks"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/services/notifications"
"github.com/coreos/clair/services/vulnerabilities"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/utils"
"github.com/coreos/pkg/capnslog"
@ -41,26 +46,57 @@ func Boot(config *config.Config) {
rand.Seed(time.Now().UnixNano())
st := utils.NewStopper()
// Open database
db, err := database.Open(config.Database)
// Open services
ls, err := locks.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer db.Close()
defer ls.Close()
kvs, err := keyvalue.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer kvs.Close()
vuln, err := vulnerabilities.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer vuln.Close()
layers, err := layers.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer layers.Close()
names, err := namespaces.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer names.Close()
ns, err := notifications.Open(config.Database)
if err != nil {
log.Fatal(err)
}
defer ns.Close()
// Start notifier
st.Begin()
go notifier.Run(config.Notifier, db, st)
go notifier.Run(config.Notifier, ls, ns, st)
// Start API
st.Begin()
go api.Run(config.API, &context.RouteContext{db, config.API}, st)
ctx := &context.RouteContext{ls, kvs, vuln, layers, names, ns, config.API}
go api.Run(config.API, ctx, st)
st.Begin()
go api.RunHealth(config.API, &context.RouteContext{db, config.API}, st)
go api.RunHealth(config.API, ctx, st)
// Start updater
st.Begin()
go updater.Run(config.Updater, db, st)
go updater.Run(config.Updater, ls, kvs, vuln, st)
// Wait for interruption and shutdown gracefully.
waitForSignals(syscall.SIGINT, syscall.SIGTERM)

168
database/database.go
View File

@ -14,171 +14,3 @@
// Package database defines the Clair's models and a common interface for database implementations.
package database
import (
"errors"
"fmt"
"time"
"github.com/coreos/clair/config"
)
var (
// ErrBackendException is an error that occurs when the database backend does
// not work properly (ie. unreachable).
ErrBackendException = errors.New("database: an error occured when querying the backend")
// ErrInconsistent is an error that occurs when a database consistency check
// fails (ie. when an entity which is supposed to be unique is detected twice)
ErrInconsistent = errors.New("database: inconsistent database")
)
var drivers = make(map[string]Driver)
// Driver is a function that opens a Datastore specified by its database driver type and specific
// configuration.
type Driver func(config.RegistrableComponentConfig) (Datastore, error)
// Register makes a Constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("database: could not register nil Driver")
}
if _, dup := drivers[name]; dup {
panic("database: could not register duplicate Driver: " + name)
}
drivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (Datastore, error) {
driver, ok := drivers[cfg.Type]
if !ok {
return nil, fmt.Errorf("database: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
}
return driver(cfg)
}
// Datastore is the interface that describes a database backend implementation.
type Datastore interface {
// # Namespace
// ListNamespaces returns the entire list of known Namespaces.
ListNamespaces() ([]Namespace, error)
// # Layer
// InsertLayer stores a Layer in the database.
// A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory.
// If a Parent is specified, it is expected that it has been retrieved using FindLayer.
// If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher
// than the stored one, the stored Layer should be updated.
// The function has to be idempotent, inserting a layer that already exists shouln'd return an
// error.
InsertLayer(Layer) error
// FindLayer retrieves a Layer from the database.
// withFeatures specifies whether the Features field should be filled. When withVulnerabilities is
// true, the Features field should be filled and their AffectedBy fields should contain every
// vulnerabilities that affect them.
FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error)
// DeleteLayer deletes a Layer from the database and every layers that are based on it,
// recursively.
DeleteLayer(name string) error
// # Vulnerability
// ListVulnerabilities returns the list of vulnerabilies of a certain Namespace.
// The Limit and page parameters are used to paginate the return list.
// The first given page should be 0. The function will then return the next available page.
// If there is no more page, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if
// necessary. A vulnerability is uniquely identified by its Namespace and its Name.
// The FixedIn field may only contain a partial list of Features that are affected by the
// Vulnerability, along with the version in which the vulnerability is fixed. It is the
// responsibility of the implementation to update the list properly. A version equals to
// types.MinVersion means that the given Feature is not being affected by the Vulnerability at
// all and thus, should be removed from the list. It is important that Features should be unique
// in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as
// a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and
// Features are Namespaced (i.e. specific to one operating system).
// Each vulnerability insertion or update has to create a Notification that will contain the
// old and the updated Vulnerability, unless createNotification equals to true.
InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list.
FindVulnerability(namespaceName, name string) (Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database.
// It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to
// the specified Vulnerability in the database.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the
// database. It can be used to store the fact that a Vulnerability no longer affects the given
// Feature in any Version.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
// # Notification
// GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a
// Notification that should be handled. The renotify interval defines how much time after being
// marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should
// be returned again by this function. A Notification for which there is a valid Lock with the
// same Name should not be returned.
GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error)
// GetNotification returns a Notification, including its OldVulnerability and NewVulnerability
// fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with
// every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion).
// The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first
// given page should be VulnerabilityNotificationFirstPage. The function will then return the next
// availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned.
GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
// SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for
// GetAvailableNotification, until the renotify duration is elapsed.
SetNotificationNotified(name string) error
// DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for
// GetAvailableNotification.
DeleteNotification(name string) error
// # Key/Value
// InsertKeyValue stores or updates a simple key/value pair in the database.
InsertKeyValue(key, value string) error
// GetKeyValue retrieves a value from the database from the given key.
// It returns an empty string if there is no such key.
GetKeyValue(key string) (string, error)
// # Lock
// Lock creates or renew a Lock in the database with the given name, owner and duration.
// After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus,
// let other users create a Lock with the same name. However, the owner can renew its Lock by
// setting renew to true. Lock should not block, it should instead returns whether the Lock has
// been successfully acquired/renewed. If it's the case, the expiration time of that Lock is
// returned as well.
Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time)
// Unlock releases an existing Lock.
Unlock(name, owner string)
// FindLock returns the owner of a Lock specified by the name, and its experation time if it
// exists.
FindLock(name string) (string, time.Time, error)
// # Miscellaneous
// Ping returns the health status of the database.
Ping() bool
// Close closes the database and free any allocated resource.
Close()
}

41
database/mock.go
View File

@ -14,23 +14,26 @@
package database
import "time"
import (
"github.com/coreos/clair/services"
"time"
)
// MockDatastore implements Datastore and enables overriding each available method.
// The default behavior of each method is to simply panic.
type MockDatastore struct {
FctListNamespaces func() ([]Namespace, error)
FctInsertLayer func(Layer) error
FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error)
FctListNamespaces func() ([]services.Namespace, error)
FctInsertLayer func(services.Layer) error
FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (services.Layer, error)
FctDeleteLayer func(name string) error
FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error
FctFindVulnerability func(namespaceName, name string) (Vulnerability, error)
FctListVulnerabilities func(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error)
FctInsertVulnerabilities func(vulnerabilities []services.Vulnerability, createNotification bool) error
FctFindVulnerability func(namespaceName, name string) (services.Vulnerability, error)
FctDeleteVulnerability func(namespaceName, name string) error
FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error
FctDeleteVulnerabilityFix func(vulnerabilityNamespace, vulnerabilityName, featureName string) error
FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error)
FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
FctGetAvailableNotification func(renotifyInterval time.Duration) (services.VulnerabilityNotification, error)
FctGetNotification func(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error)
FctSetNotificationNotified func(name string) error
FctDeleteNotification func(name string) error
FctInsertKeyValue func(key, value string) error
@ -42,21 +45,21 @@ type MockDatastore struct {
FctClose func()
}
func (mds *MockDatastore) ListNamespaces() ([]Namespace, error) {
func (mds *MockDatastore) ListNamespaces() ([]services.Namespace, error) {
if mds.FctListNamespaces != nil {
return mds.FctListNamespaces()
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) InsertLayer(layer Layer) error {
func (mds *MockDatastore) InsertLayer(layer services.Layer) error {
if mds.FctInsertLayer != nil {
return mds.FctInsertLayer(layer)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error) {
func (mds *MockDatastore) FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) {
if mds.FctFindLayer != nil {
return mds.FctFindLayer(name, withFeatures, withVulnerabilities)
}
@ -70,21 +73,21 @@ func (mds *MockDatastore) DeleteLayer(name string) error {
panic("required mock function not implemented")
}
func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) {
func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error) {
if mds.FctListVulnerabilities != nil {
return mds.FctListVulnerabilities(namespaceName, limit, page)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error {
func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []services.Vulnerability, createNotification bool) error {
if mds.FctInsertVulnerabilities != nil {
return mds.FctInsertVulnerabilities(vulnerabilities, createNotification)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (Vulnerability, error) {
func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (services.Vulnerability, error) {
if mds.FctFindVulnerability != nil {
return mds.FctFindVulnerability(namespaceName, name)
}
@ -98,7 +101,7 @@ func (mds *MockDatastore) DeleteVulnerability(namespaceName, name string) error
panic("required mock function not implemented")
}
func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error {
func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error {
if mds.FctInsertVulnerabilityFixes != nil {
return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes)
}
@ -112,14 +115,14 @@ func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnera
panic("required mock function not implemented")
}
func (mds *MockDatastore) GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error) {
func (mds *MockDatastore) GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) {
if mds.FctGetAvailableNotification != nil {
return mds.FctGetAvailableNotification(renotifyInterval)
}
panic("required mock function not implemented")
}
func (mds *MockDatastore) GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) {
func (mds *MockDatastore) GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) {
if mds.FctGetNotification != nil {
return mds.FctGetNotification(name, limit, page)
}

20
database/pgsql/complex_test.go
View File

@ -26,7 +26,7 @@ import (
"github.com/pborman/uuid"
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
"github.com/coreos/clair/utils/types"
)
@ -45,8 +45,8 @@ func TestRaceAffects(t *testing.T) {
defer datastore.Close()
// Insert the Feature on which we'll work.
feature := database.Feature{
Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"},
feature := services.Feature{
Namespace: services.Namespace{Name: "TestRaceAffectsFeatureNamespace1"},
Name: "TestRaceAffecturesFeature1",
}
_, err = datastore.insertFeature(feature)
@ -60,11 +60,11 @@ func TestRaceAffects(t *testing.T) {
runtime.GOMAXPROCS(runtime.NumCPU())
// Generate FeatureVersions.
featureVersions := make([]database.FeatureVersion, numFeatureVersions)
featureVersions := make([]services.FeatureVersion, numFeatureVersions)
for i := 0; i < numFeatureVersions; i++ {
version := rand.Intn(numFeatureVersions)
featureVersions[i] = database.FeatureVersion{
featureVersions[i] = services.FeatureVersion{
Feature: feature,
Version: types.NewVersionUnsafe(strconv.Itoa(version)),
}
@ -72,18 +72,18 @@ func TestRaceAffects(t *testing.T) {
// Generate vulnerabilities.
// They are mapped by fixed version, which will make verification really easy afterwards.
vulnerabilities := make(map[int][]database.Vulnerability)
vulnerabilities := make(map[int][]services.Vulnerability)
for i := 0; i < numVulnerabilities; i++ {
version := rand.Intn(numFeatureVersions) + 1
// if _, ok := vulnerabilities[version]; !ok {
// vulnerabilities[version] = make([]database.Vulnerability)
// vulnerabilities[version] = make([]services.Vulnerability)
// }
vulnerability := database.Vulnerability{
vulnerability := services.Vulnerability{
Name: uuid.New(),
Namespace: feature.Namespace,
FixedIn: []database.FeatureVersion{
FixedIn: []services.FeatureVersion{
{
Feature: feature,
Version: types.NewVersionUnsafe(strconv.Itoa(version)),
@ -103,7 +103,7 @@ func TestRaceAffects(t *testing.T) {
defer wg.Done()
for _, vulnerabilitiesM := range vulnerabilities {
for _, vulnerability := range vulnerabilitiesM {
err = datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true)
err = datastore.InsertVulnerabilities([]services.Vulnerability{vulnerability}, true)
assert.Nil(t, err)
}
}

10
database/pgsql/feature.go
View File

@ -18,12 +18,12 @@ import (
"database/sql"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
func (pgSQL *pgSQL) insertFeature(feature services.Feature) (int, error) {
if feature.Name == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature")
}
@ -61,7 +61,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
return id, nil
}
func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) {
func (pgSQL *pgSQL) insertFeatureVersion(featureVersion services.FeatureVersion) (id int, err error) {
if featureVersion.Version.String() == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
}
@ -177,7 +177,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
}
// TODO(Quentin-M): Batch me
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []services.FeatureVersion) ([]int, error) {
IDs := make([]int, 0, len(featureVersions))
for i := 0; i < len(featureVersions); i++ {
@ -197,7 +197,7 @@ type vulnerabilityAffectsFeatureVersion struct {
fixedInVersion types.Version
}
func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error {
func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion services.FeatureVersion) error {
// Select every vulnerability and the fixed version that affect this Feature.
// TODO(Quentin-M): LIMIT
rows, err := tx.Query(searchVulnerabilityFixedInFeature, featureVersion.Feature.ID)

34
database/pgsql/feature_test.go
View File

@ -19,7 +19,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
)
@ -32,20 +32,20 @@ func TestInsertFeature(t *testing.T) {
defer datastore.Close()
// Invalid Feature.
id0, err := datastore.insertFeature(database.Feature{})
id0, err := datastore.insertFeature(services.Feature{})
assert.NotNil(t, err)
assert.Zero(t, id0)
id0, err = datastore.insertFeature(database.Feature{
Namespace: database.Namespace{},
id0, err = datastore.insertFeature(services.Feature{
Namespace: services.Namespace{},
Name: "TestInsertFeature0",
})
assert.NotNil(t, err)
assert.Zero(t, id0)
// Insert Feature and ensure we can find it.
feature := database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"},
feature := services.Feature{
Namespace: services.Namespace{Name: "TestInsertFeatureNamespace1"},
Name: "TestInsertFeature1",
}
id1, err := datastore.insertFeature(feature)
@ -55,28 +55,28 @@ func TestInsertFeature(t *testing.T) {
assert.Equal(t, id1, id2)
// Insert invalid FeatureVersion.
for _, invalidFeatureVersion := range []database.FeatureVersion{
for _, invalidFeatureVersion := range []services.FeatureVersion{
{
Feature: database.Feature{},
Feature: services.Feature{},
Version: types.NewVersionUnsafe("1.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{},
Feature: services.Feature{
Namespace: services.Namespace{},
Name: "TestInsertFeature2",
},
Version: types.NewVersionUnsafe("1.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertFeatureNamespace2"},
Name: "TestInsertFeature2",
},
Version: types.NewVersionUnsafe(""),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertFeatureNamespace2"},
Name: "TestInsertFeature2",
},
Version: types.NewVersionUnsafe("bad version"),
@ -88,9 +88,9 @@ func TestInsertFeature(t *testing.T) {
}
// Insert FeatureVersion and ensure we can find it.
featureVersion := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"},
featureVersion := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertFeatureNamespace1"},
Name: "TestInsertFeature1",
},
Version: types.NewVersionUnsafe("2:3.0-imba"),

46
database/pgsql/layer.go
View File

@ -18,13 +18,13 @@ import (
"database/sql"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/guregu/null/zero"
)
func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) {
subquery := "all"
if withFeatures {
subquery += "/features"
@ -34,7 +34,7 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
defer observeQueryTime("FindLayer", subquery, time.Now())
// Find the layer
var layer database.Layer
var layer services.Layer
var parentID zero.Int
var parentName zero.String
var namespaceID zero.Int
@ -49,14 +49,14 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
}
if !parentID.IsZero() {
layer.Parent = &database.Layer{
Model: database.Model{ID: int(parentID.Int64)},
layer.Parent = &services.Layer{
Model: services.Model{ID: int(parentID.Int64)},
Name: parentName.String,
}
}
if !namespaceID.IsZero() {
layer.Namespace = &database.Namespace{
Model: database.Model{ID: int(namespaceID.Int64)},
layer.Namespace = &services.Namespace{
Model: services.Model{ID: int(namespaceID.Int64)},
Name: namespaceName.String,
}
}
@ -110,9 +110,9 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
return layer, nil
}
// getLayerFeatureVersions returns list of database.FeatureVersion that a database.Layer has.
func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion, error) {
var featureVersions []database.FeatureVersion
// getLayerFeatureVersions returns list of services.FeatureVersion that a services.Layer has.
func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]services.FeatureVersion, error) {
var featureVersions []services.FeatureVersion
// Query.
rows, err := tx.Query(searchLayerFeatureVersion, layerID)
@ -123,9 +123,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion
// Scan query.
var modification string
mapFeatureVersions := make(map[int]database.FeatureVersion)
mapFeatureVersions := make(map[int]services.FeatureVersion)
for rows.Next() {
var featureVersion database.FeatureVersion
var featureVersion services.FeatureVersion
err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID,
&featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID,
@ -143,7 +143,7 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion
delete(mapFeatureVersions, featureVersion.ID)
default:
log.Warningf("unknown Layer_diff_FeatureVersion's modification: %s", modification)
return featureVersions, database.ErrInconsistent
return featureVersions, services.ErrInconsistent
}
}
if err = rows.Err(); err != nil {
@ -158,9 +158,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion
return featureVersions, nil
}
// loadAffectedBy returns the list of database.Vulnerability that affect the given
// loadAffectedBy returns the list of services.Vulnerability that affect the given
// FeatureVersion.
func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error {
func loadAffectedBy(tx *sql.Tx, featureVersions []services.FeatureVersion) error {
if len(featureVersions) == 0 {
return nil
}
@ -178,10 +178,10 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error
}
defer rows.Close()
vulnerabilities := make(map[int][]database.Vulnerability, len(featureVersions))
vulnerabilities := make(map[int][]services.Vulnerability, len(featureVersions))
var featureversionID int
for rows.Next() {
var vulnerability database.Vulnerability
var vulnerability services.Vulnerability
err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy)
@ -209,7 +209,7 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error
// (happens when Feature detectors relies on the detected layer Namespace). However, if the listed
// Feature has the same Name/Version as its parent, InsertLayer considers that the Feature hasn't
// been modified.
func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
func (pgSQL *pgSQL) InsertLayer(layer services.Layer) error {
tf := time.Now()
// Verify parameters
@ -314,10 +314,10 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
return nil
}
func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error {
func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *services.Layer) error {
// add and del are the FeatureVersion diff we should insert.
var add []database.FeatureVersion
var del []database.FeatureVersion
var add []services.FeatureVersion
var del []services.FeatureVersion
if layer.Parent == nil {
// There is no parent, every Features are added.
@ -369,8 +369,8 @@ func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *
return nil
}
func createNV(features []database.FeatureVersion) (map[string]*database.FeatureVersion, []string) {
mapNV := make(map[string]*database.FeatureVersion, 0)
func createNV(features []services.FeatureVersion) (map[string]*services.FeatureVersion, []string) {
mapNV := make(map[string]*services.FeatureVersion, 0)
sliceNV := make([]string, 0, len(features))
for i := 0; i < len(features); i++ {

127
database/pgsql/layer_test.go
View File

@ -20,7 +20,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/layers"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
@ -123,93 +124,93 @@ func TestInsertLayer(t *testing.T) {
testInsertLayerDelete(t, datastore)
}
func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) {
invalidLayers := []database.Layer{
func testInsertLayerInvalid(t *testing.T, ls layers.Service) {
invalidLayers := []services.Layer{
{},
{Name: "layer0", Parent: &database.Layer{}},
{Name: "layer0", Parent: &database.Layer{Name: "UnknownLayer"}},
{Name: "layer0", Parent: &services.Layer{}},
{Name: "layer0", Parent: &services.Layer{Name: "UnknownLayer"}},
}
for _, invalidLayer := range invalidLayers {
err := datastore.InsertLayer(invalidLayer)
err := ls.InsertLayer(invalidLayer)
assert.Error(t, err)
}
}
func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
f1 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
func testInsertLayerTree(t *testing.T, ls layers.Service) {
f1 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"},
Name: "TestInsertLayerFeature1",
},
Version: types.NewVersionUnsafe("1.0"),
}
f2 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
f2 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"},
Name: "TestInsertLayerFeature2",
},
Version: types.NewVersionUnsafe("0.34"),
}
f3 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
f3 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"},
Name: "TestInsertLayerFeature3",
},
Version: types.NewVersionUnsafe("0.56"),
}
f4 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
f4 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature2",
},
Version: types.NewVersionUnsafe("0.34"),
}
f5 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
f5 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature3",
},
Version: types.NewVersionUnsafe("0.56"),
}
f6 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
f6 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature4",
},
Version: types.NewVersionUnsafe("0.666"),
}
layers := []database.Layer{
layers := []services.Layer{
{
Name: "TestInsertLayer1",
},
{
Name: "TestInsertLayer2",
Parent: &database.Layer{Name: "TestInsertLayer1"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"},
Parent: &services.Layer{Name: "TestInsertLayer1"},
Namespace: &services.Namespace{Name: "TestInsertLayerNamespace1"},
},
// This layer changes the namespace and adds Features.
{
Name: "TestInsertLayer3",
Parent: &database.Layer{Name: "TestInsertLayer2"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"},
Features: []database.FeatureVersion{f1, f2, f3},
Parent: &services.Layer{Name: "TestInsertLayer2"},
Namespace: &services.Namespace{Name: "TestInsertLayerNamespace2"},
Features: []services.FeatureVersion{f1, f2, f3},
},
// This layer covers the case where the last layer doesn't provide any new Feature.
{
Name: "TestInsertLayer4a",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Features: []database.FeatureVersion{f1, f2, f3},
Parent: &services.Layer{Name: "TestInsertLayer3"},
Features: []services.FeatureVersion{f1, f2, f3},
},
// This layer covers the case where the last layer provides Features.
// It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their
// Namespaces should then remain unchanged.
{
Name: "TestInsertLayer4b",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"},
Features: []database.FeatureVersion{
Parent: &services.Layer{Name: "TestInsertLayer3"},
Namespace: &services.Namespace{Name: "TestInsertLayerNamespace3"},
Features: []services.FeatureVersion{
// Deletes TestInsertLayerFeature1.
// Keep TestInsertLayerFeature2 (old Namespace should be kept):
f4,
@ -222,7 +223,7 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}
var err error
retrievedLayers := make(map[string]database.Layer)
retrievedLayers := make(map[string]services.Layer)
for _, layer := range layers {
if layer.Parent != nil {
// Retrieve from database its parent and assign.
@ -230,10 +231,10 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
layer.Parent = &parent
}
err = datastore.InsertLayer(layer)
err = ls.InsertLayer(layer)
assert.Nil(t, err)
retrievedLayers[layer.Name], err = datastore.FindLayer(layer.Name, true, false)
retrievedLayers[layer.Name], err = ls.FindLayer(layer.Name, true, false)
assert.Nil(t, err)
}
@ -260,35 +261,35 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}
}
func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
f7 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
func testInsertLayerUpdate(t *testing.T, ls layers.Service) {
f7 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature7",
},
Version: types.NewVersionUnsafe("0.01"),
}
l3, _ := datastore.FindLayer("TestInsertLayer3", true, false)
l3u := database.Layer{
l3, _ := ls.FindLayer("TestInsertLayer3", true, false)
l3u := services.Layer{
Name: l3.Name,
Parent: l3.Parent,
Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"},
Features: []database.FeatureVersion{f7},
Namespace: &services.Namespace{Name: "TestInsertLayerNamespaceUpdated1"},
Features: []services.FeatureVersion{f7},
}
l4u := database.Layer{
l4u := services.Layer{
Name: "TestInsertLayer4",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Features: []database.FeatureVersion{f7},
Parent: &services.Layer{Name: "TestInsertLayer3"},
Features: []services.FeatureVersion{f7},
EngineVersion: 2,
}
// Try to re-insert without increasing the EngineVersion.
err := datastore.InsertLayer(l3u)
err := ls.InsertLayer(l3u)
assert.Nil(t, err)
l3uf, err := datastore.FindLayer(l3u.Name, true, false)
l3uf, err := ls.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name)
assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion)
@ -298,10 +299,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
// Update layer l3.
// Verify that the Namespace, EngineVersion and FeatureVersions got updated.
l3u.EngineVersion = 2
err = datastore.InsertLayer(l3u)
err = ls.InsertLayer(l3u)
assert.Nil(t, err)
l3uf, err = datastore.FindLayer(l3u.Name, true, false)
l3uf, err = ls.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name)
assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion)
@ -314,10 +315,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
// Verify that the Namespace got updated from its new Parent's, and also verify the
// EnginVersion and FeatureVersions.
l4u.Parent = &l3uf
err = datastore.InsertLayer(l4u)
err = ls.InsertLayer(l4u)
assert.Nil(t, err)
l4uf, err := datastore.FindLayer(l3u.Name, true, false)
l4uf, err := ls.FindLayer(l3u.Name, true, false)
if assert.Nil(t, err) {
assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name)
assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion)
@ -327,24 +328,24 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) {
}
}
func testInsertLayerDelete(t *testing.T, datastore database.Datastore) {
err := datastore.DeleteLayer("TestInsertLayerX")
func testInsertLayerDelete(t *testing.T, ls layers.Service) {
err := ls.DeleteLayer("TestInsertLayerX")
assert.Equal(t, cerrors.ErrNotFound, err)
err = datastore.DeleteLayer("TestInsertLayer3")
err = ls.DeleteLayer("TestInsertLayer3")
assert.Nil(t, err)
_, err = datastore.FindLayer("TestInsertLayer3", false, false)
_, err = ls.FindLayer("TestInsertLayer3", false, false)
assert.Equal(t, cerrors.ErrNotFound, err)
_, err = datastore.FindLayer("TestInsertLayer4a", false, false)
_, err = ls.FindLayer("TestInsertLayer4a", false, false)
assert.Equal(t, cerrors.ErrNotFound, err)
_, err = datastore.FindLayer("TestInsertLayer4b", true, false)
_, err = ls.FindLayer("TestInsertLayer4b", true, false)
assert.Equal(t, cerrors.ErrNotFound, err)
}
func cmpFV(a, b database.FeatureVersion) bool {
func cmpFV(a, b services.FeatureVersion) bool {
return a.Feature.Name == b.Feature.Name &&
a.Feature.Namespace.Name == b.Feature.Namespace.Name &&
a.Version.String() == b.Version.String()

8
database/pgsql/namespace.go
View File

@ -17,11 +17,11 @@ package pgsql
import (
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
)
func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
func (pgSQL *pgSQL) insertNamespace(namespace services.Namespace) (int, error) {
if namespace.Name == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace")
}
@ -50,7 +50,7 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) {
return id, nil
}
func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error) {
func (pgSQL *pgSQL) ListNamespaces() (namespaces []services.Namespace, err error) {
rows, err := pgSQL.Query(listNamespace)
if err != nil {
return namespaces, handleError("listNamespace", err)
@ -58,7 +58,7 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error
defer rows.Close()
for rows.Next() {
var namespace database.Namespace
var namespace services.Namespace
err = rows.Scan(&namespace.ID, &namespace.Name)
if err != nil {

8
database/pgsql/namespace_test.go
View File

@ -20,7 +20,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
)
func TestInsertNamespace(t *testing.T) {
@ -32,14 +32,14 @@ func TestInsertNamespace(t *testing.T) {
defer datastore.Close()
// Invalid Namespace.
id0, err := datastore.insertNamespace(database.Namespace{})
id0, err := datastore.insertNamespace(services.Namespace{})
assert.NotNil(t, err)
assert.Zero(t, id0)
// Insert Namespace and ensure we can find it.
id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"})
id1, err := datastore.insertNamespace(services.Namespace{Name: "TestInsertNamespace1"})
assert.Nil(t, err)
id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"})
id2, err := datastore.insertNamespace(services.Namespace{Name: "TestInsertNamespace1"})
assert.Nil(t, err)
assert.Equal(t, id1, id2)
}

16
database/pgsql/notification.go
View File

@ -18,7 +18,7 @@ import (
"database/sql"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/guregu/null/zero"
"github.com/pborman/uuid"
@ -43,7 +43,7 @@ func createNotification(tx *sql.Tx, oldVulnerabilityID, newVulnerabilityID int)
// Get one available notification name (!locked && !deleted && (!notified || notified_but_timed-out)).
// Does not fill new/old vuln.
func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (database.VulnerabilityNotification, error) {
func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) {
defer observeQueryTime("GetAvailableNotification", "all", time.Now())
before := time.Now().Add(-renotifyInterval)
@ -53,7 +53,7 @@ func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (da
return notification, handleError("searchNotificationAvailable", err)
}
func (pgSQL *pgSQL) GetNotification(name string, limit int, page database.VulnerabilityNotificationPageNumber) (database.VulnerabilityNotification, database.VulnerabilityNotificationPageNumber, error) {
func (pgSQL *pgSQL) GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) {
defer observeQueryTime("GetNotification", "all", time.Now())
// Get Notification.
@ -86,8 +86,8 @@ func (pgSQL *pgSQL) GetNotification(name string, limit int, page database.Vulner
return notification, page, nil
}
func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (database.VulnerabilityNotification, error) {
var notification database.VulnerabilityNotification
func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (services.VulnerabilityNotification, error) {
var notification services.VulnerabilityNotification
var created zero.Time
var notified zero.Time
var deleted zero.Time
@ -147,7 +147,7 @@ func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (database.Vuln
// Fills Vulnerability.LayersIntroducingVulnerability.
// limit -1: won't do anything
// limit 0: will just get the startID of the second page
func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *database.Vulnerability, limit, startID int) (int, error) {
func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *services.Vulnerability, limit, startID int) (int, error) {
tf := time.Now()
if vulnerability == nil {
@ -170,9 +170,9 @@ func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *database.Vu
}
defer rows.Close()
var layers []database.Layer
var layers []services.Layer
for rows.Next() {
var layer database.Layer
var layer services.Layer
if err := rows.Scan(&layer.ID, &layer.Name); err != nil {
return -1, handleError("searchNotificationLayerIntroducingVulnerability.Scan()", err)

42
database/pgsql/notification_test.go
View File

@ -20,7 +20,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
@ -38,19 +38,19 @@ func TestNotification(t *testing.T) {
assert.Equal(t, cerrors.ErrNotFound, err)
// Create some data.
f1 := database.Feature{
f1 := services.Feature{
Name: "TestNotificationFeature1",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"},
Namespace: services.Namespace{Name: "TestNotificationNamespace1"},
}
f2 := database.Feature{
f2 := services.Feature{
Name: "TestNotificationFeature2",
Namespace: database.Namespace{Name: "TestNotificationNamespace1"},
Namespace: services.Namespace{Name: "TestNotificationNamespace1"},
}
l1 := database.Layer{
l1 := services.Layer{
Name: "TestNotificationLayer1",
Features: []database.FeatureVersion{
Features: []services.FeatureVersion{
{
Feature: f1,
Version: types.NewVersionUnsafe("0.1"),
@ -58,9 +58,9 @@ func TestNotification(t *testing.T) {
},
}
l2 := database.Layer{
l2 := services.Layer{
Name: "TestNotificationLayer2",
Features: []database.FeatureVersion{
Features: []services.FeatureVersion{
{
Feature: f1,
Version: types.NewVersionUnsafe("0.2"),
@ -68,9 +68,9 @@ func TestNotification(t *testing.T) {
},
}
l3 := database.Layer{
l3 := services.Layer{
Name: "TestNotificationLayer3",
Features: []database.FeatureVersion{
Features: []services.FeatureVersion{
{
Feature: f1,
Version: types.NewVersionUnsafe("0.3"),
@ -78,9 +78,9 @@ func TestNotification(t *testing.T) {
},
}
l4 := database.Layer{
l4 := services.Layer{
Name: "TestNotificationLayer4",
Features: []database.FeatureVersion{
Features: []services.FeatureVersion{
{
Feature: f2,
Version: types.NewVersionUnsafe("0.1"),
@ -96,13 +96,13 @@ func TestNotification(t *testing.T) {
}
// Insert a new vulnerability that is introduced by three layers.
v1 := database.Vulnerability{
v1 := services.Vulnerability{
Name: "TestNotificationVulnerability1",
Namespace: f1.Namespace,
Description: "TestNotificationDescription1",
Link: "TestNotificationLink1",
Severity: "Unknown",
FixedIn: []database.FeatureVersion{
FixedIn: []services.FeatureVersion{
{
Feature: f1,
Version: types.NewVersionUnsafe("1.0"),
@ -129,9 +129,9 @@ func TestNotification(t *testing.T) {
}
// Get notification.
filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage)
filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage)
if assert.Nil(t, err) {
assert.NotEqual(t, database.NoVulnerabilityNotificationPage, nextPage)
assert.NotEqual(t, services.NoVulnerabilityNotificationPage, nextPage)
assert.Nil(t, filledNotification.OldVulnerability)
if assert.NotNil(t, filledNotification.NewVulnerability) {
@ -143,7 +143,7 @@ func TestNotification(t *testing.T) {
// Get second page.
filledNotification, nextPage, err = datastore.GetNotification(notification.Name, 2, nextPage)
if assert.Nil(t, err) {
assert.Equal(t, database.NoVulnerabilityNotificationPage, nextPage)
assert.Equal(t, services.NoVulnerabilityNotificationPage, nextPage)
assert.Nil(t, filledNotification.OldVulnerability)
if assert.NotNil(t, filledNotification.NewVulnerability) {
@ -162,7 +162,7 @@ func TestNotification(t *testing.T) {
// Update a vulnerability and ensure that the old/new vulnerabilities are correct.
v1b := v1
v1b.Severity = types.High
v1b.FixedIn = []database.FeatureVersion{
v1b.FixedIn = []services.FeatureVersion{
{
Feature: f1,
Version: types.MinVersion,
@ -179,7 +179,7 @@ func TestNotification(t *testing.T) {
assert.NotEmpty(t, notification.Name)
if assert.Nil(t, err) && assert.NotEmpty(t, notification.Name) {
filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage)
filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage)
if assert.Nil(t, err) {
if assert.NotNil(t, filledNotification.OldVulnerability) {
assert.Equal(t, v1.Name, filledNotification.OldVulnerability.Name)
@ -207,7 +207,7 @@ func TestNotification(t *testing.T) {
assert.NotEmpty(t, notification.Name)
if assert.Nil(t, err) && assert.NotEmpty(t, notification.Name) {
filledNotification, _, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage)
filledNotification, _, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage)
if assert.Nil(t, err) {
assert.Nil(t, filledNotification.NewVulnerability)

36
database/pgsql/pgsql.go
View File

@ -27,13 +27,19 @@ import (
"bitbucket.org/liamstask/goose/lib/goose"
"github.com/coreos/pkg/capnslog"
"github.com/hashicorp/golang-lru"
lru "github.com/hashicorp/golang-lru"
"github.com/lib/pq"
"github.com/prometheus/client_golang/prometheus"
"gopkg.in/yaml.v2"
yaml "gopkg.in/yaml.v2"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/layers"
"github.com/coreos/clair/services/locks"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/services/notifications"
"github.com/coreos/clair/services/vulnerabilities"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
)
@ -74,7 +80,25 @@ func init() {
prometheus.MustRegister(promQueryDurationMilliseconds)
prometheus.MustRegister(promConcurrentLockVAFV)
database.Register("pgsql", openDatabase)
// The types must match exactly because Go doesn't seem to allow covariance.
locks.Register("pgsql", func(cfg config.RegistrableComponentConfig) (locks.Service, error) {
return openDatabase(cfg)
})
keyvalue.Register("pgsql", func(cfg config.RegistrableComponentConfig) (keyvalue.Service, error) {
return openDatabase(cfg)
})
vulnerabilities.Register("pgsql", func(cfg config.RegistrableComponentConfig) (vulnerabilities.Service, error) {
return openDatabase(cfg)
})
layers.Register("pgsql", func(cfg config.RegistrableComponentConfig) (layers.Service, error) {
return openDatabase(cfg)
})
notifications.Register("pgsql", func(cfg config.RegistrableComponentConfig) (notifications.Service, error) {
return openDatabase(cfg)
})
namespaces.Register("pgsql", func(cfg config.RegistrableComponentConfig) (namespaces.Service, error) {
return openDatabase(cfg)
})
}
type Queryer interface {
@ -119,7 +143,7 @@ type Config struct {
// It immediately every necessary migrations. If ManageDatabaseLifecycle is specified,
// the database will be created first. If FixturePath is specified, every SQL queries that are
// present insides will be executed.
func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) (database.Datastore, error) {
func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) (*pgSQL, error) {
var pg pgSQL
var err error
@ -306,7 +330,7 @@ func handleError(desc string, err error) error {
promErrorsTotal.WithLabelValues(desc).Inc()
if _, o := err.(*pq.Error); o || err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") {
return database.ErrBackendException
return services.ErrBackendException
}
return err

7
database/pgsql/pgsql_test.go
View File

@ -26,12 +26,7 @@ import (
)
func openDatabaseForTest(testName string, loadFixture bool) (*pgSQL, error) {
ds, err := openDatabase(generateTestConfig(testName, loadFixture))
if err != nil {
return nil, err
}
datastore := ds.(*pgSQL)
return datastore, nil
return openDatabase(generateTestConfig(testName, loadFixture))
}
func generateTestConfig(testName string, loadFixture bool) config.RegistrableComponentConfig {

72
database/pgsql/vulnerability.go
View File

@ -21,14 +21,14 @@ import (
"reflect"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
"github.com/guregu/null/zero"
)
func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) {
func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]services.Vulnerability, int, error) {
defer observeQueryTime("listVulnerabilities", "all", time.Now())
// Query Namespace.
@ -48,12 +48,12 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
}
defer rows.Close()
var vulns []database.Vulnerability
var vulns []services.Vulnerability
nextID := -1
size := 0
// Scan query.
for rows.Next() {
var vulnerability database.Vulnerability
var vulnerability services.Vulnerability
err := rows.Scan(
&vulnerability.ID,
@ -83,11 +83,11 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID
return vulns, nextID, nil
}
func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) {
func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (services.Vulnerability, error) {
return findVulnerability(pgSQL, namespaceName, name, false)
}
func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) {
func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (services.Vulnerability, error) {
defer observeQueryTime("findVulnerability", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName"
@ -100,7 +100,7 @@ func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bo
return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name))
}
func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) {
func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (services.Vulnerability, error) {
defer observeQueryTime("findVulnerabilityByIDWithDeleted", "all", time.Now())
queryName := "searchVulnerabilityBase+searchVulnerabilityByID"
@ -109,8 +109,8 @@ func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerabi
return scanVulnerability(pgSQL, queryName, pgSQL.QueryRow(query, id))
}
func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.Row) (database.Vulnerability, error) {
var vulnerability database.Vulnerability
func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.Row) (services.Vulnerability, error) {
var vulnerability services.Vulnerability
err := vulnerabilityRow.Scan(
&vulnerability.ID,
@ -156,10 +156,10 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.
if !featureVersionID.IsZero() {
// Note that the ID we fill in featureVersion is actually a Feature ID, and not
// a FeatureVersion ID.
featureVersion := database.FeatureVersion{
Model: database.Model{ID: int(featureVersionID.Int64)},
Feature: database.Feature{
Model: database.Model{ID: int(featureVersionID.Int64)},
featureVersion := services.FeatureVersion{
Model: services.Model{ID: int(featureVersionID.Int64)},
Feature: services.Feature{
Model: services.Model{ID: int(featureVersionID.Int64)},
Namespace: vulnerability.Namespace,
Name: featureVersionFeatureName.String,
},
@ -178,7 +178,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.
// FixedIn.Namespace are not necessary, they are overwritten by the vuln.
// By setting the fixed version to minVersion, we can say that the vuln does'nt affect anymore.
func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerability, generateNotifications bool) error {
func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []services.Vulnerability, generateNotifications bool) error {
for _, vulnerability := range vulnerabilities {
err := pgSQL.insertVulnerability(vulnerability, false, generateNotifications)
if err != nil {
@ -189,7 +189,7 @@ func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerabili
return nil
}
func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, onlyFixedIn, generateNotification bool) error {
func (pgSQL *pgSQL) insertVulnerability(vulnerability services.Vulnerability, onlyFixedIn, generateNotification bool) error {
tf := time.Now()
// Verify parameters
@ -272,7 +272,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
} else {
// The vulnerability is new, we don't want to have any types.MinVersion as they are only used
// for diffing existing vulnerabilities.
var fixedIn []database.FeatureVersion
var fixedIn []services.FeatureVersion
for _, fv := range vulnerability.FixedIn {
if fv.Version != types.MinVersion {
fixedIn = append(fixedIn, fv)
@ -328,19 +328,19 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on
return nil
}
// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that
// castMetadata marshals the given services.MetadataMap and unmarshals it again to make sure that
// everything has the interface{} type.
// It is required when comparing crafted MetadataMap against MetadataMap that we get from the
// database.
func castMetadata(m database.MetadataMap) database.MetadataMap {
c := make(database.MetadataMap)
func castMetadata(m services.MetadataMap) services.MetadataMap {
c := make(services.MetadataMap)
j, _ := json.Marshal(m)
json.Unmarshal(j, &c)
return c
}
// applyFixedInDiff applies a FeatureVersion diff on a FeatureVersion list and returns the result.
func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.FeatureVersion, bool) {
func applyFixedInDiff(currentList, diff []services.FeatureVersion) ([]services.FeatureVersion, bool) {
currentMap, currentNames := createFeatureVersionNameMap(currentList)
diffMap, diffNames := createFeatureVersionNameMap(diff)
@ -375,7 +375,7 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F
}
// Convert currentMap to a slice and return it.
var newList []database.FeatureVersion
var newList []services.FeatureVersion
for _, fv := range currentMap {
newList = append(newList, fv)
}
@ -383,8 +383,8 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F
return newList, different
}
func createFeatureVersionNameMap(features []database.FeatureVersion) (map[string]database.FeatureVersion, []string) {
m := make(map[string]database.FeatureVersion, 0)
func createFeatureVersionNameMap(features []services.FeatureVersion) (map[string]services.FeatureVersion, []string) {
m := make(map[string]services.FeatureVersion, 0)
s := make([]string, 0, len(features))
for i := 0; i < len(features); i++ {
@ -397,16 +397,16 @@ func createFeatureVersionNameMap(features []database.FeatureVersion) (map[string
}
// insertVulnerabilityFixedInFeatureVersions populates Vulnerability_FixedIn_Feature for the given
// vulnerability with the specified database.FeatureVersion list and uses
// vulnerability with the specified services.FeatureVersion list and uses
// linkVulnerabilityToFeatureVersions to propagate the changes on Vulnerability_FixedIn_Feature to
// Vulnerability_Affects_FeatureVersion.
func (pgSQL *pgSQL) insertVulnerabilityFixedInFeatureVersions(tx *sql.Tx, vulnerabilityID int, fixedIn []database.FeatureVersion) error {
func (pgSQL *pgSQL) insertVulnerabilityFixedInFeatureVersions(tx *sql.Tx, vulnerabilityID int, fixedIn []services.FeatureVersion) error {
defer observeQueryTime("insertVulnerabilityFixedInFeatureVersions", "all", time.Now())
// Insert or find the Features.
// TODO(Quentin-M): Batch me.
var err error
var features []*database.Feature
var features []*services.Feature
for i := 0; i < len(fixedIn); i++ {
features = append(features, &fixedIn[i].Feature)
}
@ -464,9 +464,9 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
}
defer rows.Close()
var affecteds []database.FeatureVersion
var affecteds []services.FeatureVersion
for rows.Next() {
var affected database.FeatureVersion
var affected services.FeatureVersion
err := rows.Scan(&affected.ID, &affected.Version)
if err != nil {
@ -497,12 +497,12 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
return nil
}
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error {
func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error {
defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now())
v := database.Vulnerability{
v := services.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Namespace: services.Namespace{
Name: vulnerabilityNamespace,
},
FixedIn: fixes,
@ -514,16 +514,16 @@ func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabili
func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error {
defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now())
v := database.Vulnerability{
v := services.Vulnerability{
Name: vulnerabilityName,
Namespace: database.Namespace{
Namespace: services.Namespace{
Name: vulnerabilityNamespace,
},
FixedIn: []database.FeatureVersion{
FixedIn: []services.FeatureVersion{
{
Feature: database.Feature{
Feature: services.Feature{
Name: featureName,
Namespace: database.Namespace{
Namespace: services.Namespace{
Name: vulnerabilityNamespace,
},
},

82
database/pgsql/vulnerability_test.go
View File

@ -20,7 +20,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
)
@ -38,19 +38,19 @@ func TestFindVulnerability(t *testing.T) {
assert.Equal(t, cerrors.ErrNotFound, err)
// Find a normal vulnerability.
v1 := database.Vulnerability{
v1 := services.Vulnerability{
Name: "CVE-OPENSSL-1-DEB7",
Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0",
Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7",
Severity: types.High,
Namespace: database.Namespace{Name: "debian:7"},
FixedIn: []database.FeatureVersion{
Namespace: services.Namespace{Name: "debian:7"},
FixedIn: []services.FeatureVersion{
{
Feature: database.Feature{Name: "openssl"},
Feature: services.Feature{Name: "openssl"},
Version: types.NewVersionUnsafe("2.0"),
},
{
Feature: database.Feature{Name: "libssl"},
Feature: services.Feature{Name: "libssl"},
Version: types.NewVersionUnsafe("1.9-abc"),
},
},
@ -62,10 +62,10 @@ func TestFindVulnerability(t *testing.T) {
}
// Find a vulnerability that has no link, no severity and no FixedIn.
v2 := database.Vulnerability{
v2 := services.Vulnerability{
Name: "CVE-NOPE",
Description: "A vulnerability affecting nothing",
Namespace: database.Namespace{Name: "debian:7"},
Namespace: services.Namespace{Name: "debian:7"},
Severity: types.Unknown,
}
@ -106,93 +106,93 @@ func TestInsertVulnerability(t *testing.T) {
defer datastore.Close()
// Create some data.
n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace1"}
n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace2"}
n1 := services.Namespace{Name: "TestInsertVulnerabilityNamespace1"}
n2 := services.Namespace{Name: "TestInsertVulnerabilityNamespace2"}
f1 := database.FeatureVersion{
Feature: database.Feature{
f1 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion1",
Namespace: n1,
},
Version: types.NewVersionUnsafe("1.0"),
}
f2 := database.FeatureVersion{
Feature: database.Feature{
f2 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion1",
Namespace: n2,
},
Version: types.NewVersionUnsafe("1.0"),
}
f3 := database.FeatureVersion{
Feature: database.Feature{
f3 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion2",
},
Version: types.MaxVersion,
}
f4 := database.FeatureVersion{
Feature: database.Feature{
f4 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion2",
},
Version: types.NewVersionUnsafe("1.4"),
}
f5 := database.FeatureVersion{
Feature: database.Feature{
f5 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion3",
},
Version: types.NewVersionUnsafe("1.5"),
}
f6 := database.FeatureVersion{
Feature: database.Feature{
f6 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion4",
},
Version: types.NewVersionUnsafe("0.1"),
}
f7 := database.FeatureVersion{
Feature: database.Feature{
f7 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion5",
},
Version: types.MaxVersion,
}
f8 := database.FeatureVersion{
Feature: database.Feature{
f8 := services.FeatureVersion{
Feature: services.Feature{
Name: "TestInsertVulnerabilityFeatureVersion5",
},
Version: types.MinVersion,
}
// Insert invalid vulnerabilities.
for _, vulnerability := range []database.Vulnerability{
for _, vulnerability := range []services.Vulnerability{
{
Name: "",
Namespace: n1,
FixedIn: []database.FeatureVersion{f1},
FixedIn: []services.FeatureVersion{f1},
Severity: types.Unknown,
},
{
Name: "TestInsertVulnerability0",
Namespace: database.Namespace{},
FixedIn: []database.FeatureVersion{f1},
Namespace: services.Namespace{},
FixedIn: []services.FeatureVersion{f1},
Severity: types.Unknown,
},
{
Name: "TestInsertVulnerability0-",
Namespace: database.Namespace{},
FixedIn: []database.FeatureVersion{f1},
Namespace: services.Namespace{},
FixedIn: []services.FeatureVersion{f1},
},
{
Name: "TestInsertVulnerability0",
Namespace: n1,
FixedIn: []database.FeatureVersion{f1},
FixedIn: []services.FeatureVersion{f1},
Severity: types.Priority(""),
},
{
Name: "TestInsertVulnerability0",
Namespace: n1,
FixedIn: []database.FeatureVersion{f2},
FixedIn: []services.FeatureVersion{f2},
Severity: types.Unknown,
},
} {
err := datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true)
err := datastore.InsertVulnerabilities([]services.Vulnerability{vulnerability}, true)
assert.Error(t, err)
}
@ -205,16 +205,16 @@ func TestInsertVulnerability(t *testing.T) {
Test: "TestInsertVulnerabilityMetadataValue1",
}
v1 := database.Vulnerability{
v1 := services.Vulnerability{
Name: "TestInsertVulnerability1",
Namespace: n1,
FixedIn: []database.FeatureVersion{f1, f3, f6, f7},
FixedIn: []services.FeatureVersion{f1, f3, f6, f7},
Severity: types.Low,
Description: "TestInsertVulnerabilityDescription1",
Link: "TestInsertVulnerabilityLink1",
Metadata: v1meta,
}
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
err = datastore.InsertVulnerabilities([]services.Vulnerability{v1}, true)
if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name)
if assert.Nil(t, err) {
@ -228,9 +228,9 @@ func TestInsertVulnerability(t *testing.T) {
v1.Severity = types.High
// Update f3 in f4, add fixed in f5, add fixed in f6 which already exists, removes fixed in f7 by
// adding f8 which is f7 but with MinVersion.
v1.FixedIn = []database.FeatureVersion{f4, f5, f6, f8}
v1.FixedIn = []services.FeatureVersion{f4, f5, f6, f8}
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true)
err = datastore.InsertVulnerabilities([]services.Vulnerability{v1}, true)
if assert.Nil(t, err) {
v1f, err := datastore.FindVulnerability(n1.Name, v1.Name)
if assert.Nil(t, err) {
@ -250,7 +250,7 @@ func TestInsertVulnerability(t *testing.T) {
}
}
func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
func equalsVuln(t *testing.T, expected, actual *services.Vulnerability) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Namespace.Name, actual.Namespace.Name)
assert.Equal(t, expected.Description, actual.Description)

24
notifier/notifier.go
View File

@ -25,7 +25,9 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/locks"
"github.com/coreos/clair/services/notifications"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
)
@ -59,7 +61,7 @@ type Notifier interface {
// It returns whether the notifier is enabled or not.
Configure(*config.NotifierConfig) (bool, error)
// Send informs the existence of the specified notification.
Send(notification database.VulnerabilityNotification) error
Send(notification services.VulnerabilityNotification) error
}
func init() {
@ -87,7 +89,7 @@ func RegisterNotifier(name string, n Notifier) {
}
// Run starts the Notifier service.
func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) {
func Run(config *config.NotifierConfig, ls locks.Service, ns notifications.Service, stopper *utils.Stopper) {
defer stopper.End()
// Configure registered notifiers.
@ -113,7 +115,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u
for running := true; running; {
// Find task.
notification := findTask(datastore, config.RenotifyInterval, whoAmI, stopper)
notification := findTask(ls, ns, config.RenotifyInterval, whoAmI, stopper)
if notification == nil {
// Interrupted while finding a task, Clair is stopping.
break
@ -125,12 +127,12 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u
success, interrupted := handleTask(*notification, stopper, config.Attempts)
if success {
utils.PrometheusObserveTimeMilliseconds(promNotifierLatencyMilliseconds, notification.Created)
datastore.SetNotificationNotified(notification.Name)
ns.SetNotificationNotified(notification.Name)
}
if interrupted {
running = false
}
datastore.Unlock(notification.Name, whoAmI)
ls.Unlock(notification.Name, whoAmI)
done <- true
}()
@ -141,7 +143,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u
case <-done:
break outer
case <-time.After(refreshLockDuration):
datastore.Lock(notification.Name, whoAmI, lockDuration, true)
ls.Lock(notification.Name, whoAmI, lockDuration, true)
}
}
}
@ -149,10 +151,10 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u
log.Info("notifier service stopped")
}
func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *database.VulnerabilityNotification {
func findTask(ls locks.Service, ns notifications.Service, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *services.VulnerabilityNotification {
for {
// Find a notification to send.
notification, err := datastore.GetAvailableNotification(renotifyInterval)
notification, err := ns.GetAvailableNotification(renotifyInterval)
if err != nil {
// There is no notification or an error occurred.
if err != cerrors.ErrNotFound {
@ -168,14 +170,14 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA
}
// Lock the notification.
if hasLock, _ := datastore.Lock(notification.Name, whoAmI, lockDuration, false); hasLock {
if hasLock, _ := ls.Lock(notification.Name, whoAmI, lockDuration, false); hasLock {
log.Infof("found and locked a notification: %s", notification.Name)
return &notification
}
}
}
func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
func handleTask(notification services.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) {
// Send notification.
for notifierName, notifier := range notifiers {
var attempts int

4
notifier/notifiers/webhook.go
View File

@ -30,8 +30,8 @@ import (
"gopkg.in/yaml.v2"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/notifier"
"github.com/coreos/clair/services"
)
const timeout = 5 * time.Second
@ -114,7 +114,7 @@ type notificationEnvelope struct {
}
}
func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error {
func (h *WebhookNotifier) Send(notification services.VulnerabilityNotification) error {
// Marshal notification.
jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}})
if err != nil {

61
services/keyvalue/keyvalue.go Normal file
View File

@ -0,0 +1,61 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package keyvalue defines an interface for a simple keyvalue store.
package keyvalue
import (
"fmt"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var keyValueDrivers = make(map[string]Driver)
// Register makes a Service constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("keyvalue: could not register nil Driver")
}
if _, dup := keyValueDrivers[name]; dup {
panic("keyvalue: could not register duplicate Driver: " + name)
}
keyValueDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := keyValueDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("keyvalue: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Key/Value
// InsertKeyValue stores or updates a simple key/value pair in the database.
InsertKeyValue(key, value string) error
// GetKeyValue retrieves a value from the database from the given key.
// It returns an empty string if there is no such key.
GetKeyValue(key string) (string, error)
}

74
services/layers/layers.go Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package layers defines an interface for reading and writing layer metadata.
package layers
import (
"fmt"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var layerDrivers = make(map[string]Driver)
// Register makes a Layer constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("layers: could not register nil Driver")
}
if _, dup := layerDrivers[name]; dup {
panic("layers: could not register duplicate Driver: " + name)
}
layerDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := layerDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("layers: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Layer
// InsertLayer stores a Layer in the database.
// A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory.
// If a Parent is specified, it is expected that it has been retrieved using FindLayer.
// If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher
// than the stored one, the stored Layer should be updated.
// The function has to be idempotent, inserting a layer that already exists shouln'd return an
// error.
InsertLayer(services.Layer) error
// FindLayer retrieves a Layer from the database.
// withFeatures specifies whether the Features field should be filled. When withVulnerabilities is
// true, the Features field should be filled and their AffectedBy fields should contain every
// vulnerabilities that affect them.
FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error)
// DeleteLayer deletes a Layer from the database and every layers that are based on it,
// recursively.
DeleteLayer(name string) error
}

69
services/locks/locks.go Normal file
View File

@ -0,0 +1,69 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package locks defines an interface for interacting with named locks.
package locks
import (
"fmt"
"time"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var lockDrivers = make(map[string]Driver)
// Register makes a Service constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("locks: could not register nil Driver")
}
if _, dup := lockDrivers[name]; dup {
panic("locks: could not register duplicate Driver: " + name)
}
lockDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := lockDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("locks: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Lock
// Lock creates or renew a Lock in the database with the given name, owner and duration.
// After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus,
// let other users create a Lock with the same name. However, the owner can renew its Lock by
// setting renew to true. Lock should not block, it should instead returns whether the Lock has
// been successfully acquired/renewed. If it's the case, the expiration time of that Lock is
// returned as well.
Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time)
// Unlock releases an existing Lock.
Unlock(name, owner string)
// FindLock returns the owner of a Lock specified by the name, and its experation time if it
// exists.
FindLock(name string) (string, time.Time, error)
}

4
database/models.go → services/models.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package database
package services
import (
"database/sql/driver"
@ -22,6 +22,8 @@ import (
"github.com/coreos/clair/utils/types"
)
// TODO(mattmoor): Clean up the dependencies of these types so that they can be decomposed into their respective service layers. This will require eliminating some of the overloading via optional fields that is done today (e.g. AddedBy "only makes sense when ..." should probably be a FeatureVersion wrapped in a new type used in the context of an image).
// ID is only meant to be used by database implementations and should never be used for anything else.
type Model struct {
ID int

2
database/namespace_mapping.go → services/namespaces/mapping.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package database
package namespaces
// DebianReleasesMapping translates Debian code names and class names to version numbers
var DebianReleasesMapping = map[string]string{

59
services/namespaces/namespaces.go Normal file
View File

@ -0,0 +1,59 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package namespaces defines an interface for listing the available namespaces, and a few maps between equivalent namespaces.
package namespaces
import (
"fmt"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var namespacesDrivers = make(map[string]Driver)
// Register makes a Service constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("namespaces: could not register nil Driver")
}
if _, dup := namespacesDrivers[name]; dup {
panic("namespaces: could not register duplicate Driver: " + name)
}
namespacesDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := namespacesDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("namespaces: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Namespace
// ListNamespaces returns the entire list of known Namespaces.
ListNamespaces() ([]services.Namespace, error)
}

79
services/notifications/notifications.go Normal file
View File

@ -0,0 +1,79 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package notifications defines an interface for interacting with notification state.
package notifications
import (
"fmt"
"time"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var notificationDrivers = make(map[string]Driver)
// Register makes a Service constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("notifications: could not register nil Driver")
}
if _, dup := notificationDrivers[name]; dup {
panic("notifications: could not register duplicate Driver: " + name)
}
notificationDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := notificationDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("notifications: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Notification
// GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a
// Notification that should be handled. The renotify interval defines how much time after being
// marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should
// be returned again by this function. A Notification for which there is a valid Lock with the
// same Name should not be returned.
GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error)
// GetNotification returns a Notification, including its OldVulnerability and NewVulnerability
// fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with
// every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion).
// The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first
// given page should be VulnerabilityNotificationFirstPage. The function will then return the next
// availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned.
GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error)
// SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for
// GetAvailableNotification, until the renotify duration is elapsed.
SetNotificationNotified(name string) error
// DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for
// GetAvailableNotification.
DeleteNotification(name string) error
}

39
services/services.go Normal file
View File

@ -0,0 +1,39 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package services defines a set of service interfaces needed by Clair internally.
package services
import (
"errors"
)
var (
// ErrBackendException is an error that occurs when the service backend does
// not work properly (ie. unreachable).
ErrBackendException = errors.New("services: an error occured when querying the backend")
// ErrInconsistent is an error that occurs when a service consistency check
// fails (ie. when an entity which is supposed to be unique is detected twice)
ErrInconsistent = errors.New("services: inconsistent state")
)
type Base interface {
// # Miscellaneous
// Ping returns the health status of the service.
Ping() bool
// Close closes the connection to the service and free any allocated resources.
Close()
}

89
services/vulnerabilities/vulnerabilities.go Normal file
View File

@ -0,0 +1,89 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package vulnerabilities defines an interface for listing, reading and writing vulnerability information
package vulnerabilities
import (
"fmt"
"github.com/coreos/clair/config"
"github.com/coreos/clair/services"
)
type Driver func(cfg config.RegistrableComponentConfig) (Service, error)
var vulnzDrivers = make(map[string]Driver)
// Register makes a Vulnerability constructor available by the provided name.
//
// If this function is called twice with the same name or if the Constructor is
// nil, it panics.
func Register(name string, driver Driver) {
if driver == nil {
panic("vulnerabilities: could not register nil Driver")
}
if _, dup := vulnzDrivers[name]; dup {
panic("vulnerabilities: could not register duplicate Driver: " + name)
}
vulnzDrivers[name] = driver
}
// Open opens a Datastore specified by a configuration.
func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) {
driver, ok := vulnzDrivers[cfg.Type]
if !ok {
err = fmt.Errorf("vulnerabilities: unknown Driver %q (forgotten configuration or import?)", cfg.Type)
return
}
return driver(cfg)
}
type Service interface {
services.Base
// # Vulnerability
// ListVulnerabilities returns the list of vulnerabilies of a certain Namespace.
// The Limit and page parameters are used to paginate the return list.
// The first given page should be 0. The function will then return the next available page.
// If there is no more page, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if
// necessary. A vulnerability is uniquely identified by its Namespace and its Name.
// The FixedIn field may only contain a partial list of Features that are affected by the
// Vulnerability, along with the version in which the vulnerability is fixed. It is the
// responsibility of the implementation to update the list properly. A version equals to
// types.MinVersion means that the given Feature is not being affected by the Vulnerability at
// all and thus, should be removed from the list. It is important that Features should be unique
// in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as
// a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and
// Features are Namespaced (i.e. specific to one operating system).
// Each vulnerability insertion or update has to create a Notification that will contain the
// old and the updated Vulnerability, unless createNotification equals to true.
InsertVulnerabilities(vulnerabilities []services.Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list.
FindVulnerability(namespaceName, name string) (services.Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database.
// It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to
// the specified Vulnerability in the database.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the
// database. It can be used to store the fact that a Vulnerability no longer affects the given
// Feature in any Version.
// It has has to create a Notification that will contain the old and the updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
}

9
updater/fetchers.go
View File

@ -14,14 +14,17 @@
package updater
import "github.com/coreos/clair/database"
import (
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
)
var fetchers = make(map[string]Fetcher)
// Fetcher represents anything that can fetch vulnerabilities.
type Fetcher interface {
// FetchUpdate gets vulnerability updates.
FetchUpdate(database.Datastore) (FetcherResponse, error)
FetchUpdate(keyvalue.Service) (FetcherResponse, error)
// Clean deletes any allocated resources.
// It is invoked when Clair stops.
@ -33,7 +36,7 @@ type FetcherResponse struct {
FlagName string
FlagValue string
Notes []string
Vulnerabilities []database.Vulnerability
Vulnerabilities []services.Vulnerability
}
// RegisterFetcher makes a Fetcher available by the provided name.

24
updater/fetchers/debian/debian.go
View File

@ -23,7 +23,9 @@ import (
"net/http"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -60,7 +62,7 @@ func init() {
}
// FetchUpdate fetches vulnerability updates from the Debian Security Tracker.
func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (fetcher *DebianFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) {
log.Info("fetching Debian vulnerabilities")
// Download JSON.
@ -71,7 +73,7 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up
}
// Get the SHA-1 of the latest update's JSON data
latestHash, err := datastore.GetKeyValue(updaterFlag)
latestHash, err := kvstore.GetKeyValue(updaterFlag)
if err != nil {
return resp, err
}
@ -130,15 +132,15 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
return resp, nil
}
func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, unknownReleases map[string]struct{}) {
mvulnerabilities := make(map[string]*database.Vulnerability)
func parseDebianJSON(data *jsonData) (vulnerabilities []services.Vulnerability, unknownReleases map[string]struct{}) {
mvulnerabilities := make(map[string]*services.Vulnerability)
unknownReleases = make(map[string]struct{})
for pkgName, pkgNode := range *data {
for vulnName, vulnNode := range pkgNode {
for releaseName, releaseNode := range vulnNode.Releases {
// Attempt to detect the release number.
if _, isReleaseKnown := database.DebianReleasesMapping[releaseName]; !isReleaseKnown {
if _, isReleaseKnown := namespaces.DebianReleasesMapping[releaseName]; !isReleaseKnown {
unknownReleases[releaseName] = struct{}{}
continue
}
@ -151,7 +153,7 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability,
// Get or create the vulnerability.
vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
if !vulnerabilityAlreadyExists {
vulnerability = &database.Vulnerability{
vulnerability = &services.Vulnerability{
Name: vulnName,
Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""),
Severity: types.Unknown,
@ -188,11 +190,11 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability,
}
// Create and add the feature version.
pkg := database.FeatureVersion{
Feature: database.Feature{
pkg := services.FeatureVersion{
Feature: services.Feature{
Name: pkgName,
Namespace: database.Namespace{
Name: "debian:" + database.DebianReleasesMapping[releaseName],
Namespace: services.Namespace{
Name: "debian:" + namespaces.DebianReleasesMapping[releaseName],
},
},
Version: version,

32
updater/fetchers/debian/debian_test.go
View File

@ -20,7 +20,7 @@ import (
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
)
@ -38,17 +38,17 @@ func TestDebianParser(t *testing.T) {
assert.Equal(t, types.Low, vulnerability.Severity)
assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:8"},
Name: "aptdaemon",
},
Version: types.MaxVersion,
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:unstable"},
Name: "aptdaemon",
},
@ -64,24 +64,24 @@ func TestDebianParser(t *testing.T) {
assert.Equal(t, types.High, vulnerability.Severity)
assert.Equal(t, "But this one is very dangerous.", vulnerability.Description)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:8"},
Name: "aptdaemon",
},
Version: types.NewVersionUnsafe("0.7.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:unstable"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:unstable"},
Name: "aptdaemon",
},
Version: types.NewVersionUnsafe("0.7.0"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:8"},
Name: "asterisk",
},
Version: types.NewVersionUnsafe("0.5.56"),
@ -96,10 +96,10 @@ func TestDebianParser(t *testing.T) {
assert.Equal(t, types.Negligible, vulnerability.Severity)
assert.Equal(t, "Un-affected packages.", vulnerability.Description)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "debian:8"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "debian:8"},
Name: "asterisk",
},
Version: types.MinVersion,

19
updater/fetchers/rhel/rhel.go
View File

@ -23,7 +23,8 @@ import (
"strconv"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -88,11 +89,11 @@ func init() {
}
// FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions.
func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (f *RHELFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) {
log.Info("fetching Red Hat vulnerabilities")
// Get the first RHSA we have to manage.
flagValue, err := datastore.GetKeyValue(updaterFlag)
flagValue, err := kvstore.GetKeyValue(updaterFlag)
if err != nil {
return resp, err
}
@ -153,7 +154,7 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe
return resp, nil
}
func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) {
func parseRHSA(ovalReader io.Reader) (vulnerabilities []services.Vulnerability, err error) {
// Decode the XML.
var ov oval
err = xml.NewDecoder(ovalReader).Decode(&ov)
@ -168,7 +169,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability,
for _, definition := range ov.Definitions {
pkgs := toFeatureVersions(definition.Criteria)
if len(pkgs) > 0 {
vulnerability := database.Vulnerability{
vulnerability := services.Vulnerability{
Name: name(definition),
Link: link(definition),
Severity: priority(definition),
@ -259,15 +260,15 @@ func getPossibilities(node criteria) [][]criterion {
return possibilities
}
func toFeatureVersions(criteria criteria) []database.FeatureVersion {
func toFeatureVersions(criteria criteria) []services.FeatureVersion {
// There are duplicates in Red Hat .xml files.
// This map is for deduplication.
featureVersionParameters := make(map[string]database.FeatureVersion)
featureVersionParameters := make(map[string]services.FeatureVersion)
possibilities := getPossibilities(criteria)
for _, criterions := range possibilities {
var (
featureVersion database.FeatureVersion
featureVersion services.FeatureVersion
osVersion int
err error
)
@ -304,7 +305,7 @@ func toFeatureVersions(criteria criteria) []database.FeatureVersion {
}
// Convert the map to slice.
var featureVersionParametersArray []database.FeatureVersion
var featureVersionParametersArray []services.FeatureVersion
for _, fv := range featureVersionParameters {
featureVersionParametersArray = append(featureVersionParametersArray, fv)
}

26
updater/fetchers/rhel/rhel_test.go
View File

@ -20,7 +20,7 @@ import (
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
)
@ -38,24 +38,24 @@ func TestRHELParser(t *testing.T) {
assert.Equal(t, types.Medium, vulnerabilities[0].Severity)
assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "centos:7"},
Name: "xerces-c",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "centos:7"},
Name: "xerces-c-devel",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "centos:7"},
Name: "xerces-c-doc",
},
Version: types.NewVersionUnsafe("3.1.1-7.el7_1"),
@ -76,17 +76,17 @@ func TestRHELParser(t *testing.T) {
assert.Equal(t, types.Critical, vulnerabilities[0].Severity)
assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:6"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "centos:6"},
Name: "firefox",
},
Version: types.NewVersionUnsafe("38.1.0-1.el6_6"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "centos:7"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "centos:7"},
Name: "firefox",
},
Version: types.NewVersionUnsafe("38.1.0-1.el7_1"),

18
updater/fetchers/ubuntu/ubuntu.go
View File

@ -26,7 +26,9 @@ import (
"strconv"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/updater"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
@ -89,7 +91,7 @@ func init() {
}
// FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker.
func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
func (fetcher *UbuntuFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) {
log.Info("fetching Ubuntu vulnerabilities")
// Check to see if the repository does not already exist.
@ -123,7 +125,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up
}
// Get the latest revision number we successfully applied in the database.
dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater")
dbRevisionNumber, err := kvstore.GetKeyValue("ubuntuUpdater")
if err != nil {
return resp, err
}
@ -281,7 +283,7 @@ func getRevisionNumber(pathToRepo string) (int, error) {
return revno, nil
}
func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) {
func parseUbuntuCVE(fileContent io.Reader) (vulnerability services.Vulnerability, unknownReleases map[string]struct{}, err error) {
unknownReleases = make(map[string]struct{})
readingDescription := false
scanner := bufio.NewScanner(fileContent)
@ -350,7 +352,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored {
continue
}
if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
if _, isReleaseKnown := namespaces.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown {
unknownReleases[md["release"]] = struct{}{}
continue
}
@ -374,9 +376,9 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
}
// Create and add the new package.
featureVersion := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]},
featureVersion := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "ubuntu:" + namespaces.UbuntuReleasesMapping[md["release"]]},
Name: md["package"],
},
Version: version,

16
updater/fetchers/ubuntu/ubuntu_test.go
View File

@ -20,7 +20,7 @@ import (
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
)
@ -42,24 +42,24 @@ func TestUbuntuParser(t *testing.T) {
_, hasUnkownRelease := unknownReleases["unknown"]
assert.True(t, hasUnkownRelease)
expectedFeatureVersions := []database.FeatureVersion{
expectedFeatureVersions := []services.FeatureVersion{
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:14.04"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "ubuntu:14.04"},
Name: "libmspack",
},
Version: types.MaxVersion,
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.04"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "ubuntu:15.04"},
Name: "libmspack",
},
Version: types.NewVersionUnsafe("0.4-3"),
},
{
Feature: database.Feature{
Namespace: database.Namespace{Name: "ubuntu:15.10"},
Feature: services.Feature{
Namespace: services.Namespace{Name: "ubuntu:15.10"},
Name: "libmspack-anotherpkg",
},
Version: types.NewVersionUnsafe("0.1"),

9
updater/metadata_fetchers.go
View File

@ -17,22 +17,23 @@ package updater
import (
"sync"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/vulnerabilities"
)
var metadataFetchers = make(map[string]MetadataFetcher)
type VulnerabilityWithLock struct {
*database.Vulnerability
*services.Vulnerability
Lock sync.Mutex
}
// MetadataFetcher
type MetadataFetcher interface {
// Load runs right before the Updater calls AddMetadata for each vulnerabilities.
Load(database.Datastore) error
Load(vulnerabilities.Service) error
// AddMetadata adds metadata to the given database.Vulnerability.
// AddMetadata adds metadata to the given services.Vulnerability.
// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
AddMetadata(*VulnerabilityWithLock) error

4
updater/metadata_fetchers/nvd/nvd.go
View File

@ -15,7 +15,7 @@ import (
"sync"
"time"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services/vulnerabilities"
"github.com/coreos/clair/updater"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/pkg/capnslog"
@ -53,7 +53,7 @@ func init() {
updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{})
}
func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error {
func (fetcher *NVDMetadataFetcher) Load(vulnstore vulnerabilities.Service) error {
fetcher.lock.Lock()
defer fetcher.lock.Unlock()

55
updater/updater.go
View File

@ -23,7 +23,10 @@ import (
"time"
"github.com/coreos/clair/config"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/keyvalue"
"github.com/coreos/clair/services/locks"
"github.com/coreos/clair/services/vulnerabilities"
"github.com/coreos/clair/utils"
"github.com/coreos/pkg/capnslog"
"github.com/pborman/uuid"
@ -65,7 +68,7 @@ func init() {
}
// Run updates the vulnerability database at regular intervals.
func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) {
func Run(config *config.UpdaterConfig, locksvc locks.Service, kvstore keyvalue.Service, vulnstore vulnerabilities.Service, st *utils.Stopper) {
defer st.End()
// Do not run the updater if there is no config or if the interval is 0.
@ -83,7 +86,7 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
// Determine if this is the first update and define the next update time.
// The next update time is (last update time + interval) or now if this is the first update.
nextUpdate := time.Now().UTC()
lastUpdate, firstUpdate, err := getLastUpdate(datastore)
lastUpdate, firstUpdate, err := getLastUpdate(kvstore)
if err != nil {
log.Errorf("an error occured while getting the last update time")
nextUpdate = nextUpdate.Add(config.Interval)
@ -95,12 +98,12 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
if nextUpdate.Before(time.Now().UTC()) {
// Attempt to get a lock on the the update.
log.Debug("attempting to obtain update lock")
hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false)
hasLock, hasLockUntil := locksvc.Lock(lockName, whoAmI, lockDuration, false)
if hasLock {
// Launch update in a new go routine.
doneC := make(chan bool, 1)
go func() {
Update(datastore, firstUpdate)
Update(kvstore, vulnstore, firstUpdate)
doneC <- true
}()
@ -110,21 +113,21 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
done = true
case <-time.After(refreshLockDuration):
// Refresh the lock until the update is done.
datastore.Lock(lockName, whoAmI, lockDuration, true)
locksvc.Lock(lockName, whoAmI, lockDuration, true)
case <-st.Chan():
stop = true
}
}
// Unlock the update.
datastore.Unlock(lockName, whoAmI)
locksvc.Unlock(lockName, whoAmI)
if stop {
break
}
continue
} else {
lockOwner, lockExpiration, err := datastore.FindLock(lockName)
lockOwner, lockExpiration, err := locksvc.FindLock(lockName)
if err != nil {
log.Debug("update lock is already taken")
nextUpdate = hasLockUntil
@ -159,17 +162,17 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S
// Update fetches all the vulnerabilities from the registered fetchers, upserts
// them into the database and then sends notifications.
func Update(datastore database.Datastore, firstUpdate bool) {
func Update(kvstore keyvalue.Service, vulnstore vulnerabilities.Service, firstUpdate bool) {
defer setUpdaterDuration(time.Now())
log.Info("updating vulnerabilities")
// Fetch updates.
status, vulnerabilities, flags, notes := fetch(datastore)
status, vulnerabilities, flags, notes := fetch(kvstore, vulnstore)
// Insert vulnerabilities.
log.Tracef("inserting %d vulnerabilities for update", len(vulnerabilities))
err := datastore.InsertVulnerabilities(vulnerabilities, !firstUpdate)
err := vulnstore.InsertVulnerabilities(vulnerabilities, !firstUpdate)
if err != nil {
promUpdaterErrorsTotal.Inc()
log.Errorf("an error occured when inserting vulnerabilities for update: %s", err)
@ -179,7 +182,7 @@ func Update(datastore database.Datastore, firstUpdate bool) {
// Update flags.
for flagName, flagValue := range flags {
datastore.InsertKeyValue(flagName, flagValue)
kvstore.InsertKeyValue(flagName, flagValue)
}
// Log notes.
@ -190,7 +193,7 @@ func Update(datastore database.Datastore, firstUpdate bool) {
// Update last successful update if every fetchers worked properly.
if status {
datastore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10))
kvstore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10))
}
log.Info("update finished")
@ -201,8 +204,8 @@ func setUpdaterDuration(start time.Time) {
}
// fetch get data from the registered fetchers, in parallel.
func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[string]string, []string) {
var vulnerabilities []database.Vulnerability
func fetch(kvstore keyvalue.Service, vulnstore vulnerabilities.Service) (bool, []services.Vulnerability, map[string]string, []string) {
var vulnerabilities []services.Vulnerability
var notes []string
status := true
flags := make(map[string]string)
@ -212,7 +215,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
var responseC = make(chan *FetcherResponse, 0)
for n, f := range fetchers {
go func(name string, fetcher Fetcher) {
response, err := fetcher.FetchUpdate(datastore)
response, err := fetcher.FetchUpdate(kvstore)
if err != nil {
promUpdaterErrorsTotal.Inc()
log.Errorf("an error occured when fetching update '%s': %s.", name, err)
@ -238,11 +241,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st
}
close(responseC)
return status, addMetadata(datastore, vulnerabilities), flags, notes
return status, addMetadata(vulnstore, vulnerabilities), flags, notes
}
// Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel.
func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability {
func addMetadata(vulnstore vulnerabilities.Service, vulnerabilities []services.Vulnerability) []services.Vulnerability {
if len(metadataFetchers) == 0 {
return vulnerabilities
}
@ -266,7 +269,7 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner
defer wg.Done()
// Load the metadata fetcher.
if err := metadataFetcher.Load(datastore); err != nil {
if err := metadataFetcher.Load(vulnstore); err != nil {
promUpdaterErrorsTotal.Inc()
log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err)
return
@ -286,8 +289,8 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner
return vulnerabilities
}
func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) {
lastUpdateTSS, err := datastore.GetKeyValue(flagName)
func getLastUpdate(kvstore keyvalue.Service) (time.Time, bool, error) {
lastUpdateTSS, err := kvstore.GetKeyValue(flagName)
if err != nil {
return time.Time{}, false, err
}
@ -311,12 +314,12 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) {
//
// It helps simplifying the fetchers that share the same metadata about a Vulnerability regardless
// of their actual namespace (ie. same vulnerability information for every version of a distro).
func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []database.Vulnerability {
vulnerabilitiesMap := make(map[string]*database.Vulnerability)
func doVulnerabilitiesNamespacing(vulnerabilities []services.Vulnerability) []services.Vulnerability {
vulnerabilitiesMap := make(map[string]*services.Vulnerability)
for _, v := range vulnerabilities {
featureVersions := v.FixedIn
v.FixedIn = []database.FeatureVersion{}
v.FixedIn = []services.FeatureVersion{}
for _, fv := range featureVersions {
index := fv.Feature.Namespace.Name + ":" + v.Name
@ -324,7 +327,7 @@ func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []da
if vulnerability, ok := vulnerabilitiesMap[index]; !ok {
newVulnerability := v
newVulnerability.Namespace.Name = fv.Feature.Namespace.Name
newVulnerability.FixedIn = []database.FeatureVersion{fv}
newVulnerability.FixedIn = []services.FeatureVersion{fv}
vulnerabilitiesMap[index] = &newVulnerability
} else {
@ -334,7 +337,7 @@ func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []da
}
// Convert map into a slice.
var response []database.Vulnerability
var response []services.Vulnerability
for _, vulnerability := range vulnerabilitiesMap {
response = append(response, *vulnerability)
}

26
updater/updater_test.go
View File

@ -4,42 +4,42 @@ import (
"fmt"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/stretchr/testify/assert"
)
func TestDoVulnerabilitiesNamespacing(t *testing.T) {
fv1 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "Namespace1"},
fv1 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "Namespace1"},
Name: "Feature1",
},
Version: types.NewVersionUnsafe("0.1"),
}
fv2 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "Namespace2"},
fv2 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "Namespace2"},
Name: "Feature1",
},
Version: types.NewVersionUnsafe("0.2"),
}
fv3 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "Namespace2"},
fv3 := services.FeatureVersion{
Feature: services.Feature{
Namespace: services.Namespace{Name: "Namespace2"},
Name: "Feature2",
},
Version: types.NewVersionUnsafe("0.3"),
}
vulnerability := database.Vulnerability{
vulnerability := services.Vulnerability{
Name: "DoVulnerabilityNamespacing",
FixedIn: []database.FeatureVersion{fv1, fv2, fv3},
FixedIn: []services.FeatureVersion{fv1, fv2, fv3},
}
vulnerabilities := doVulnerabilitiesNamespacing([]database.Vulnerability{vulnerability})
vulnerabilities := doVulnerabilitiesNamespacing([]services.Vulnerability{vulnerability})
for _, vulnerability := range vulnerabilities {
switch vulnerability.Namespace.Name {
case fv1.Feature.Namespace.Name:

4
utils/http/http.go
View File

@ -20,7 +20,7 @@ import (
"io"
"net/http"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker"
@ -54,7 +54,7 @@ func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) {
switch err {
case cerrors.ErrNotFound:
httpStatus = http.StatusNotFound
case database.ErrBackendException:
case services.ErrBackendException:
httpStatus = http.StatusServiceUnavailable
case worker.ErrParentUnknown, worker.ErrUnsupported, utils.ErrCouldNotExtract, utils.ErrExtractedFileTooBig:
httpStatus = http.StatusBadRequest

12
worker/detectors/feature/dpkg/dpkg.go
View File

@ -19,7 +19,7 @@ import (
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors"
"github.com/coreos/pkg/capnslog"
@ -40,16 +40,16 @@ func init() {
}
// Detect detects packages using var/lib/dpkg/status from the input data
func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]services.FeatureVersion, error) {
f, hasFile := data["var/lib/dpkg/status"]
if !hasFile {
return []database.FeatureVersion{}, nil
return []services.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
packagesMap := make(map[string]services.FeatureVersion)
var pkg database.FeatureVersion
var pkg services.FeatureVersion
var err error
scanner := bufio.NewScanner(strings.NewReader(string(f)))
for scanner.Scan() {
@ -100,7 +100,7 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
packages := make([]services.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}

10
worker/detectors/feature/dpkg/dpkg_test.go
View File

@ -17,7 +17,7 @@ package dpkg
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/feature"
)
@ -25,18 +25,18 @@ import (
var dpkgPackagesTests = []feature.FeatureVersionTest{
// Test an Ubuntu dpkg status file
{
FeatureVersions: []database.FeatureVersion{
FeatureVersions: []services.FeatureVersion{
// Two packages from this source are installed, it should only appear one time
{
Feature: database.Feature{Name: "pam"},
Feature: services.Feature{Name: "pam"},
Version: types.NewVersionUnsafe("1.1.8-3.1ubuntu3"),
},
{
Feature: database.Feature{Name: "makedev"}, // The source name and the package name are equals
Feature: services.Feature{Name: "makedev"}, // The source name and the package name are equals
Version: types.NewVersionUnsafe("2.3.1-93ubuntu1"), // The version comes from the "Version:" line
},
{
Feature: database.Feature{Name: "gcc-5"},
Feature: services.Feature{Name: "gcc-5"},
Version: types.NewVersionUnsafe("5.1.1-12ubuntu1"), // The version comes from the "Source:" line
},
},

20
worker/detectors/feature/rpm/rpm.go
View File

@ -20,7 +20,7 @@ import (
"os"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -39,27 +39,27 @@ func init() {
}
// Detect detects packages using var/lib/rpm/Packages from the input data
func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) {
func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]services.FeatureVersion, error) {
f, hasFile := data["var/lib/rpm/Packages"]
if !hasFile {
return []database.FeatureVersion{}, nil
return []services.FeatureVersion{}, nil
}
// Create a map to store packages and ensure their uniqueness
packagesMap := make(map[string]database.FeatureVersion)
packagesMap := make(map[string]services.FeatureVersion)
// Write the required "Packages" file to disk
tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm")
defer os.RemoveAll(tmpDir)
if err != nil {
log.Errorf("could not create temporary folder for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
return []services.FeatureVersion{}, cerrors.ErrFilesystem
}
err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700)
if err != nil {
log.Errorf("could not create temporary file for RPM detection: %s", err)
return []database.FeatureVersion{}, cerrors.ErrFilesystem
return []services.FeatureVersion{}, cerrors.ErrFilesystem
}
// Query RPM
@ -70,7 +70,7 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
log.Errorf("could not query RPM: %s. output: %s", err, string(out))
// Do not bubble up because we probably won't be able to fix it,
// the database must be corrupted
return []database.FeatureVersion{}, nil
return []services.FeatureVersion{}, nil
}
scanner := bufio.NewScanner(strings.NewReader(string(out)))
@ -95,8 +95,8 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
}
// Add package
pkg := database.FeatureVersion{
Feature: database.Feature{
pkg := services.FeatureVersion{
Feature: services.Feature{
Name: line[0],
},
Version: version,
@ -105,7 +105,7 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.
}
// Convert the map to a slice
packages := make([]database.FeatureVersion, 0, len(packagesMap))
packages := make([]services.FeatureVersion, 0, len(packagesMap))
for _, pkg := range packagesMap {
packages = append(packages, pkg)
}

8
worker/detectors/feature/rpm/rpm_test.go
View File

@ -17,7 +17,7 @@ package rpm
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/utils/types"
"github.com/coreos/clair/worker/detectors/feature"
)
@ -26,15 +26,15 @@ var rpmPackagesTests = []feature.FeatureVersionTest{
// Test a CentOS 7 RPM database
// Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb
{
FeatureVersions: []database.FeatureVersion{
FeatureVersions: []services.FeatureVersion{
// Two packages from this source are installed, it should only appear once
{
Feature: database.Feature{Name: "centos-release"},
Feature: services.Feature{Name: "centos-release"},
Version: types.NewVersionUnsafe("7-1.1503.el7.centos.2.8"),
},
// Two packages from this source are installed, it should only appear once
{
Feature: database.Feature{Name: "filesystem"},
Feature: services.Feature{Name: "filesystem"},
Version: types.NewVersionUnsafe("3.2-18.el7"),
},
},

4
worker/detectors/feature/test.go
View File

@ -20,13 +20,13 @@ import (
"runtime"
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors"
"github.com/stretchr/testify/assert"
)
type FeatureVersionTest struct {
FeatureVersions []database.FeatureVersion
FeatureVersions []services.FeatureVersion
Data map[string][]byte
}

10
worker/detectors/features.go
View File

@ -18,13 +18,13 @@ import (
"fmt"
"sync"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
)
// The FeaturesDetector interface defines a way to detect packages from input data.
type FeaturesDetector interface {
// Detect detects a list of FeatureVersion from the input data.
Detect(map[string][]byte) ([]database.FeatureVersion, error)
Detect(map[string][]byte) ([]services.FeatureVersion, error)
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
@ -54,13 +54,13 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) {
}
// DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector.
func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) {
var packages []database.FeatureVersion
func DetectFeatures(data map[string][]byte) ([]services.FeatureVersion, error) {
var packages []services.FeatureVersion
for _, detector := range featuresDetectors {
pkgs, err := detector.Detect(data)
if err != nil {
return []database.FeatureVersion{}, err
return []services.FeatureVersion{}, err
}
packages = append(packages, pkgs...)
}

6
worker/detectors/namespace.go
View File

@ -20,14 +20,14 @@ import (
"fmt"
"sync"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
)
// The NamespaceDetector interface defines a way to detect a Namespace from input data.
// A namespace is usually made of an Operating System name and its version.
type NamespaceDetector interface {
// Detect detects a Namespace and its version from input data.
Detect(map[string][]byte) *database.Namespace
Detect(map[string][]byte) *services.Namespace
// GetRequiredFiles returns the list of files required for Detect, without
// leading /.
GetRequiredFiles() []string
@ -61,7 +61,7 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) {
}
// DetectNamespace finds the OS of the layer by using every registered NamespaceDetector.
func DetectNamespace(data map[string][]byte) *database.Namespace {
func DetectNamespace(data map[string][]byte) *services.Namespace {
for _, detector := range namespaceDetectors {
if namespace := detector.Detect(data); namespace != nil {
return namespace

11
worker/detectors/namespace/aptsources/aptsources.go
View File

@ -18,7 +18,8 @@ import (
"bufio"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/namespaces"
"github.com/coreos/clair/worker/detectors"
)
@ -33,7 +34,7 @@ func init() {
detectors.RegisterNamespaceDetector("apt-sources", &AptSourcesNamespaceDetector{})
}
func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *services.Namespace {
f, hasFile := data["etc/apt/sources.list"]
if !hasFile {
return nil
@ -61,12 +62,12 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat
}
var found bool
version, found = database.DebianReleasesMapping[line[2]]
version, found = namespaces.DebianReleasesMapping[line[2]]
if found {
OS = "debian"
break
}
version, found = database.UbuntuReleasesMapping[line[2]]
version, found = namespaces.UbuntuReleasesMapping[line[2]]
if found {
OS = "ubuntu"
break
@ -75,7 +76,7 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &services.Namespace{Name: OS + ":" + version}
}
return nil
}

4
worker/detectors/namespace/aptsources/aptsources_test.go
View File

@ -17,13 +17,13 @@ package aptsources
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors/namespace"
)
var aptSourcesOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "debian:unstable"},
ExpectedNamespace: services.Namespace{Name: "debian:unstable"},
Data: map[string][]byte{
"etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux stretch/sid"

6
worker/detectors/namespace/lsbrelease/lsbrelease.go
View File

@ -19,7 +19,7 @@ import (
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors"
)
@ -38,7 +38,7 @@ func init() {
detectors.RegisterNamespaceDetector("lsb-release", &LsbReleaseNamespaceDetector{})
}
func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace {
f, hasFile := data["etc/lsb-release"]
if !hasFile {
return nil
@ -70,7 +70,7 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &services.Namespace{Name: OS + ":" + version}
}
return nil
}

6
worker/detectors/namespace/lsbrelease/lsbrelease_test.go
View File

@ -17,13 +17,13 @@ package lsbrelease
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors/namespace"
)
var lsbReleaseOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"},
ExpectedNamespace: services.Namespace{Name: "ubuntu:12.04"},
Data: map[string][]byte{
"etc/lsb-release": []byte(
`DISTRIB_ID=Ubuntu
@ -33,7 +33,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`),
},
},
{ // We don't care about the minor version of Debian
ExpectedNamespace: database.Namespace{Name: "debian:7"},
ExpectedNamespace: services.Namespace{Name: "debian:7"},
Data: map[string][]byte{
"etc/lsb-release": []byte(
`DISTRIB_ID=Debian

6
worker/detectors/namespace/osrelease/osrelease.go
View File

@ -19,7 +19,7 @@ import (
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors"
)
@ -39,7 +39,7 @@ func init() {
// Detect tries to detect OS/Version using "/etc/os-release" and "/usr/lib/os-release"
// Typically for Debian / Ubuntu
// /etc/debian_version can't be used, it does not make any difference between testing and unstable, it returns stretch/sid
func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace {
var OS, version string
for _, filePath := range detector.GetRequiredFiles() {
@ -65,7 +65,7 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data
}
if OS != "" && version != "" {
return &database.Namespace{Name: OS + ":" + version}
return &services.Namespace{Name: OS + ":" + version}
}
return nil
}

8
worker/detectors/namespace/osrelease/osrelease_test.go
View File

@ -17,13 +17,13 @@ package osrelease
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors/namespace"
)
var osReleaseOSTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "debian:8"},
ExpectedNamespace: services.Namespace{Name: "debian:8"},
Data: map[string][]byte{
"etc/os-release": []byte(
`PRETTY_NAME="Debian GNU/Linux 8 (jessie)"
@ -37,7 +37,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`),
},
},
{
ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"},
ExpectedNamespace: services.Namespace{Name: "ubuntu:15.10"},
Data: map[string][]byte{
"etc/os-release": []byte(
`NAME="Ubuntu"
@ -52,7 +52,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`),
},
},
{ // Doesn't have quotes around VERSION_ID
ExpectedNamespace: database.Namespace{Name: "fedora:20"},
ExpectedNamespace: services.Namespace{Name: "fedora:20"},
Data: map[string][]byte{
"etc/os-release": []byte(
`NAME=Fedora

6
worker/detectors/namespace/redhatrelease/redhatrelease.go
View File

@ -18,7 +18,7 @@ import (
"regexp"
"strings"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors"
)
@ -37,7 +37,7 @@ func init() {
detectors.RegisterNamespaceDetector("redhat-release", &RedhatReleaseNamespaceDetector{})
}
func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace {
func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace {
for _, filePath := range detector.GetRequiredFiles() {
f, hasFile := data[filePath]
if !hasFile {
@ -46,7 +46,7 @@ func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *
r := redhatReleaseRegexp.FindStringSubmatch(string(f))
if len(r) == 4 {
return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]}
return &services.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]}
}
}

6
worker/detectors/namespace/redhatrelease/redhatrelease_test.go
View File

@ -17,19 +17,19 @@ package redhatrelease
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors/namespace"
)
var redhatReleaseTests = []namespace.NamespaceTest{
{
ExpectedNamespace: database.Namespace{Name: "centos:6"},
ExpectedNamespace: services.Namespace{Name: "centos:6"},
Data: map[string][]byte{
"etc/centos-release": []byte(`CentOS release 6.6 (Final)`),
},
},
{
ExpectedNamespace: database.Namespace{Name: "centos:7"},
ExpectedNamespace: services.Namespace{Name: "centos:7"},
Data: map[string][]byte{
"etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`),
},

4
worker/detectors/namespace/test.go
View File

@ -17,14 +17,14 @@ package namespace
import (
"testing"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/worker/detectors"
"github.com/stretchr/testify/assert"
)
type NamespaceTest struct {
Data map[string][]byte
ExpectedNamespace database.Namespace
ExpectedNamespace services.Namespace
}
func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) {

21
worker/worker.go
View File

@ -19,7 +19,8 @@ package worker
import (
"github.com/coreos/pkg/capnslog"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
"github.com/coreos/clair/services/layers"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/worker/detectors"
@ -50,7 +51,7 @@ var (
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error {
func Process(ls layers.Service, imageFormat, name, parentName, path string, headers map[string]string) error {
// Verify parameters.
if name == "" {
return cerrors.NewBadRequestError("could not process a layer which does not have a name")
@ -68,19 +69,19 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
name, utils.CleanURL(path), Version, parentName, imageFormat)
// Check to see if the layer is already in the database.
layer, err := datastore.FindLayer(name, false, false)
layer, err := ls.FindLayer(name, false, false)
if err != nil && err != cerrors.ErrNotFound {
return err
}
if err == cerrors.ErrNotFound {
// New layer case.
layer = database.Layer{Name: name, EngineVersion: Version}
layer = services.Layer{Name: name, EngineVersion: Version}
// Retrieve the parent if it has one.
// We need to get it with its Features in order to diff them.
if parentName != "" {
parent, err := datastore.FindLayer(parentName, true, false)
parent, err := ls.FindLayer(parentName, true, false)
if err != nil && err != cerrors.ErrNotFound {
return err
}
@ -109,11 +110,11 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
return err
}
return datastore.InsertLayer(layer)
return ls.InsertLayer(layer)
}
// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) {
func detectContent(imageFormat, name, path string, headers map[string]string, parent *services.Layer) (namespace *services.Namespace, featureVersions []services.FeatureVersion, err error) {
data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize)
if err != nil {
log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
@ -135,7 +136,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
return
}
func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) {
func detectNamespace(name string, data map[string][]byte, parent *services.Layer) (namespace *services.Namespace) {
// Use registered detectors to get the Namespace.
namespace = detectors.DetectNamespace(data)
if namespace != nil {
@ -155,7 +156,7 @@ func detectNamespace(name string, data map[string][]byte, parent *database.Layer
return
}
func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) {
func detectFeatureVersions(name string, data map[string][]byte, namespace *services.Namespace, parent *services.Layer) (features []services.FeatureVersion, err error) {
// TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that
// some detectors would need it in order to produce the entire feature list (if they can only
// detect a diff). Also, we should probably pass the detected namespace so detectors could
@ -175,7 +176,7 @@ func detectFeatureVersions(name string, data map[string][]byte, namespace *datab
}
// Build a map of the namespaces for each FeatureVersion in our parent layer.
parentFeatureNamespaces := make(map[string]database.Namespace)
parentFeatureNamespaces := make(map[string]services.Namespace)
if parent != nil {
for _, parentFeature := range parent.Features {
parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace

29
worker/worker_test.go
View File

@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/coreos/clair/database"
"github.com/coreos/clair/services"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
@ -34,12 +35,12 @@ import (
type mockDatastore struct {
database.MockDatastore
layers map[string]database.Layer
layers map[string]services.Layer
}
func newMockDatastore() *mockDatastore {
return &mockDatastore{
layers: make(map[string]database.Layer),
layers: make(map[string]services.Layer),
}
}
@ -49,27 +50,27 @@ func TestProcessWithDistUpgrade(t *testing.T) {
// Create a mock datastore.
datastore := newMockDatastore()
datastore.FctInsertLayer = func(layer database.Layer) error {
datastore.FctInsertLayer = func(layer services.Layer) error {
datastore.layers[layer.Name] = layer
return nil
}
datastore.FctFindLayer = func(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
datastore.FctFindLayer = func(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) {
if layer, exists := datastore.layers[name]; exists {
return layer, nil
}
return database.Layer{}, cerrors.ErrNotFound
return services.Layer{}, cerrors.ErrNotFound
}
// Create the list of FeatureVersions that should not been upgraded from one layer to another.
nonUpgradedFeatureVersions := []database.FeatureVersion{
{Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")},
{Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")},
{Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")},
{Feature: database.Feature{Name: "mawk"}, Version: types.NewVersionUnsafe("1.3.3-17")},
{Feature: database.Feature{Name: "insserv"}, Version: types.NewVersionUnsafe("1.14.0-5")},
{Feature: database.Feature{Name: "db"}, Version: types.NewVersionUnsafe("5.1.29-5")},
{Feature: database.Feature{Name: "ustr"}, Version: types.NewVersionUnsafe("1.0.4-3")},
{Feature: database.Feature{Name: "xz-utils"}, Version: types.NewVersionUnsafe("5.1.1alpha+20120614-2")},
nonUpgradedFeatureVersions := []services.FeatureVersion{
{Feature: services.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")},
{Feature: services.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")},
{Feature: services.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")},
{Feature: services.Feature{Name: "mawk"}, Version: types.NewVersionUnsafe("1.3.3-17")},
{Feature: services.Feature{Name: "insserv"}, Version: types.NewVersionUnsafe("1.14.0-5")},
{Feature: services.Feature{Name: "db"}, Version: types.NewVersionUnsafe("5.1.29-5")},
{Feature: services.Feature{Name: "ustr"}, Version: types.NewVersionUnsafe("1.0.4-3")},
{Feature: services.Feature{Name: "xz-utils"}, Version: types.NewVersionUnsafe("5.1.1alpha+20120614-2")},
}
// Process test layers.