diff --git a/.travis.yml b/.travis.yml index 8bac1448..4cd948ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ install: - echo 'nop' script: + - mkdir -p $HOME/gopath/src/github.com/coreos/clair/ + - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/coreos/clair/ + - cd $HOME/gopath/src/github.com/coreos/clair/ - go test -v $(go list ./... | grep -v /vendor/) services: diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..7274d6ac --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,36 @@ +# Copyright 2015 clair authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.6 + +MAINTAINER Quentin Machu + +RUN apt-get update && \ + apt-get install -y bzr rpm xz-utils && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # 18MAR2016 + +VOLUME /config + +EXPOSE 6060 6061 + +ADD . /go/src/github.com/coreos/clair/ +WORKDIR /go/src/github.com/coreos/clair/ + +RUN go install -v github.com/coreos/clair/cmd/clair + +RUN go test $(go list ./... | grep -v /vendor/) + +ENTRYPOINT ["clair"] diff --git a/api/context/context.go b/api/context/context.go index 77fa388b..41c1c841 100644 --- a/api/context/context.go +++ b/api/context/context.go @@ -24,7 +24,12 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/layers" + "github.com/coreos/clair/services/locks" + "github.com/coreos/clair/services/namespaces" + "github.com/coreos/clair/services/notifications" + "github.com/coreos/clair/services/vulnerabilities" "github.com/coreos/clair/utils" ) @@ -59,6 +64,11 @@ func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle { } type RouteContext struct { - Store database.Datastore - Config *config.APIConfig + LockService locks.Service + KeyValueStore keyvalue.Service + VulnerabilityStore vulnerabilities.Service + LayerService layers.Service + NamespaceStore namespaces.Service + NotificationState notifications.Service + Config *config.APIConfig } diff --git a/api/router.go b/api/router.go index 808ca5f7..76faa08f 100644 --- a/api/router.go +++ b/api/router.go @@ -66,9 +66,21 @@ func getHealth(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx header := w.Header() header.Set("Server", "clair") - status := http.StatusInternalServerError - if ctx.Store.Ping() { - status = http.StatusOK + status := http.StatusOK + if !ctx.LockService.Ping() { + status = http.StatusInternalServerError + } + if !ctx.KeyValueStore.Ping() { + status = http.StatusInternalServerError + } + if !ctx.VulnerabilityStore.Ping() { + status = http.StatusInternalServerError + } + if !ctx.NotificationState.Ping() { + status = http.StatusInternalServerError + } + if !ctx.LayerService.Ping() { + status = http.StatusInternalServerError } w.WriteHeader(status) diff --git a/api/v1/models.go b/api/v1/models.go index 0d6a6383..f3529634 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/coreos/pkg/capnslog" "github.com/fernet/fernet-go" @@ -44,7 +44,7 @@ type Layer struct { Features []Feature `json:"Features,omitempty"` } -func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer { +func LayerFromDatabaseModel(dbLayer services.Layer, withFeatures, withVulnerabilities bool) Layer { layer := Layer{ Name: dbLayer.Name, IndexedByVersion: dbLayer.EngineVersion, @@ -104,25 +104,25 @@ type Vulnerability struct { FixedIn []Feature `json:"FixedIn,omitempty"` } -func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { +func (v Vulnerability) DatabaseModel() (services.Vulnerability, error) { severity := types.Priority(v.Severity) if !severity.IsValid() { - return database.Vulnerability{}, errors.New("Invalid severity") + return services.Vulnerability{}, errors.New("Invalid severity") } - var dbFeatures []database.FeatureVersion + var dbFeatures []services.FeatureVersion for _, feature := range v.FixedIn { dbFeature, err := feature.DatabaseModel() if err != nil { - return database.Vulnerability{}, err + return services.Vulnerability{}, err } dbFeatures = append(dbFeatures, dbFeature) } - return database.Vulnerability{ + return services.Vulnerability{ Name: v.Name, - Namespace: database.Namespace{Name: v.NamespaceName}, + Namespace: services.Namespace{Name: v.NamespaceName}, Description: v.Description, Link: v.Link, Severity: severity, @@ -131,7 +131,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { }, nil } -func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn bool) Vulnerability { +func VulnerabilityFromDatabaseModel(dbVuln services.Vulnerability, withFixedIn bool) Vulnerability { vuln := Vulnerability{ Name: dbVuln.Name, NamespaceName: dbVuln.Namespace.Name, @@ -158,7 +158,7 @@ type Feature struct { AddedBy string `json:"AddedBy,omitempty"` } -func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature { +func FeatureFromDatabaseModel(dbFeatureVersion services.FeatureVersion) Feature { versionStr := dbFeatureVersion.Version.String() if versionStr == types.MaxVersion.String() { versionStr = "None" @@ -172,7 +172,7 @@ func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature } } -func (f Feature) DatabaseModel() (database.FeatureVersion, error) { +func (f Feature) DatabaseModel() (services.FeatureVersion, error) { var version types.Version if f.Version == "None" { version = types.MaxVersion @@ -180,14 +180,14 @@ func (f Feature) DatabaseModel() (database.FeatureVersion, error) { var err error version, err = types.NewVersion(f.Version) if err != nil { - return database.FeatureVersion{}, err + return services.FeatureVersion{}, err } } - return database.FeatureVersion{ - Feature: database.Feature{ + return services.FeatureVersion{ + Feature: services.Feature{ Name: f.Name, - Namespace: database.Namespace{Name: f.NamespaceName}, + Namespace: services.Namespace{Name: f.NamespaceName}, }, Version: version, }, nil @@ -205,7 +205,7 @@ type Notification struct { New *VulnerabilityWithLayers `json:"New,omitempty"` } -func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotification, limit int, pageToken string, nextPage database.VulnerabilityNotificationPageNumber, key string) Notification { +func NotificationFromDatabaseModel(dbNotification services.VulnerabilityNotification, limit int, pageToken string, nextPage services.VulnerabilityNotificationPageNumber, key string) Notification { var oldVuln *VulnerabilityWithLayers if dbNotification.OldVulnerability != nil { v := VulnerabilityWithLayersFromDatabaseModel(*dbNotification.OldVulnerability) @@ -219,7 +219,7 @@ func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotifica } var nextPageStr string - if nextPage != database.NoVulnerabilityNotificationPage { + if nextPage != services.NoVulnerabilityNotificationPage { nextPageBytes, _ := tokenMarshal(nextPage, key) nextPageStr = string(nextPageBytes) } @@ -255,7 +255,7 @@ type VulnerabilityWithLayers struct { LayersIntroducingVulnerability []string `json:"LayersIntroducingVulnerability,omitempty"` } -func VulnerabilityWithLayersFromDatabaseModel(dbVuln database.Vulnerability) VulnerabilityWithLayers { +func VulnerabilityWithLayersFromDatabaseModel(dbVuln services.Vulnerability) VulnerabilityWithLayers { vuln := VulnerabilityFromDatabaseModel(dbVuln, true) var layers []string diff --git a/api/v1/routes.go b/api/v1/routes.go index 482840bf..6976ed02 100644 --- a/api/v1/routes.go +++ b/api/v1/routes.go @@ -26,7 +26,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/coreos/clair/api/context" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" @@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx return postLayerRoute, http.StatusBadRequest } - err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) + err = worker.Process(ctx.LayerService, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers) if err != nil { if err == utils.ErrCouldNotExtract || err == utils.ErrExtractedFileTooBig || @@ -142,7 +142,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * _, withFeatures := r.URL.Query()["features"] _, withVulnerabilities := r.URL.Query()["vulnerabilities"] - dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities) + dbLayer, err := ctx.LayerService.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}}) return getLayerRoute, http.StatusNotFound @@ -158,7 +158,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx * } func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - err := ctx.Store.DeleteLayer(p.ByName("layerName")) + err := ctx.LayerService.DeleteLayer(p.ByName("layerName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}}) return deleteLayerRoute, http.StatusNotFound @@ -172,7 +172,7 @@ func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ct } func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - dbNamespaces, err := ctx.Store.ListNamespaces() + dbNamespaces, err := ctx.NamespaceStore.ListNamespaces() if err != nil { writeResponse(w, r, http.StatusInternalServerError, NamespaceEnvelope{Error: &Error{err.Error()}}) return getNamespacesRoute, http.StatusInternalServerError @@ -219,7 +219,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par return getNotificationRoute, http.StatusBadRequest } - dbVulns, nextPage, err := ctx.Store.ListVulnerabilities(namespace, limit, page) + dbVulns, nextPage, err := ctx.VulnerabilityStore.ListVulnerabilities(namespace, limit, page) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return getVulnerabilityRoute, http.StatusNotFound @@ -267,7 +267,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para return postVulnerabilityRoute, http.StatusBadRequest } - err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true) + err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true) if err != nil { switch err.(type) { case *cerrors.ErrBadRequest: @@ -286,7 +286,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { _, withFixedIn := r.URL.Query()["fixedIn"] - dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) + dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return getVulnerabilityRoute, http.StatusNotFound @@ -328,7 +328,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param vuln.Namespace.Name = p.ByName("namespaceName") vuln.Name = p.ByName("vulnerabilityName") - err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true) + err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true) if err != nil { switch err.(type) { case *cerrors.ErrBadRequest: @@ -345,7 +345,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param } func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) + err := ctx.VulnerabilityStore.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}}) return deleteVulnerabilityRoute, http.StatusNotFound @@ -359,7 +359,7 @@ func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Pa } func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) + dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) return getFixesRoute, http.StatusNotFound @@ -397,7 +397,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co return putFixRoute, http.StatusBadRequest } - err = ctx.Store.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []database.FeatureVersion{dbFix}) + err = ctx.VulnerabilityStore.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []services.FeatureVersion{dbFix}) if err != nil { switch err.(type) { case *cerrors.ErrBadRequest: @@ -418,7 +418,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co } func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName")) + err := ctx.VulnerabilityStore.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}}) return deleteFixRoute, http.StatusNotFound @@ -446,7 +446,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params } var pageToken string - page := database.VulnerabilityNotificationFirstPage + page := services.VulnerabilityNotificationFirstPage pageStrs, pageExists := query["page"] if pageExists { err := tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page) @@ -464,7 +464,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params pageToken = string(pageTokenBytes) } - dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page) + dbNotification, nextPage, err := ctx.NotificationState.GetNotification(p.ByName("notificationName"), limit, page) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}}) return deleteNotificationRoute, http.StatusNotFound @@ -480,7 +480,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params } func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) { - err := ctx.Store.DeleteNotification(p.ByName("notificationName")) + err := ctx.NotificationState.DeleteNotification(p.ByName("notificationName")) if err == cerrors.ErrNotFound { writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}}) return deleteNotificationRoute, http.StatusNotFound diff --git a/clair.go b/clair.go index 788ab617..b9895338 100644 --- a/clair.go +++ b/clair.go @@ -26,8 +26,13 @@ import ( "github.com/coreos/clair/api" "github.com/coreos/clair/api/context" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" "github.com/coreos/clair/notifier" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/layers" + "github.com/coreos/clair/services/locks" + "github.com/coreos/clair/services/namespaces" + "github.com/coreos/clair/services/notifications" + "github.com/coreos/clair/services/vulnerabilities" "github.com/coreos/clair/updater" "github.com/coreos/clair/utils" "github.com/coreos/pkg/capnslog" @@ -41,26 +46,57 @@ func Boot(config *config.Config) { rand.Seed(time.Now().UnixNano()) st := utils.NewStopper() - // Open database - db, err := database.Open(config.Database) + // Open services + ls, err := locks.Open(config.Database) if err != nil { log.Fatal(err) } - defer db.Close() + defer ls.Close() + + kvs, err := keyvalue.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer kvs.Close() + + vuln, err := vulnerabilities.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer vuln.Close() + + layers, err := layers.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer layers.Close() + + names, err := namespaces.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer names.Close() + + ns, err := notifications.Open(config.Database) + if err != nil { + log.Fatal(err) + } + defer ns.Close() // Start notifier st.Begin() - go notifier.Run(config.Notifier, db, st) + go notifier.Run(config.Notifier, ls, ns, st) // Start API st.Begin() - go api.Run(config.API, &context.RouteContext{db, config.API}, st) + ctx := &context.RouteContext{ls, kvs, vuln, layers, names, ns, config.API} + go api.Run(config.API, ctx, st) st.Begin() - go api.RunHealth(config.API, &context.RouteContext{db, config.API}, st) + go api.RunHealth(config.API, ctx, st) // Start updater st.Begin() - go updater.Run(config.Updater, db, st) + go updater.Run(config.Updater, ls, kvs, vuln, st) // Wait for interruption and shutdown gracefully. waitForSignals(syscall.SIGINT, syscall.SIGTERM) diff --git a/database/database.go b/database/database.go index 4ca13e42..d08501e3 100644 --- a/database/database.go +++ b/database/database.go @@ -14,171 +14,3 @@ // Package database defines the Clair's models and a common interface for database implementations. package database - -import ( - "errors" - "fmt" - "time" - - "github.com/coreos/clair/config" -) - -var ( - // ErrBackendException is an error that occurs when the database backend does - // not work properly (ie. unreachable). - ErrBackendException = errors.New("database: an error occured when querying the backend") - - // ErrInconsistent is an error that occurs when a database consistency check - // fails (ie. when an entity which is supposed to be unique is detected twice) - ErrInconsistent = errors.New("database: inconsistent database") -) - -var drivers = make(map[string]Driver) - -// Driver is a function that opens a Datastore specified by its database driver type and specific -// configuration. -type Driver func(config.RegistrableComponentConfig) (Datastore, error) - -// Register makes a Constructor available by the provided name. -// -// If this function is called twice with the same name or if the Constructor is -// nil, it panics. -func Register(name string, driver Driver) { - if driver == nil { - panic("database: could not register nil Driver") - } - if _, dup := drivers[name]; dup { - panic("database: could not register duplicate Driver: " + name) - } - drivers[name] = driver -} - -// Open opens a Datastore specified by a configuration. -func Open(cfg config.RegistrableComponentConfig) (Datastore, error) { - driver, ok := drivers[cfg.Type] - if !ok { - return nil, fmt.Errorf("database: unknown Driver %q (forgotten configuration or import?)", cfg.Type) - } - return driver(cfg) -} - -// Datastore is the interface that describes a database backend implementation. -type Datastore interface { - // # Namespace - // ListNamespaces returns the entire list of known Namespaces. - ListNamespaces() ([]Namespace, error) - - // # Layer - // InsertLayer stores a Layer in the database. - // A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory. - // If a Parent is specified, it is expected that it has been retrieved using FindLayer. - // If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher - // than the stored one, the stored Layer should be updated. - // The function has to be idempotent, inserting a layer that already exists shouln'd return an - // error. - InsertLayer(Layer) error - - // FindLayer retrieves a Layer from the database. - // withFeatures specifies whether the Features field should be filled. When withVulnerabilities is - // true, the Features field should be filled and their AffectedBy fields should contain every - // vulnerabilities that affect them. - FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error) - - // DeleteLayer deletes a Layer from the database and every layers that are based on it, - // recursively. - DeleteLayer(name string) error - - // # Vulnerability - // ListVulnerabilities returns the list of vulnerabilies of a certain Namespace. - // The Limit and page parameters are used to paginate the return list. - // The first given page should be 0. The function will then return the next available page. - // If there is no more page, -1 has to be returned. - ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) - - // InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if - // necessary. A vulnerability is uniquely identified by its Namespace and its Name. - // The FixedIn field may only contain a partial list of Features that are affected by the - // Vulnerability, along with the version in which the vulnerability is fixed. It is the - // responsibility of the implementation to update the list properly. A version equals to - // types.MinVersion means that the given Feature is not being affected by the Vulnerability at - // all and thus, should be removed from the list. It is important that Features should be unique - // in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as - // a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and - // Features are Namespaced (i.e. specific to one operating system). - // Each vulnerability insertion or update has to create a Notification that will contain the - // old and the updated Vulnerability, unless createNotification equals to true. - InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error - - // FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list. - FindVulnerability(namespaceName, name string) (Vulnerability, error) - - // DeleteVulnerability removes a Vulnerability from the database. - // It has to create a Notification that will contain the old Vulnerability. - DeleteVulnerability(namespaceName, name string) error - - // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to - // the specified Vulnerability in the database. - // It has has to create a Notification that will contain the old and the updated Vulnerability. - InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error - - // DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the - // database. It can be used to store the fact that a Vulnerability no longer affects the given - // Feature in any Version. - // It has has to create a Notification that will contain the old and the updated Vulnerability. - DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error - - // # Notification - // GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a - // Notification that should be handled. The renotify interval defines how much time after being - // marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should - // be returned again by this function. A Notification for which there is a valid Lock with the - // same Name should not be returned. - GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error) - - // GetNotification returns a Notification, including its OldVulnerability and NewVulnerability - // fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with - // every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion). - // The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first - // given page should be VulnerabilityNotificationFirstPage. The function will then return the next - // availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned. - GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) - - // SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for - // GetAvailableNotification, until the renotify duration is elapsed. - SetNotificationNotified(name string) error - - // DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for - // GetAvailableNotification. - DeleteNotification(name string) error - - // # Key/Value - // InsertKeyValue stores or updates a simple key/value pair in the database. - InsertKeyValue(key, value string) error - - // GetKeyValue retrieves a value from the database from the given key. - // It returns an empty string if there is no such key. - GetKeyValue(key string) (string, error) - - // # Lock - // Lock creates or renew a Lock in the database with the given name, owner and duration. - // After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus, - // let other users create a Lock with the same name. However, the owner can renew its Lock by - // setting renew to true. Lock should not block, it should instead returns whether the Lock has - // been successfully acquired/renewed. If it's the case, the expiration time of that Lock is - // returned as well. - Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time) - - // Unlock releases an existing Lock. - Unlock(name, owner string) - - // FindLock returns the owner of a Lock specified by the name, and its experation time if it - // exists. - FindLock(name string) (string, time.Time, error) - - // # Miscellaneous - // Ping returns the health status of the database. - Ping() bool - - // Close closes the database and free any allocated resource. - Close() -} diff --git a/database/mock.go b/database/mock.go index 9a0963c8..cefbb0f1 100644 --- a/database/mock.go +++ b/database/mock.go @@ -14,23 +14,26 @@ package database -import "time" +import ( + "github.com/coreos/clair/services" + "time" +) // MockDatastore implements Datastore and enables overriding each available method. // The default behavior of each method is to simply panic. type MockDatastore struct { - FctListNamespaces func() ([]Namespace, error) - FctInsertLayer func(Layer) error - FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (Layer, error) + FctListNamespaces func() ([]services.Namespace, error) + FctInsertLayer func(services.Layer) error + FctFindLayer func(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) FctDeleteLayer func(name string) error - FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error) - FctInsertVulnerabilities func(vulnerabilities []Vulnerability, createNotification bool) error - FctFindVulnerability func(namespaceName, name string) (Vulnerability, error) + FctListVulnerabilities func(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error) + FctInsertVulnerabilities func(vulnerabilities []services.Vulnerability, createNotification bool) error + FctFindVulnerability func(namespaceName, name string) (services.Vulnerability, error) FctDeleteVulnerability func(namespaceName, name string) error - FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error + FctInsertVulnerabilityFixes func(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error FctDeleteVulnerabilityFix func(vulnerabilityNamespace, vulnerabilityName, featureName string) error - FctGetAvailableNotification func(renotifyInterval time.Duration) (VulnerabilityNotification, error) - FctGetNotification func(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) + FctGetAvailableNotification func(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) + FctGetNotification func(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) FctSetNotificationNotified func(name string) error FctDeleteNotification func(name string) error FctInsertKeyValue func(key, value string) error @@ -42,21 +45,21 @@ type MockDatastore struct { FctClose func() } -func (mds *MockDatastore) ListNamespaces() ([]Namespace, error) { +func (mds *MockDatastore) ListNamespaces() ([]services.Namespace, error) { if mds.FctListNamespaces != nil { return mds.FctListNamespaces() } panic("required mock function not implemented") } -func (mds *MockDatastore) InsertLayer(layer Layer) error { +func (mds *MockDatastore) InsertLayer(layer services.Layer) error { if mds.FctInsertLayer != nil { return mds.FctInsertLayer(layer) } panic("required mock function not implemented") } -func (mds *MockDatastore) FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error) { +func (mds *MockDatastore) FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) { if mds.FctFindLayer != nil { return mds.FctFindLayer(name, withFeatures, withVulnerabilities) } @@ -70,21 +73,21 @@ func (mds *MockDatastore) DeleteLayer(name string) error { panic("required mock function not implemented") } -func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) { +func (mds *MockDatastore) ListVulnerabilities(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error) { if mds.FctListVulnerabilities != nil { return mds.FctListVulnerabilities(namespaceName, limit, page) } panic("required mock function not implemented") } -func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error { +func (mds *MockDatastore) InsertVulnerabilities(vulnerabilities []services.Vulnerability, createNotification bool) error { if mds.FctInsertVulnerabilities != nil { return mds.FctInsertVulnerabilities(vulnerabilities, createNotification) } panic("required mock function not implemented") } -func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (Vulnerability, error) { +func (mds *MockDatastore) FindVulnerability(namespaceName, name string) (services.Vulnerability, error) { if mds.FctFindVulnerability != nil { return mds.FctFindVulnerability(namespaceName, name) } @@ -98,7 +101,7 @@ func (mds *MockDatastore) DeleteVulnerability(namespaceName, name string) error panic("required mock function not implemented") } -func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error { +func (mds *MockDatastore) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error { if mds.FctInsertVulnerabilityFixes != nil { return mds.FctInsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName, fixes) } @@ -112,14 +115,14 @@ func (mds *MockDatastore) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnera panic("required mock function not implemented") } -func (mds *MockDatastore) GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error) { +func (mds *MockDatastore) GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) { if mds.FctGetAvailableNotification != nil { return mds.FctGetAvailableNotification(renotifyInterval) } panic("required mock function not implemented") } -func (mds *MockDatastore) GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) { +func (mds *MockDatastore) GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) { if mds.FctGetNotification != nil { return mds.FctGetNotification(name, limit, page) } diff --git a/database/pgsql/complex_test.go b/database/pgsql/complex_test.go index 46ba504a..e0f1207e 100644 --- a/database/pgsql/complex_test.go +++ b/database/pgsql/complex_test.go @@ -26,7 +26,7 @@ import ( "github.com/pborman/uuid" "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" "github.com/coreos/clair/utils/types" ) @@ -45,8 +45,8 @@ func TestRaceAffects(t *testing.T) { defer datastore.Close() // Insert the Feature on which we'll work. - feature := database.Feature{ - Namespace: database.Namespace{Name: "TestRaceAffectsFeatureNamespace1"}, + feature := services.Feature{ + Namespace: services.Namespace{Name: "TestRaceAffectsFeatureNamespace1"}, Name: "TestRaceAffecturesFeature1", } _, err = datastore.insertFeature(feature) @@ -60,11 +60,11 @@ func TestRaceAffects(t *testing.T) { runtime.GOMAXPROCS(runtime.NumCPU()) // Generate FeatureVersions. - featureVersions := make([]database.FeatureVersion, numFeatureVersions) + featureVersions := make([]services.FeatureVersion, numFeatureVersions) for i := 0; i < numFeatureVersions; i++ { version := rand.Intn(numFeatureVersions) - featureVersions[i] = database.FeatureVersion{ + featureVersions[i] = services.FeatureVersion{ Feature: feature, Version: types.NewVersionUnsafe(strconv.Itoa(version)), } @@ -72,18 +72,18 @@ func TestRaceAffects(t *testing.T) { // Generate vulnerabilities. // They are mapped by fixed version, which will make verification really easy afterwards. - vulnerabilities := make(map[int][]database.Vulnerability) + vulnerabilities := make(map[int][]services.Vulnerability) for i := 0; i < numVulnerabilities; i++ { version := rand.Intn(numFeatureVersions) + 1 // if _, ok := vulnerabilities[version]; !ok { - // vulnerabilities[version] = make([]database.Vulnerability) + // vulnerabilities[version] = make([]services.Vulnerability) // } - vulnerability := database.Vulnerability{ + vulnerability := services.Vulnerability{ Name: uuid.New(), Namespace: feature.Namespace, - FixedIn: []database.FeatureVersion{ + FixedIn: []services.FeatureVersion{ { Feature: feature, Version: types.NewVersionUnsafe(strconv.Itoa(version)), @@ -103,7 +103,7 @@ func TestRaceAffects(t *testing.T) { defer wg.Done() for _, vulnerabilitiesM := range vulnerabilities { for _, vulnerability := range vulnerabilitiesM { - err = datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true) + err = datastore.InsertVulnerabilities([]services.Vulnerability{vulnerability}, true) assert.Nil(t, err) } } diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go index a2f2abe8..b78cbec1 100644 --- a/database/pgsql/feature.go +++ b/database/pgsql/feature.go @@ -18,12 +18,12 @@ import ( "database/sql" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) -func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { +func (pgSQL *pgSQL) insertFeature(feature services.Feature) (int, error) { if feature.Name == "" { return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature") } @@ -61,7 +61,7 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) { return id, nil } -func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) { +func (pgSQL *pgSQL) insertFeatureVersion(featureVersion services.FeatureVersion) (id int, err error) { if featureVersion.Version.String() == "" { return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion") } @@ -177,7 +177,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) } // TODO(Quentin-M): Batch me -func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) { +func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []services.FeatureVersion) ([]int, error) { IDs := make([]int, 0, len(featureVersions)) for i := 0; i < len(featureVersions); i++ { @@ -197,7 +197,7 @@ type vulnerabilityAffectsFeatureVersion struct { fixedInVersion types.Version } -func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error { +func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion services.FeatureVersion) error { // Select every vulnerability and the fixed version that affect this Feature. // TODO(Quentin-M): LIMIT rows, err := tx.Query(searchVulnerabilityFixedInFeature, featureVersion.Feature.ID) diff --git a/database/pgsql/feature_test.go b/database/pgsql/feature_test.go index a857c30f..8a08e247 100644 --- a/database/pgsql/feature_test.go +++ b/database/pgsql/feature_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" ) @@ -32,20 +32,20 @@ func TestInsertFeature(t *testing.T) { defer datastore.Close() // Invalid Feature. - id0, err := datastore.insertFeature(database.Feature{}) + id0, err := datastore.insertFeature(services.Feature{}) assert.NotNil(t, err) assert.Zero(t, id0) - id0, err = datastore.insertFeature(database.Feature{ - Namespace: database.Namespace{}, + id0, err = datastore.insertFeature(services.Feature{ + Namespace: services.Namespace{}, Name: "TestInsertFeature0", }) assert.NotNil(t, err) assert.Zero(t, id0) // Insert Feature and ensure we can find it. - feature := database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, + feature := services.Feature{ + Namespace: services.Namespace{Name: "TestInsertFeatureNamespace1"}, Name: "TestInsertFeature1", } id1, err := datastore.insertFeature(feature) @@ -55,28 +55,28 @@ func TestInsertFeature(t *testing.T) { assert.Equal(t, id1, id2) // Insert invalid FeatureVersion. - for _, invalidFeatureVersion := range []database.FeatureVersion{ + for _, invalidFeatureVersion := range []services.FeatureVersion{ { - Feature: database.Feature{}, + Feature: services.Feature{}, Version: types.NewVersionUnsafe("1.0"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{}, + Feature: services.Feature{ + Namespace: services.Namespace{}, Name: "TestInsertFeature2", }, Version: types.NewVersionUnsafe("1.0"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertFeatureNamespace2"}, Name: "TestInsertFeature2", }, Version: types.NewVersionUnsafe(""), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace2"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertFeatureNamespace2"}, Name: "TestInsertFeature2", }, Version: types.NewVersionUnsafe("bad version"), @@ -88,9 +88,9 @@ func TestInsertFeature(t *testing.T) { } // Insert FeatureVersion and ensure we can find it. - featureVersion := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertFeatureNamespace1"}, + featureVersion := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertFeatureNamespace1"}, Name: "TestInsertFeature1", }, Version: types.NewVersionUnsafe("2:3.0-imba"), diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index 66a41000..f756e4a5 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -18,13 +18,13 @@ import ( "database/sql" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/guregu/null/zero" ) -func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) { +func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) { subquery := "all" if withFeatures { subquery += "/features" @@ -34,7 +34,7 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo defer observeQueryTime("FindLayer", subquery, time.Now()) // Find the layer - var layer database.Layer + var layer services.Layer var parentID zero.Int var parentName zero.String var namespaceID zero.Int @@ -49,14 +49,14 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo } if !parentID.IsZero() { - layer.Parent = &database.Layer{ - Model: database.Model{ID: int(parentID.Int64)}, + layer.Parent = &services.Layer{ + Model: services.Model{ID: int(parentID.Int64)}, Name: parentName.String, } } if !namespaceID.IsZero() { - layer.Namespace = &database.Namespace{ - Model: database.Model{ID: int(namespaceID.Int64)}, + layer.Namespace = &services.Namespace{ + Model: services.Model{ID: int(namespaceID.Int64)}, Name: namespaceName.String, } } @@ -110,9 +110,9 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo return layer, nil } -// getLayerFeatureVersions returns list of database.FeatureVersion that a database.Layer has. -func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion, error) { - var featureVersions []database.FeatureVersion +// getLayerFeatureVersions returns list of services.FeatureVersion that a services.Layer has. +func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]services.FeatureVersion, error) { + var featureVersions []services.FeatureVersion // Query. rows, err := tx.Query(searchLayerFeatureVersion, layerID) @@ -123,9 +123,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion // Scan query. var modification string - mapFeatureVersions := make(map[int]database.FeatureVersion) + mapFeatureVersions := make(map[int]services.FeatureVersion) for rows.Next() { - var featureVersion database.FeatureVersion + var featureVersion services.FeatureVersion err = rows.Scan(&featureVersion.ID, &modification, &featureVersion.Feature.Namespace.ID, &featureVersion.Feature.Namespace.Name, &featureVersion.Feature.ID, @@ -143,7 +143,7 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion delete(mapFeatureVersions, featureVersion.ID) default: log.Warningf("unknown Layer_diff_FeatureVersion's modification: %s", modification) - return featureVersions, database.ErrInconsistent + return featureVersions, services.ErrInconsistent } } if err = rows.Err(); err != nil { @@ -158,9 +158,9 @@ func getLayerFeatureVersions(tx *sql.Tx, layerID int) ([]database.FeatureVersion return featureVersions, nil } -// loadAffectedBy returns the list of database.Vulnerability that affect the given +// loadAffectedBy returns the list of services.Vulnerability that affect the given // FeatureVersion. -func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error { +func loadAffectedBy(tx *sql.Tx, featureVersions []services.FeatureVersion) error { if len(featureVersions) == 0 { return nil } @@ -178,10 +178,10 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error } defer rows.Close() - vulnerabilities := make(map[int][]database.Vulnerability, len(featureVersions)) + vulnerabilities := make(map[int][]services.Vulnerability, len(featureVersions)) var featureversionID int for rows.Next() { - var vulnerability database.Vulnerability + var vulnerability services.Vulnerability err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, &vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy) @@ -209,7 +209,7 @@ func loadAffectedBy(tx *sql.Tx, featureVersions []database.FeatureVersion) error // (happens when Feature detectors relies on the detected layer Namespace). However, if the listed // Feature has the same Name/Version as its parent, InsertLayer considers that the Feature hasn't // been modified. -func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { +func (pgSQL *pgSQL) InsertLayer(layer services.Layer) error { tf := time.Now() // Verify parameters @@ -314,10 +314,10 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error { return nil } -func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error { +func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *services.Layer) error { // add and del are the FeatureVersion diff we should insert. - var add []database.FeatureVersion - var del []database.FeatureVersion + var add []services.FeatureVersion + var del []services.FeatureVersion if layer.Parent == nil { // There is no parent, every Features are added. @@ -369,8 +369,8 @@ func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer * return nil } -func createNV(features []database.FeatureVersion) (map[string]*database.FeatureVersion, []string) { - mapNV := make(map[string]*database.FeatureVersion, 0) +func createNV(features []services.FeatureVersion) (map[string]*services.FeatureVersion, []string) { + mapNV := make(map[string]*services.FeatureVersion, 0) sliceNV := make([]string, 0, len(features)) for i := 0; i < len(features); i++ { diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer_test.go index c45cbbed..1d9f4153 100644 --- a/database/pgsql/layer_test.go +++ b/database/pgsql/layer_test.go @@ -20,7 +20,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/layers" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -123,93 +124,93 @@ func TestInsertLayer(t *testing.T) { testInsertLayerDelete(t, datastore) } -func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) { - invalidLayers := []database.Layer{ +func testInsertLayerInvalid(t *testing.T, ls layers.Service) { + invalidLayers := []services.Layer{ {}, - {Name: "layer0", Parent: &database.Layer{}}, - {Name: "layer0", Parent: &database.Layer{Name: "UnknownLayer"}}, + {Name: "layer0", Parent: &services.Layer{}}, + {Name: "layer0", Parent: &services.Layer{Name: "UnknownLayer"}}, } for _, invalidLayer := range invalidLayers { - err := datastore.InsertLayer(invalidLayer) + err := ls.InsertLayer(invalidLayer) assert.Error(t, err) } } -func testInsertLayerTree(t *testing.T, datastore database.Datastore) { - f1 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, +func testInsertLayerTree(t *testing.T, ls layers.Service) { + f1 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"}, Name: "TestInsertLayerFeature1", }, Version: types.NewVersionUnsafe("1.0"), } - f2 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, + f2 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"}, Name: "TestInsertLayerFeature2", }, Version: types.NewVersionUnsafe("0.34"), } - f3 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"}, + f3 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace2"}, Name: "TestInsertLayerFeature3", }, Version: types.NewVersionUnsafe("0.56"), } - f4 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + f4 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"}, Name: "TestInsertLayerFeature2", }, Version: types.NewVersionUnsafe("0.34"), } - f5 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + f5 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"}, Name: "TestInsertLayerFeature3", }, Version: types.NewVersionUnsafe("0.56"), } - f6 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, + f6 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"}, Name: "TestInsertLayerFeature4", }, Version: types.NewVersionUnsafe("0.666"), } - layers := []database.Layer{ + layers := []services.Layer{ { Name: "TestInsertLayer1", }, { Name: "TestInsertLayer2", - Parent: &database.Layer{Name: "TestInsertLayer1"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace1"}, + Parent: &services.Layer{Name: "TestInsertLayer1"}, + Namespace: &services.Namespace{Name: "TestInsertLayerNamespace1"}, }, // This layer changes the namespace and adds Features. { Name: "TestInsertLayer3", - Parent: &database.Layer{Name: "TestInsertLayer2"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"}, - Features: []database.FeatureVersion{f1, f2, f3}, + Parent: &services.Layer{Name: "TestInsertLayer2"}, + Namespace: &services.Namespace{Name: "TestInsertLayerNamespace2"}, + Features: []services.FeatureVersion{f1, f2, f3}, }, // This layer covers the case where the last layer doesn't provide any new Feature. { Name: "TestInsertLayer4a", - Parent: &database.Layer{Name: "TestInsertLayer3"}, - Features: []database.FeatureVersion{f1, f2, f3}, + Parent: &services.Layer{Name: "TestInsertLayer3"}, + Features: []services.FeatureVersion{f1, f2, f3}, }, // This layer covers the case where the last layer provides Features. // It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their // Namespaces should then remain unchanged. { Name: "TestInsertLayer4b", - Parent: &database.Layer{Name: "TestInsertLayer3"}, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespace3"}, - Features: []database.FeatureVersion{ + Parent: &services.Layer{Name: "TestInsertLayer3"}, + Namespace: &services.Namespace{Name: "TestInsertLayerNamespace3"}, + Features: []services.FeatureVersion{ // Deletes TestInsertLayerFeature1. // Keep TestInsertLayerFeature2 (old Namespace should be kept): f4, @@ -222,7 +223,7 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } var err error - retrievedLayers := make(map[string]database.Layer) + retrievedLayers := make(map[string]services.Layer) for _, layer := range layers { if layer.Parent != nil { // Retrieve from database its parent and assign. @@ -230,10 +231,10 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { layer.Parent = &parent } - err = datastore.InsertLayer(layer) + err = ls.InsertLayer(layer) assert.Nil(t, err) - retrievedLayers[layer.Name], err = datastore.FindLayer(layer.Name, true, false) + retrievedLayers[layer.Name], err = ls.FindLayer(layer.Name, true, false) assert.Nil(t, err) } @@ -260,35 +261,35 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) { } } -func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { - f7 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"}, +func testInsertLayerUpdate(t *testing.T, ls layers.Service) { + f7 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "TestInsertLayerNamespace3"}, Name: "TestInsertLayerFeature7", }, Version: types.NewVersionUnsafe("0.01"), } - l3, _ := datastore.FindLayer("TestInsertLayer3", true, false) - l3u := database.Layer{ + l3, _ := ls.FindLayer("TestInsertLayer3", true, false) + l3u := services.Layer{ Name: l3.Name, Parent: l3.Parent, - Namespace: &database.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}, - Features: []database.FeatureVersion{f7}, + Namespace: &services.Namespace{Name: "TestInsertLayerNamespaceUpdated1"}, + Features: []services.FeatureVersion{f7}, } - l4u := database.Layer{ + l4u := services.Layer{ Name: "TestInsertLayer4", - Parent: &database.Layer{Name: "TestInsertLayer3"}, - Features: []database.FeatureVersion{f7}, + Parent: &services.Layer{Name: "TestInsertLayer3"}, + Features: []services.FeatureVersion{f7}, EngineVersion: 2, } // Try to re-insert without increasing the EngineVersion. - err := datastore.InsertLayer(l3u) + err := ls.InsertLayer(l3u) assert.Nil(t, err) - l3uf, err := datastore.FindLayer(l3u.Name, true, false) + l3uf, err := ls.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { assert.Equal(t, l3.Namespace.Name, l3uf.Namespace.Name) assert.Equal(t, l3.EngineVersion, l3uf.EngineVersion) @@ -298,10 +299,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { // Update layer l3. // Verify that the Namespace, EngineVersion and FeatureVersions got updated. l3u.EngineVersion = 2 - err = datastore.InsertLayer(l3u) + err = ls.InsertLayer(l3u) assert.Nil(t, err) - l3uf, err = datastore.FindLayer(l3u.Name, true, false) + l3uf, err = ls.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { assert.Equal(t, l3u.Namespace.Name, l3uf.Namespace.Name) assert.Equal(t, l3u.EngineVersion, l3uf.EngineVersion) @@ -314,10 +315,10 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { // Verify that the Namespace got updated from its new Parent's, and also verify the // EnginVersion and FeatureVersions. l4u.Parent = &l3uf - err = datastore.InsertLayer(l4u) + err = ls.InsertLayer(l4u) assert.Nil(t, err) - l4uf, err := datastore.FindLayer(l3u.Name, true, false) + l4uf, err := ls.FindLayer(l3u.Name, true, false) if assert.Nil(t, err) { assert.Equal(t, l3u.Namespace.Name, l4uf.Namespace.Name) assert.Equal(t, l4u.EngineVersion, l4uf.EngineVersion) @@ -327,24 +328,24 @@ func testInsertLayerUpdate(t *testing.T, datastore database.Datastore) { } } -func testInsertLayerDelete(t *testing.T, datastore database.Datastore) { - err := datastore.DeleteLayer("TestInsertLayerX") +func testInsertLayerDelete(t *testing.T, ls layers.Service) { + err := ls.DeleteLayer("TestInsertLayerX") assert.Equal(t, cerrors.ErrNotFound, err) - err = datastore.DeleteLayer("TestInsertLayer3") + err = ls.DeleteLayer("TestInsertLayer3") assert.Nil(t, err) - _, err = datastore.FindLayer("TestInsertLayer3", false, false) + _, err = ls.FindLayer("TestInsertLayer3", false, false) assert.Equal(t, cerrors.ErrNotFound, err) - _, err = datastore.FindLayer("TestInsertLayer4a", false, false) + _, err = ls.FindLayer("TestInsertLayer4a", false, false) assert.Equal(t, cerrors.ErrNotFound, err) - _, err = datastore.FindLayer("TestInsertLayer4b", true, false) + _, err = ls.FindLayer("TestInsertLayer4b", true, false) assert.Equal(t, cerrors.ErrNotFound, err) } -func cmpFV(a, b database.FeatureVersion) bool { +func cmpFV(a, b services.FeatureVersion) bool { return a.Feature.Name == b.Feature.Name && a.Feature.Namespace.Name == b.Feature.Namespace.Name && a.Version.String() == b.Version.String() diff --git a/database/pgsql/namespace.go b/database/pgsql/namespace.go index 3c85c784..cfefc635 100644 --- a/database/pgsql/namespace.go +++ b/database/pgsql/namespace.go @@ -17,11 +17,11 @@ package pgsql import ( "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" ) -func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { +func (pgSQL *pgSQL) insertNamespace(namespace services.Namespace) (int, error) { if namespace.Name == "" { return 0, cerrors.NewBadRequestError("could not find/insert invalid Namespace") } @@ -50,7 +50,7 @@ func (pgSQL *pgSQL) insertNamespace(namespace database.Namespace) (int, error) { return id, nil } -func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error) { +func (pgSQL *pgSQL) ListNamespaces() (namespaces []services.Namespace, err error) { rows, err := pgSQL.Query(listNamespace) if err != nil { return namespaces, handleError("listNamespace", err) @@ -58,7 +58,7 @@ func (pgSQL *pgSQL) ListNamespaces() (namespaces []database.Namespace, err error defer rows.Close() for rows.Next() { - var namespace database.Namespace + var namespace services.Namespace err = rows.Scan(&namespace.ID, &namespace.Name) if err != nil { diff --git a/database/pgsql/namespace_test.go b/database/pgsql/namespace_test.go index b9bf96fe..ec539354 100644 --- a/database/pgsql/namespace_test.go +++ b/database/pgsql/namespace_test.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" ) func TestInsertNamespace(t *testing.T) { @@ -32,14 +32,14 @@ func TestInsertNamespace(t *testing.T) { defer datastore.Close() // Invalid Namespace. - id0, err := datastore.insertNamespace(database.Namespace{}) + id0, err := datastore.insertNamespace(services.Namespace{}) assert.NotNil(t, err) assert.Zero(t, id0) // Insert Namespace and ensure we can find it. - id1, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) + id1, err := datastore.insertNamespace(services.Namespace{Name: "TestInsertNamespace1"}) assert.Nil(t, err) - id2, err := datastore.insertNamespace(database.Namespace{Name: "TestInsertNamespace1"}) + id2, err := datastore.insertNamespace(services.Namespace{Name: "TestInsertNamespace1"}) assert.Nil(t, err) assert.Equal(t, id1, id2) } diff --git a/database/pgsql/notification.go b/database/pgsql/notification.go index 3d42107d..c5f23caa 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification.go @@ -18,7 +18,7 @@ import ( "database/sql" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" "github.com/guregu/null/zero" "github.com/pborman/uuid" @@ -43,7 +43,7 @@ func createNotification(tx *sql.Tx, oldVulnerabilityID, newVulnerabilityID int) // Get one available notification name (!locked && !deleted && (!notified || notified_but_timed-out)). // Does not fill new/old vuln. -func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (database.VulnerabilityNotification, error) { +func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) { defer observeQueryTime("GetAvailableNotification", "all", time.Now()) before := time.Now().Add(-renotifyInterval) @@ -53,7 +53,7 @@ func (pgSQL *pgSQL) GetAvailableNotification(renotifyInterval time.Duration) (da return notification, handleError("searchNotificationAvailable", err) } -func (pgSQL *pgSQL) GetNotification(name string, limit int, page database.VulnerabilityNotificationPageNumber) (database.VulnerabilityNotification, database.VulnerabilityNotificationPageNumber, error) { +func (pgSQL *pgSQL) GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) { defer observeQueryTime("GetNotification", "all", time.Now()) // Get Notification. @@ -86,8 +86,8 @@ func (pgSQL *pgSQL) GetNotification(name string, limit int, page database.Vulner return notification, page, nil } -func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (database.VulnerabilityNotification, error) { - var notification database.VulnerabilityNotification +func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (services.VulnerabilityNotification, error) { + var notification services.VulnerabilityNotification var created zero.Time var notified zero.Time var deleted zero.Time @@ -147,7 +147,7 @@ func (pgSQL *pgSQL) scanNotification(row *sql.Row, hasVulns bool) (database.Vuln // Fills Vulnerability.LayersIntroducingVulnerability. // limit -1: won't do anything // limit 0: will just get the startID of the second page -func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *database.Vulnerability, limit, startID int) (int, error) { +func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *services.Vulnerability, limit, startID int) (int, error) { tf := time.Now() if vulnerability == nil { @@ -170,9 +170,9 @@ func (pgSQL *pgSQL) loadLayerIntroducingVulnerability(vulnerability *database.Vu } defer rows.Close() - var layers []database.Layer + var layers []services.Layer for rows.Next() { - var layer database.Layer + var layer services.Layer if err := rows.Scan(&layer.ID, &layer.Name); err != nil { return -1, handleError("searchNotificationLayerIntroducingVulnerability.Scan()", err) diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification_test.go index 3f90f349..91f61894 100644 --- a/database/pgsql/notification_test.go +++ b/database/pgsql/notification_test.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -38,19 +38,19 @@ func TestNotification(t *testing.T) { assert.Equal(t, cerrors.ErrNotFound, err) // Create some data. - f1 := database.Feature{ + f1 := services.Feature{ Name: "TestNotificationFeature1", - Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, + Namespace: services.Namespace{Name: "TestNotificationNamespace1"}, } - f2 := database.Feature{ + f2 := services.Feature{ Name: "TestNotificationFeature2", - Namespace: database.Namespace{Name: "TestNotificationNamespace1"}, + Namespace: services.Namespace{Name: "TestNotificationNamespace1"}, } - l1 := database.Layer{ + l1 := services.Layer{ Name: "TestNotificationLayer1", - Features: []database.FeatureVersion{ + Features: []services.FeatureVersion{ { Feature: f1, Version: types.NewVersionUnsafe("0.1"), @@ -58,9 +58,9 @@ func TestNotification(t *testing.T) { }, } - l2 := database.Layer{ + l2 := services.Layer{ Name: "TestNotificationLayer2", - Features: []database.FeatureVersion{ + Features: []services.FeatureVersion{ { Feature: f1, Version: types.NewVersionUnsafe("0.2"), @@ -68,9 +68,9 @@ func TestNotification(t *testing.T) { }, } - l3 := database.Layer{ + l3 := services.Layer{ Name: "TestNotificationLayer3", - Features: []database.FeatureVersion{ + Features: []services.FeatureVersion{ { Feature: f1, Version: types.NewVersionUnsafe("0.3"), @@ -78,9 +78,9 @@ func TestNotification(t *testing.T) { }, } - l4 := database.Layer{ + l4 := services.Layer{ Name: "TestNotificationLayer4", - Features: []database.FeatureVersion{ + Features: []services.FeatureVersion{ { Feature: f2, Version: types.NewVersionUnsafe("0.1"), @@ -96,13 +96,13 @@ func TestNotification(t *testing.T) { } // Insert a new vulnerability that is introduced by three layers. - v1 := database.Vulnerability{ + v1 := services.Vulnerability{ Name: "TestNotificationVulnerability1", Namespace: f1.Namespace, Description: "TestNotificationDescription1", Link: "TestNotificationLink1", Severity: "Unknown", - FixedIn: []database.FeatureVersion{ + FixedIn: []services.FeatureVersion{ { Feature: f1, Version: types.NewVersionUnsafe("1.0"), @@ -129,9 +129,9 @@ func TestNotification(t *testing.T) { } // Get notification. - filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage) + filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage) if assert.Nil(t, err) { - assert.NotEqual(t, database.NoVulnerabilityNotificationPage, nextPage) + assert.NotEqual(t, services.NoVulnerabilityNotificationPage, nextPage) assert.Nil(t, filledNotification.OldVulnerability) if assert.NotNil(t, filledNotification.NewVulnerability) { @@ -143,7 +143,7 @@ func TestNotification(t *testing.T) { // Get second page. filledNotification, nextPage, err = datastore.GetNotification(notification.Name, 2, nextPage) if assert.Nil(t, err) { - assert.Equal(t, database.NoVulnerabilityNotificationPage, nextPage) + assert.Equal(t, services.NoVulnerabilityNotificationPage, nextPage) assert.Nil(t, filledNotification.OldVulnerability) if assert.NotNil(t, filledNotification.NewVulnerability) { @@ -162,7 +162,7 @@ func TestNotification(t *testing.T) { // Update a vulnerability and ensure that the old/new vulnerabilities are correct. v1b := v1 v1b.Severity = types.High - v1b.FixedIn = []database.FeatureVersion{ + v1b.FixedIn = []services.FeatureVersion{ { Feature: f1, Version: types.MinVersion, @@ -179,7 +179,7 @@ func TestNotification(t *testing.T) { assert.NotEmpty(t, notification.Name) if assert.Nil(t, err) && assert.NotEmpty(t, notification.Name) { - filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage) + filledNotification, nextPage, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage) if assert.Nil(t, err) { if assert.NotNil(t, filledNotification.OldVulnerability) { assert.Equal(t, v1.Name, filledNotification.OldVulnerability.Name) @@ -207,7 +207,7 @@ func TestNotification(t *testing.T) { assert.NotEmpty(t, notification.Name) if assert.Nil(t, err) && assert.NotEmpty(t, notification.Name) { - filledNotification, _, err := datastore.GetNotification(notification.Name, 2, database.VulnerabilityNotificationFirstPage) + filledNotification, _, err := datastore.GetNotification(notification.Name, 2, services.VulnerabilityNotificationFirstPage) if assert.Nil(t, err) { assert.Nil(t, filledNotification.NewVulnerability) diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go index b29dad00..87188d2d 100644 --- a/database/pgsql/pgsql.go +++ b/database/pgsql/pgsql.go @@ -27,13 +27,19 @@ import ( "bitbucket.org/liamstask/goose/lib/goose" "github.com/coreos/pkg/capnslog" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" - "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v2" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/layers" + "github.com/coreos/clair/services/locks" + "github.com/coreos/clair/services/namespaces" + "github.com/coreos/clair/services/notifications" + "github.com/coreos/clair/services/vulnerabilities" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" ) @@ -74,7 +80,25 @@ func init() { prometheus.MustRegister(promQueryDurationMilliseconds) prometheus.MustRegister(promConcurrentLockVAFV) - database.Register("pgsql", openDatabase) + // The types must match exactly because Go doesn't seem to allow covariance. + locks.Register("pgsql", func(cfg config.RegistrableComponentConfig) (locks.Service, error) { + return openDatabase(cfg) + }) + keyvalue.Register("pgsql", func(cfg config.RegistrableComponentConfig) (keyvalue.Service, error) { + return openDatabase(cfg) + }) + vulnerabilities.Register("pgsql", func(cfg config.RegistrableComponentConfig) (vulnerabilities.Service, error) { + return openDatabase(cfg) + }) + layers.Register("pgsql", func(cfg config.RegistrableComponentConfig) (layers.Service, error) { + return openDatabase(cfg) + }) + notifications.Register("pgsql", func(cfg config.RegistrableComponentConfig) (notifications.Service, error) { + return openDatabase(cfg) + }) + namespaces.Register("pgsql", func(cfg config.RegistrableComponentConfig) (namespaces.Service, error) { + return openDatabase(cfg) + }) } type Queryer interface { @@ -119,7 +143,7 @@ type Config struct { // It immediately every necessary migrations. If ManageDatabaseLifecycle is specified, // the database will be created first. If FixturePath is specified, every SQL queries that are // present insides will be executed. -func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) (database.Datastore, error) { +func openDatabase(registrableComponentConfig config.RegistrableComponentConfig) (*pgSQL, error) { var pg pgSQL var err error @@ -306,7 +330,7 @@ func handleError(desc string, err error) error { promErrorsTotal.WithLabelValues(desc).Inc() if _, o := err.(*pq.Error); o || err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") { - return database.ErrBackendException + return services.ErrBackendException } return err diff --git a/database/pgsql/pgsql_test.go b/database/pgsql/pgsql_test.go index 58c516cc..52be0636 100644 --- a/database/pgsql/pgsql_test.go +++ b/database/pgsql/pgsql_test.go @@ -26,12 +26,7 @@ import ( ) func openDatabaseForTest(testName string, loadFixture bool) (*pgSQL, error) { - ds, err := openDatabase(generateTestConfig(testName, loadFixture)) - if err != nil { - return nil, err - } - datastore := ds.(*pgSQL) - return datastore, nil + return openDatabase(generateTestConfig(testName, loadFixture)) } func generateTestConfig(testName string, loadFixture bool) config.RegistrableComponentConfig { diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index 74ee9828..7a6a2a7b 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -21,14 +21,14 @@ import ( "reflect" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" "github.com/guregu/null/zero" ) -func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) { +func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]services.Vulnerability, int, error) { defer observeQueryTime("listVulnerabilities", "all", time.Now()) // Query Namespace. @@ -48,12 +48,12 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID } defer rows.Close() - var vulns []database.Vulnerability + var vulns []services.Vulnerability nextID := -1 size := 0 // Scan query. for rows.Next() { - var vulnerability database.Vulnerability + var vulnerability services.Vulnerability err := rows.Scan( &vulnerability.ID, @@ -83,11 +83,11 @@ func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID return vulns, nextID, nil } -func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vulnerability, error) { +func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (services.Vulnerability, error) { return findVulnerability(pgSQL, namespaceName, name, false) } -func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (database.Vulnerability, error) { +func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bool) (services.Vulnerability, error) { defer observeQueryTime("findVulnerability", "all", time.Now()) queryName := "searchVulnerabilityBase+searchVulnerabilityByNamespaceAndName" @@ -100,7 +100,7 @@ func findVulnerability(queryer Queryer, namespaceName, name string, forUpdate bo return scanVulnerability(queryer, queryName, queryer.QueryRow(query, namespaceName, name)) } -func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerability, error) { +func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (services.Vulnerability, error) { defer observeQueryTime("findVulnerabilityByIDWithDeleted", "all", time.Now()) queryName := "searchVulnerabilityBase+searchVulnerabilityByID" @@ -109,8 +109,8 @@ func (pgSQL *pgSQL) findVulnerabilityByIDWithDeleted(id int) (database.Vulnerabi return scanVulnerability(pgSQL, queryName, pgSQL.QueryRow(query, id)) } -func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.Row) (database.Vulnerability, error) { - var vulnerability database.Vulnerability +func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql.Row) (services.Vulnerability, error) { + var vulnerability services.Vulnerability err := vulnerabilityRow.Scan( &vulnerability.ID, @@ -156,10 +156,10 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql. if !featureVersionID.IsZero() { // Note that the ID we fill in featureVersion is actually a Feature ID, and not // a FeatureVersion ID. - featureVersion := database.FeatureVersion{ - Model: database.Model{ID: int(featureVersionID.Int64)}, - Feature: database.Feature{ - Model: database.Model{ID: int(featureVersionID.Int64)}, + featureVersion := services.FeatureVersion{ + Model: services.Model{ID: int(featureVersionID.Int64)}, + Feature: services.Feature{ + Model: services.Model{ID: int(featureVersionID.Int64)}, Namespace: vulnerability.Namespace, Name: featureVersionFeatureName.String, }, @@ -178,7 +178,7 @@ func scanVulnerability(queryer Queryer, queryName string, vulnerabilityRow *sql. // FixedIn.Namespace are not necessary, they are overwritten by the vuln. // By setting the fixed version to minVersion, we can say that the vuln does'nt affect anymore. -func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerability, generateNotifications bool) error { +func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []services.Vulnerability, generateNotifications bool) error { for _, vulnerability := range vulnerabilities { err := pgSQL.insertVulnerability(vulnerability, false, generateNotifications) if err != nil { @@ -189,7 +189,7 @@ func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerabili return nil } -func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, onlyFixedIn, generateNotification bool) error { +func (pgSQL *pgSQL) insertVulnerability(vulnerability services.Vulnerability, onlyFixedIn, generateNotification bool) error { tf := time.Now() // Verify parameters @@ -272,7 +272,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on } else { // The vulnerability is new, we don't want to have any types.MinVersion as they are only used // for diffing existing vulnerabilities. - var fixedIn []database.FeatureVersion + var fixedIn []services.FeatureVersion for _, fv := range vulnerability.FixedIn { if fv.Version != types.MinVersion { fixedIn = append(fixedIn, fv) @@ -328,19 +328,19 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability, on return nil } -// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that +// castMetadata marshals the given services.MetadataMap and unmarshals it again to make sure that // everything has the interface{} type. // It is required when comparing crafted MetadataMap against MetadataMap that we get from the // database. -func castMetadata(m database.MetadataMap) database.MetadataMap { - c := make(database.MetadataMap) +func castMetadata(m services.MetadataMap) services.MetadataMap { + c := make(services.MetadataMap) j, _ := json.Marshal(m) json.Unmarshal(j, &c) return c } // applyFixedInDiff applies a FeatureVersion diff on a FeatureVersion list and returns the result. -func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.FeatureVersion, bool) { +func applyFixedInDiff(currentList, diff []services.FeatureVersion) ([]services.FeatureVersion, bool) { currentMap, currentNames := createFeatureVersionNameMap(currentList) diffMap, diffNames := createFeatureVersionNameMap(diff) @@ -375,7 +375,7 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F } // Convert currentMap to a slice and return it. - var newList []database.FeatureVersion + var newList []services.FeatureVersion for _, fv := range currentMap { newList = append(newList, fv) } @@ -383,8 +383,8 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F return newList, different } -func createFeatureVersionNameMap(features []database.FeatureVersion) (map[string]database.FeatureVersion, []string) { - m := make(map[string]database.FeatureVersion, 0) +func createFeatureVersionNameMap(features []services.FeatureVersion) (map[string]services.FeatureVersion, []string) { + m := make(map[string]services.FeatureVersion, 0) s := make([]string, 0, len(features)) for i := 0; i < len(features); i++ { @@ -397,16 +397,16 @@ func createFeatureVersionNameMap(features []database.FeatureVersion) (map[string } // insertVulnerabilityFixedInFeatureVersions populates Vulnerability_FixedIn_Feature for the given -// vulnerability with the specified database.FeatureVersion list and uses +// vulnerability with the specified services.FeatureVersion list and uses // linkVulnerabilityToFeatureVersions to propagate the changes on Vulnerability_FixedIn_Feature to // Vulnerability_Affects_FeatureVersion. -func (pgSQL *pgSQL) insertVulnerabilityFixedInFeatureVersions(tx *sql.Tx, vulnerabilityID int, fixedIn []database.FeatureVersion) error { +func (pgSQL *pgSQL) insertVulnerabilityFixedInFeatureVersions(tx *sql.Tx, vulnerabilityID int, fixedIn []services.FeatureVersion) error { defer observeQueryTime("insertVulnerabilityFixedInFeatureVersions", "all", time.Now()) // Insert or find the Features. // TODO(Quentin-M): Batch me. var err error - var features []*database.Feature + var features []*services.Feature for i := 0; i < len(fixedIn); i++ { features = append(features, &fixedIn[i].Feature) } @@ -464,9 +464,9 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID, } defer rows.Close() - var affecteds []database.FeatureVersion + var affecteds []services.FeatureVersion for rows.Next() { - var affected database.FeatureVersion + var affected services.FeatureVersion err := rows.Scan(&affected.ID, &affected.Version) if err != nil { @@ -497,12 +497,12 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID, return nil } -func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []database.FeatureVersion) error { +func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error { defer observeQueryTime("InsertVulnerabilityFixes", "all", time.Now()) - v := database.Vulnerability{ + v := services.Vulnerability{ Name: vulnerabilityName, - Namespace: database.Namespace{ + Namespace: services.Namespace{ Name: vulnerabilityNamespace, }, FixedIn: fixes, @@ -514,16 +514,16 @@ func (pgSQL *pgSQL) InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabili func (pgSQL *pgSQL) DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error { defer observeQueryTime("DeleteVulnerabilityFix", "all", time.Now()) - v := database.Vulnerability{ + v := services.Vulnerability{ Name: vulnerabilityName, - Namespace: database.Namespace{ + Namespace: services.Namespace{ Name: vulnerabilityNamespace, }, - FixedIn: []database.FeatureVersion{ + FixedIn: []services.FeatureVersion{ { - Feature: database.Feature{ + Feature: services.Feature{ Name: featureName, - Namespace: database.Namespace{ + Namespace: services.Namespace{ Name: vulnerabilityNamespace, }, }, diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index d20c2e35..71446419 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" ) @@ -38,19 +38,19 @@ func TestFindVulnerability(t *testing.T) { assert.Equal(t, cerrors.ErrNotFound, err) // Find a normal vulnerability. - v1 := database.Vulnerability{ + v1 := services.Vulnerability{ Name: "CVE-OPENSSL-1-DEB7", Description: "A vulnerability affecting OpenSSL < 2.0 on Debian 7.0", Link: "http://google.com/#q=CVE-OPENSSL-1-DEB7", Severity: types.High, - Namespace: database.Namespace{Name: "debian:7"}, - FixedIn: []database.FeatureVersion{ + Namespace: services.Namespace{Name: "debian:7"}, + FixedIn: []services.FeatureVersion{ { - Feature: database.Feature{Name: "openssl"}, + Feature: services.Feature{Name: "openssl"}, Version: types.NewVersionUnsafe("2.0"), }, { - Feature: database.Feature{Name: "libssl"}, + Feature: services.Feature{Name: "libssl"}, Version: types.NewVersionUnsafe("1.9-abc"), }, }, @@ -62,10 +62,10 @@ func TestFindVulnerability(t *testing.T) { } // Find a vulnerability that has no link, no severity and no FixedIn. - v2 := database.Vulnerability{ + v2 := services.Vulnerability{ Name: "CVE-NOPE", Description: "A vulnerability affecting nothing", - Namespace: database.Namespace{Name: "debian:7"}, + Namespace: services.Namespace{Name: "debian:7"}, Severity: types.Unknown, } @@ -106,93 +106,93 @@ func TestInsertVulnerability(t *testing.T) { defer datastore.Close() // Create some data. - n1 := database.Namespace{Name: "TestInsertVulnerabilityNamespace1"} - n2 := database.Namespace{Name: "TestInsertVulnerabilityNamespace2"} + n1 := services.Namespace{Name: "TestInsertVulnerabilityNamespace1"} + n2 := services.Namespace{Name: "TestInsertVulnerabilityNamespace2"} - f1 := database.FeatureVersion{ - Feature: database.Feature{ + f1 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion1", Namespace: n1, }, Version: types.NewVersionUnsafe("1.0"), } - f2 := database.FeatureVersion{ - Feature: database.Feature{ + f2 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion1", Namespace: n2, }, Version: types.NewVersionUnsafe("1.0"), } - f3 := database.FeatureVersion{ - Feature: database.Feature{ + f3 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion2", }, Version: types.MaxVersion, } - f4 := database.FeatureVersion{ - Feature: database.Feature{ + f4 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion2", }, Version: types.NewVersionUnsafe("1.4"), } - f5 := database.FeatureVersion{ - Feature: database.Feature{ + f5 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion3", }, Version: types.NewVersionUnsafe("1.5"), } - f6 := database.FeatureVersion{ - Feature: database.Feature{ + f6 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion4", }, Version: types.NewVersionUnsafe("0.1"), } - f7 := database.FeatureVersion{ - Feature: database.Feature{ + f7 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion5", }, Version: types.MaxVersion, } - f8 := database.FeatureVersion{ - Feature: database.Feature{ + f8 := services.FeatureVersion{ + Feature: services.Feature{ Name: "TestInsertVulnerabilityFeatureVersion5", }, Version: types.MinVersion, } // Insert invalid vulnerabilities. - for _, vulnerability := range []database.Vulnerability{ + for _, vulnerability := range []services.Vulnerability{ { Name: "", Namespace: n1, - FixedIn: []database.FeatureVersion{f1}, + FixedIn: []services.FeatureVersion{f1}, Severity: types.Unknown, }, { Name: "TestInsertVulnerability0", - Namespace: database.Namespace{}, - FixedIn: []database.FeatureVersion{f1}, + Namespace: services.Namespace{}, + FixedIn: []services.FeatureVersion{f1}, Severity: types.Unknown, }, { Name: "TestInsertVulnerability0-", - Namespace: database.Namespace{}, - FixedIn: []database.FeatureVersion{f1}, + Namespace: services.Namespace{}, + FixedIn: []services.FeatureVersion{f1}, }, { Name: "TestInsertVulnerability0", Namespace: n1, - FixedIn: []database.FeatureVersion{f1}, + FixedIn: []services.FeatureVersion{f1}, Severity: types.Priority(""), }, { Name: "TestInsertVulnerability0", Namespace: n1, - FixedIn: []database.FeatureVersion{f2}, + FixedIn: []services.FeatureVersion{f2}, Severity: types.Unknown, }, } { - err := datastore.InsertVulnerabilities([]database.Vulnerability{vulnerability}, true) + err := datastore.InsertVulnerabilities([]services.Vulnerability{vulnerability}, true) assert.Error(t, err) } @@ -205,16 +205,16 @@ func TestInsertVulnerability(t *testing.T) { Test: "TestInsertVulnerabilityMetadataValue1", } - v1 := database.Vulnerability{ + v1 := services.Vulnerability{ Name: "TestInsertVulnerability1", Namespace: n1, - FixedIn: []database.FeatureVersion{f1, f3, f6, f7}, + FixedIn: []services.FeatureVersion{f1, f3, f6, f7}, Severity: types.Low, Description: "TestInsertVulnerabilityDescription1", Link: "TestInsertVulnerabilityLink1", Metadata: v1meta, } - err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) + err = datastore.InsertVulnerabilities([]services.Vulnerability{v1}, true) if assert.Nil(t, err) { v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) if assert.Nil(t, err) { @@ -228,9 +228,9 @@ func TestInsertVulnerability(t *testing.T) { v1.Severity = types.High // Update f3 in f4, add fixed in f5, add fixed in f6 which already exists, removes fixed in f7 by // adding f8 which is f7 but with MinVersion. - v1.FixedIn = []database.FeatureVersion{f4, f5, f6, f8} + v1.FixedIn = []services.FeatureVersion{f4, f5, f6, f8} - err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}, true) + err = datastore.InsertVulnerabilities([]services.Vulnerability{v1}, true) if assert.Nil(t, err) { v1f, err := datastore.FindVulnerability(n1.Name, v1.Name) if assert.Nil(t, err) { @@ -250,7 +250,7 @@ func TestInsertVulnerability(t *testing.T) { } } -func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { +func equalsVuln(t *testing.T, expected, actual *services.Vulnerability) { assert.Equal(t, expected.Name, actual.Name) assert.Equal(t, expected.Namespace.Name, actual.Namespace.Name) assert.Equal(t, expected.Description, actual.Description) diff --git a/notifier/notifier.go b/notifier/notifier.go index 81e480cd..4628c5d4 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -25,7 +25,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/locks" + "github.com/coreos/clair/services/notifications" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" ) @@ -59,7 +61,7 @@ type Notifier interface { // It returns whether the notifier is enabled or not. Configure(*config.NotifierConfig) (bool, error) // Send informs the existence of the specified notification. - Send(notification database.VulnerabilityNotification) error + Send(notification services.VulnerabilityNotification) error } func init() { @@ -87,7 +89,7 @@ func RegisterNotifier(name string, n Notifier) { } // Run starts the Notifier service. -func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *utils.Stopper) { +func Run(config *config.NotifierConfig, ls locks.Service, ns notifications.Service, stopper *utils.Stopper) { defer stopper.End() // Configure registered notifiers. @@ -113,7 +115,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u for running := true; running; { // Find task. - notification := findTask(datastore, config.RenotifyInterval, whoAmI, stopper) + notification := findTask(ls, ns, config.RenotifyInterval, whoAmI, stopper) if notification == nil { // Interrupted while finding a task, Clair is stopping. break @@ -125,12 +127,12 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u success, interrupted := handleTask(*notification, stopper, config.Attempts) if success { utils.PrometheusObserveTimeMilliseconds(promNotifierLatencyMilliseconds, notification.Created) - datastore.SetNotificationNotified(notification.Name) + ns.SetNotificationNotified(notification.Name) } if interrupted { running = false } - datastore.Unlock(notification.Name, whoAmI) + ls.Unlock(notification.Name, whoAmI) done <- true }() @@ -141,7 +143,7 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u case <-done: break outer case <-time.After(refreshLockDuration): - datastore.Lock(notification.Name, whoAmI, lockDuration, true) + ls.Lock(notification.Name, whoAmI, lockDuration, true) } } } @@ -149,10 +151,10 @@ func Run(config *config.NotifierConfig, datastore database.Datastore, stopper *u log.Info("notifier service stopped") } -func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *database.VulnerabilityNotification { +func findTask(ls locks.Service, ns notifications.Service, renotifyInterval time.Duration, whoAmI string, stopper *utils.Stopper) *services.VulnerabilityNotification { for { // Find a notification to send. - notification, err := datastore.GetAvailableNotification(renotifyInterval) + notification, err := ns.GetAvailableNotification(renotifyInterval) if err != nil { // There is no notification or an error occurred. if err != cerrors.ErrNotFound { @@ -168,14 +170,14 @@ func findTask(datastore database.Datastore, renotifyInterval time.Duration, whoA } // Lock the notification. - if hasLock, _ := datastore.Lock(notification.Name, whoAmI, lockDuration, false); hasLock { + if hasLock, _ := ls.Lock(notification.Name, whoAmI, lockDuration, false); hasLock { log.Infof("found and locked a notification: %s", notification.Name) return ¬ification } } } -func handleTask(notification database.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { +func handleTask(notification services.VulnerabilityNotification, st *utils.Stopper, maxAttempts int) (bool, bool) { // Send notification. for notifierName, notifier := range notifiers { var attempts int diff --git a/notifier/notifiers/webhook.go b/notifier/notifiers/webhook.go index 28b76604..a3026fdd 100644 --- a/notifier/notifiers/webhook.go +++ b/notifier/notifiers/webhook.go @@ -30,8 +30,8 @@ import ( "gopkg.in/yaml.v2" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" "github.com/coreos/clair/notifier" + "github.com/coreos/clair/services" ) const timeout = 5 * time.Second @@ -114,7 +114,7 @@ type notificationEnvelope struct { } } -func (h *WebhookNotifier) Send(notification database.VulnerabilityNotification) error { +func (h *WebhookNotifier) Send(notification services.VulnerabilityNotification) error { // Marshal notification. jsonNotification, err := json.Marshal(notificationEnvelope{struct{ Name string }{notification.Name}}) if err != nil { diff --git a/services/keyvalue/keyvalue.go b/services/keyvalue/keyvalue.go new file mode 100644 index 00000000..4073f739 --- /dev/null +++ b/services/keyvalue/keyvalue.go @@ -0,0 +1,61 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package keyvalue defines an interface for a simple keyvalue store. +package keyvalue + +import ( + "fmt" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var keyValueDrivers = make(map[string]Driver) + +// Register makes a Service constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("keyvalue: could not register nil Driver") + } + if _, dup := keyValueDrivers[name]; dup { + panic("keyvalue: could not register duplicate Driver: " + name) + } + keyValueDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := keyValueDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("keyvalue: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + // # Key/Value + // InsertKeyValue stores or updates a simple key/value pair in the database. + InsertKeyValue(key, value string) error + // GetKeyValue retrieves a value from the database from the given key. + // It returns an empty string if there is no such key. + GetKeyValue(key string) (string, error) +} diff --git a/services/layers/layers.go b/services/layers/layers.go new file mode 100644 index 00000000..b261edbc --- /dev/null +++ b/services/layers/layers.go @@ -0,0 +1,74 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layers defines an interface for reading and writing layer metadata. +package layers + +import ( + "fmt" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var layerDrivers = make(map[string]Driver) + +// Register makes a Layer constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("layers: could not register nil Driver") + } + if _, dup := layerDrivers[name]; dup { + panic("layers: could not register duplicate Driver: " + name) + } + layerDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := layerDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("layers: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + // # Layer + // InsertLayer stores a Layer in the database. + // A Layer is uniquely identified by its Name. The Name and EngineVersion fields are mandatory. + // If a Parent is specified, it is expected that it has been retrieved using FindLayer. + // If a Layer that already exists is inserted and the EngineVersion of the given Layer is higher + // than the stored one, the stored Layer should be updated. + // The function has to be idempotent, inserting a layer that already exists shouln'd return an + // error. + InsertLayer(services.Layer) error + + // FindLayer retrieves a Layer from the database. + // withFeatures specifies whether the Features field should be filled. When withVulnerabilities is + // true, the Features field should be filled and their AffectedBy fields should contain every + // vulnerabilities that affect them. + FindLayer(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) + + // DeleteLayer deletes a Layer from the database and every layers that are based on it, + // recursively. + DeleteLayer(name string) error +} diff --git a/services/locks/locks.go b/services/locks/locks.go new file mode 100644 index 00000000..90beb379 --- /dev/null +++ b/services/locks/locks.go @@ -0,0 +1,69 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package locks defines an interface for interacting with named locks. +package locks + +import ( + "fmt" + "time" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var lockDrivers = make(map[string]Driver) + +// Register makes a Service constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("locks: could not register nil Driver") + } + if _, dup := lockDrivers[name]; dup { + panic("locks: could not register duplicate Driver: " + name) + } + lockDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := lockDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("locks: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + // # Lock + // Lock creates or renew a Lock in the database with the given name, owner and duration. + // After the specified duration, the Lock expires by itself if it hasn't been unlocked, and thus, + // let other users create a Lock with the same name. However, the owner can renew its Lock by + // setting renew to true. Lock should not block, it should instead returns whether the Lock has + // been successfully acquired/renewed. If it's the case, the expiration time of that Lock is + // returned as well. + Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time) + // Unlock releases an existing Lock. + Unlock(name, owner string) + // FindLock returns the owner of a Lock specified by the name, and its experation time if it + // exists. + FindLock(name string) (string, time.Time, error) +} diff --git a/database/models.go b/services/models.go similarity index 87% rename from database/models.go rename to services/models.go index a44291b8..116df737 100644 --- a/database/models.go +++ b/services/models.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package database +package services import ( "database/sql/driver" @@ -22,6 +22,8 @@ import ( "github.com/coreos/clair/utils/types" ) +// TODO(mattmoor): Clean up the dependencies of these types so that they can be decomposed into their respective service layers. This will require eliminating some of the overloading via optional fields that is done today (e.g. AddedBy "only makes sense when ..." should probably be a FeatureVersion wrapped in a new type used in the context of an image). + // ID is only meant to be used by database implementations and should never be used for anything else. type Model struct { ID int diff --git a/database/namespace_mapping.go b/services/namespaces/mapping.go similarity index 98% rename from database/namespace_mapping.go rename to services/namespaces/mapping.go index 97ef21d9..6b1c4027 100644 --- a/database/namespace_mapping.go +++ b/services/namespaces/mapping.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package database +package namespaces // DebianReleasesMapping translates Debian code names and class names to version numbers var DebianReleasesMapping = map[string]string{ diff --git a/services/namespaces/namespaces.go b/services/namespaces/namespaces.go new file mode 100644 index 00000000..e7baa26c --- /dev/null +++ b/services/namespaces/namespaces.go @@ -0,0 +1,59 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package namespaces defines an interface for listing the available namespaces, and a few maps between equivalent namespaces. +package namespaces + +import ( + "fmt" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var namespacesDrivers = make(map[string]Driver) + +// Register makes a Service constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("namespaces: could not register nil Driver") + } + if _, dup := namespacesDrivers[name]; dup { + panic("namespaces: could not register duplicate Driver: " + name) + } + namespacesDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := namespacesDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("namespaces: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + + // # Namespace + // ListNamespaces returns the entire list of known Namespaces. + ListNamespaces() ([]services.Namespace, error) +} diff --git a/services/notifications/notifications.go b/services/notifications/notifications.go new file mode 100644 index 00000000..84d1f2b9 --- /dev/null +++ b/services/notifications/notifications.go @@ -0,0 +1,79 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package notifications defines an interface for interacting with notification state. +package notifications + +import ( + "fmt" + "time" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var notificationDrivers = make(map[string]Driver) + +// Register makes a Service constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("notifications: could not register nil Driver") + } + if _, dup := notificationDrivers[name]; dup { + panic("notifications: could not register duplicate Driver: " + name) + } + notificationDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := notificationDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("notifications: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + // # Notification + // GetAvailableNotification returns the Name, Created, Notified and Deleted fields of a + // Notification that should be handled. The renotify interval defines how much time after being + // marked as Notified by SetNotificationNotified, a Notification that hasn't been deleted should + // be returned again by this function. A Notification for which there is a valid Lock with the + // same Name should not be returned. + GetAvailableNotification(renotifyInterval time.Duration) (services.VulnerabilityNotification, error) + + // GetNotification returns a Notification, including its OldVulnerability and NewVulnerability + // fields. On these Vulnerabilities, LayersIntroducingVulnerability should be filled with + // every Layer that introduces the Vulnerability (i.e. adds at least one affected FeatureVersion). + // The Limit and page parameters are used to paginate LayersIntroducingVulnerability. The first + // given page should be VulnerabilityNotificationFirstPage. The function will then return the next + // availage page. If there is no more page, NoVulnerabilityNotificationPage has to be returned. + GetNotification(name string, limit int, page services.VulnerabilityNotificationPageNumber) (services.VulnerabilityNotification, services.VulnerabilityNotificationPageNumber, error) + + // SetNotificationNotified marks a Notification as notified and thus, makes it unavailable for + // GetAvailableNotification, until the renotify duration is elapsed. + SetNotificationNotified(name string) error + + // DeleteNotification marks a Notification as deleted, and thus, makes it unavailable for + // GetAvailableNotification. + DeleteNotification(name string) error +} diff --git a/services/services.go b/services/services.go new file mode 100644 index 00000000..bcbd926f --- /dev/null +++ b/services/services.go @@ -0,0 +1,39 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package services defines a set of service interfaces needed by Clair internally. +package services + +import ( + "errors" +) + +var ( + // ErrBackendException is an error that occurs when the service backend does + // not work properly (ie. unreachable). + ErrBackendException = errors.New("services: an error occured when querying the backend") + + // ErrInconsistent is an error that occurs when a service consistency check + // fails (ie. when an entity which is supposed to be unique is detected twice) + ErrInconsistent = errors.New("services: inconsistent state") +) + +type Base interface { + // # Miscellaneous + // Ping returns the health status of the service. + Ping() bool + + // Close closes the connection to the service and free any allocated resources. + Close() +} diff --git a/services/vulnerabilities/vulnerabilities.go b/services/vulnerabilities/vulnerabilities.go new file mode 100644 index 00000000..550aed87 --- /dev/null +++ b/services/vulnerabilities/vulnerabilities.go @@ -0,0 +1,89 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package vulnerabilities defines an interface for listing, reading and writing vulnerability information +package vulnerabilities + +import ( + "fmt" + + "github.com/coreos/clair/config" + "github.com/coreos/clair/services" +) + +type Driver func(cfg config.RegistrableComponentConfig) (Service, error) + +var vulnzDrivers = make(map[string]Driver) + +// Register makes a Vulnerability constructor available by the provided name. +// +// If this function is called twice with the same name or if the Constructor is +// nil, it panics. +func Register(name string, driver Driver) { + if driver == nil { + panic("vulnerabilities: could not register nil Driver") + } + if _, dup := vulnzDrivers[name]; dup { + panic("vulnerabilities: could not register duplicate Driver: " + name) + } + vulnzDrivers[name] = driver +} + +// Open opens a Datastore specified by a configuration. +func Open(cfg config.RegistrableComponentConfig) (ls Service, err error) { + driver, ok := vulnzDrivers[cfg.Type] + if !ok { + err = fmt.Errorf("vulnerabilities: unknown Driver %q (forgotten configuration or import?)", cfg.Type) + return + } + return driver(cfg) +} + +type Service interface { + services.Base + + // # Vulnerability + // ListVulnerabilities returns the list of vulnerabilies of a certain Namespace. + // The Limit and page parameters are used to paginate the return list. + // The first given page should be 0. The function will then return the next available page. + // If there is no more page, -1 has to be returned. + ListVulnerabilities(namespaceName string, limit int, page int) ([]services.Vulnerability, int, error) + // InsertVulnerabilities stores the given Vulnerabilities in the database, updating them if + // necessary. A vulnerability is uniquely identified by its Namespace and its Name. + // The FixedIn field may only contain a partial list of Features that are affected by the + // Vulnerability, along with the version in which the vulnerability is fixed. It is the + // responsibility of the implementation to update the list properly. A version equals to + // types.MinVersion means that the given Feature is not being affected by the Vulnerability at + // all and thus, should be removed from the list. It is important that Features should be unique + // in the FixedIn list. For example, it doesn't make sense to have two `openssl` Feature listed as + // a Vulnerability can only be fixed in one Version. This is true because Vulnerabilities and + // Features are Namespaced (i.e. specific to one operating system). + // Each vulnerability insertion or update has to create a Notification that will contain the + // old and the updated Vulnerability, unless createNotification equals to true. + InsertVulnerabilities(vulnerabilities []services.Vulnerability, createNotification bool) error + // FindVulnerability retrieves a Vulnerability from the database, including the FixedIn list. + FindVulnerability(namespaceName, name string) (services.Vulnerability, error) + // DeleteVulnerability removes a Vulnerability from the database. + // It has to create a Notification that will contain the old Vulnerability. + DeleteVulnerability(namespaceName, name string) error + // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions of existing ones to + // the specified Vulnerability in the database. + // It has has to create a Notification that will contain the old and the updated Vulnerability. + InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []services.FeatureVersion) error + // DeleteVulnerabilityFix removes a FixedIn Feature from the specified Vulnerability in the + // database. It can be used to store the fact that a Vulnerability no longer affects the given + // Feature in any Version. + // It has has to create a Notification that will contain the old and the updated Vulnerability. + DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error +} diff --git a/updater/fetchers.go b/updater/fetchers.go index 609ecd40..d0f1bf18 100644 --- a/updater/fetchers.go +++ b/updater/fetchers.go @@ -14,14 +14,17 @@ package updater -import "github.com/coreos/clair/database" +import ( + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" +) var fetchers = make(map[string]Fetcher) // Fetcher represents anything that can fetch vulnerabilities. type Fetcher interface { // FetchUpdate gets vulnerability updates. - FetchUpdate(database.Datastore) (FetcherResponse, error) + FetchUpdate(keyvalue.Service) (FetcherResponse, error) // Clean deletes any allocated resources. // It is invoked when Clair stops. @@ -33,7 +36,7 @@ type FetcherResponse struct { FlagName string FlagValue string Notes []string - Vulnerabilities []database.Vulnerability + Vulnerabilities []services.Vulnerability } // RegisterFetcher makes a Fetcher available by the provided name. diff --git a/updater/fetchers/debian/debian.go b/updater/fetchers/debian/debian.go index e21bd3f2..a84da4d6 100644 --- a/updater/fetchers/debian/debian.go +++ b/updater/fetchers/debian/debian.go @@ -23,7 +23,9 @@ import ( "net/http" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/namespaces" "github.com/coreos/clair/updater" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -60,7 +62,7 @@ func init() { } // FetchUpdate fetches vulnerability updates from the Debian Security Tracker. -func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (fetcher *DebianFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) { log.Info("fetching Debian vulnerabilities") // Download JSON. @@ -71,7 +73,7 @@ func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp up } // Get the SHA-1 of the latest update's JSON data - latestHash, err := datastore.GetKeyValue(updaterFlag) + latestHash, err := kvstore.GetKeyValue(updaterFlag) if err != nil { return resp, err } @@ -130,15 +132,15 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F return resp, nil } -func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, unknownReleases map[string]struct{}) { - mvulnerabilities := make(map[string]*database.Vulnerability) +func parseDebianJSON(data *jsonData) (vulnerabilities []services.Vulnerability, unknownReleases map[string]struct{}) { + mvulnerabilities := make(map[string]*services.Vulnerability) unknownReleases = make(map[string]struct{}) for pkgName, pkgNode := range *data { for vulnName, vulnNode := range pkgNode { for releaseName, releaseNode := range vulnNode.Releases { // Attempt to detect the release number. - if _, isReleaseKnown := database.DebianReleasesMapping[releaseName]; !isReleaseKnown { + if _, isReleaseKnown := namespaces.DebianReleasesMapping[releaseName]; !isReleaseKnown { unknownReleases[releaseName] = struct{}{} continue } @@ -151,7 +153,7 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, // Get or create the vulnerability. vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName] if !vulnerabilityAlreadyExists { - vulnerability = &database.Vulnerability{ + vulnerability = &services.Vulnerability{ Name: vulnName, Link: strings.Join([]string{cveURLPrefix, "/", vulnName}, ""), Severity: types.Unknown, @@ -188,11 +190,11 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, } // Create and add the feature version. - pkg := database.FeatureVersion{ - Feature: database.Feature{ + pkg := services.FeatureVersion{ + Feature: services.Feature{ Name: pkgName, - Namespace: database.Namespace{ - Name: "debian:" + database.DebianReleasesMapping[releaseName], + Namespace: services.Namespace{ + Name: "debian:" + namespaces.DebianReleasesMapping[releaseName], }, }, Version: version, diff --git a/updater/fetchers/debian/debian_test.go b/updater/fetchers/debian/debian_test.go index e092909d..1411b698 100644 --- a/updater/fetchers/debian/debian_test.go +++ b/updater/fetchers/debian/debian_test.go @@ -20,7 +20,7 @@ import ( "runtime" "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -38,17 +38,17 @@ func TestDebianParser(t *testing.T) { assert.Equal(t, types.Low, vulnerability.Severity) assert.Equal(t, "This vulnerability is not very dangerous.", vulnerability.Description) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:8"}, Name: "aptdaemon", }, Version: types.MaxVersion, }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:unstable"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:unstable"}, Name: "aptdaemon", }, @@ -64,24 +64,24 @@ func TestDebianParser(t *testing.T) { assert.Equal(t, types.High, vulnerability.Severity) assert.Equal(t, "But this one is very dangerous.", vulnerability.Description) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:8"}, Name: "aptdaemon", }, Version: types.NewVersionUnsafe("0.7.0"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:unstable"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:unstable"}, Name: "aptdaemon", }, Version: types.NewVersionUnsafe("0.7.0"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:8"}, Name: "asterisk", }, Version: types.NewVersionUnsafe("0.5.56"), @@ -96,10 +96,10 @@ func TestDebianParser(t *testing.T) { assert.Equal(t, types.Negligible, vulnerability.Severity) assert.Equal(t, "Un-affected packages.", vulnerability.Description) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "debian:8"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "debian:8"}, Name: "asterisk", }, Version: types.MinVersion, diff --git a/updater/fetchers/rhel/rhel.go b/updater/fetchers/rhel/rhel.go index de4072d1..5981dff3 100644 --- a/updater/fetchers/rhel/rhel.go +++ b/updater/fetchers/rhel/rhel.go @@ -23,7 +23,8 @@ import ( "strconv" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" "github.com/coreos/clair/updater" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -88,11 +89,11 @@ func init() { } // FetchUpdate gets vulnerability updates from the Red Hat OVAL definitions. -func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (f *RHELFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) { log.Info("fetching Red Hat vulnerabilities") // Get the first RHSA we have to manage. - flagValue, err := datastore.GetKeyValue(updaterFlag) + flagValue, err := kvstore.GetKeyValue(updaterFlag) if err != nil { return resp, err } @@ -153,7 +154,7 @@ func (f *RHELFetcher) FetchUpdate(datastore database.Datastore) (resp updater.Fe return resp, nil } -func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, err error) { +func parseRHSA(ovalReader io.Reader) (vulnerabilities []services.Vulnerability, err error) { // Decode the XML. var ov oval err = xml.NewDecoder(ovalReader).Decode(&ov) @@ -168,7 +169,7 @@ func parseRHSA(ovalReader io.Reader) (vulnerabilities []database.Vulnerability, for _, definition := range ov.Definitions { pkgs := toFeatureVersions(definition.Criteria) if len(pkgs) > 0 { - vulnerability := database.Vulnerability{ + vulnerability := services.Vulnerability{ Name: name(definition), Link: link(definition), Severity: priority(definition), @@ -259,15 +260,15 @@ func getPossibilities(node criteria) [][]criterion { return possibilities } -func toFeatureVersions(criteria criteria) []database.FeatureVersion { +func toFeatureVersions(criteria criteria) []services.FeatureVersion { // There are duplicates in Red Hat .xml files. // This map is for deduplication. - featureVersionParameters := make(map[string]database.FeatureVersion) + featureVersionParameters := make(map[string]services.FeatureVersion) possibilities := getPossibilities(criteria) for _, criterions := range possibilities { var ( - featureVersion database.FeatureVersion + featureVersion services.FeatureVersion osVersion int err error ) @@ -304,7 +305,7 @@ func toFeatureVersions(criteria criteria) []database.FeatureVersion { } // Convert the map to slice. - var featureVersionParametersArray []database.FeatureVersion + var featureVersionParametersArray []services.FeatureVersion for _, fv := range featureVersionParameters { featureVersionParametersArray = append(featureVersionParametersArray, fv) } diff --git a/updater/fetchers/rhel/rhel_test.go b/updater/fetchers/rhel/rhel_test.go index 0e9ddbd0..9d14cbf0 100644 --- a/updater/fetchers/rhel/rhel_test.go +++ b/updater/fetchers/rhel/rhel_test.go @@ -20,7 +20,7 @@ import ( "runtime" "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -38,24 +38,24 @@ func TestRHELParser(t *testing.T) { assert.Equal(t, types.Medium, vulnerabilities[0].Severity) assert.Equal(t, `Xerces-C is a validating XML parser written in a portable subset of C++. A flaw was found in the way the Xerces-C XML parser processed certain XML documents. A remote attacker could provide specially crafted XML input that, when parsed by an application using Xerces-C, would cause that application to crash.`, vulnerabilities[0].Description) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "centos:7"}, Name: "xerces-c", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "centos:7"}, Name: "xerces-c-devel", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "centos:7"}, Name: "xerces-c-doc", }, Version: types.NewVersionUnsafe("3.1.1-7.el7_1"), @@ -76,17 +76,17 @@ func TestRHELParser(t *testing.T) { assert.Equal(t, types.Critical, vulnerabilities[0].Severity) assert.Equal(t, `Mozilla Firefox is an open source web browser. XULRunner provides the XUL Runtime environment for Mozilla Firefox. Several flaws were found in the processing of malformed web content. A web page containing malicious content could cause Firefox to crash or, potentially, execute arbitrary code with the privileges of the user running Firefox.`, vulnerabilities[0].Description) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:6"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "centos:6"}, Name: "firefox", }, Version: types.NewVersionUnsafe("38.1.0-1.el6_6"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "centos:7"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "centos:7"}, Name: "firefox", }, Version: types.NewVersionUnsafe("38.1.0-1.el7_1"), diff --git a/updater/fetchers/ubuntu/ubuntu.go b/updater/fetchers/ubuntu/ubuntu.go index 2bcc2b09..8fffcd6e 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/updater/fetchers/ubuntu/ubuntu.go @@ -26,7 +26,9 @@ import ( "strconv" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/namespaces" "github.com/coreos/clair/updater" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" @@ -89,7 +91,7 @@ func init() { } // FetchUpdate gets vulnerability updates from the Ubuntu CVE Tracker. -func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) { +func (fetcher *UbuntuFetcher) FetchUpdate(kvstore keyvalue.Service) (resp updater.FetcherResponse, err error) { log.Info("fetching Ubuntu vulnerabilities") // Check to see if the repository does not already exist. @@ -123,7 +125,7 @@ func (fetcher *UbuntuFetcher) FetchUpdate(datastore database.Datastore) (resp up } // Get the latest revision number we successfully applied in the database. - dbRevisionNumber, err := datastore.GetKeyValue("ubuntuUpdater") + dbRevisionNumber, err := kvstore.GetKeyValue("ubuntuUpdater") if err != nil { return resp, err } @@ -281,7 +283,7 @@ func getRevisionNumber(pathToRepo string) (int, error) { return revno, nil } -func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability, unknownReleases map[string]struct{}, err error) { +func parseUbuntuCVE(fileContent io.Reader) (vulnerability services.Vulnerability, unknownReleases map[string]struct{}, err error) { unknownReleases = make(map[string]struct{}) readingDescription := false scanner := bufio.NewScanner(fileContent) @@ -350,7 +352,7 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability if _, isReleaseIgnored := ubuntuIgnoredReleases[md["release"]]; isReleaseIgnored { continue } - if _, isReleaseKnown := database.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown { + if _, isReleaseKnown := namespaces.UbuntuReleasesMapping[md["release"]]; !isReleaseKnown { unknownReleases[md["release"]] = struct{}{} continue } @@ -374,9 +376,9 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability } // Create and add the new package. - featureVersion := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:" + database.UbuntuReleasesMapping[md["release"]]}, + featureVersion := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "ubuntu:" + namespaces.UbuntuReleasesMapping[md["release"]]}, Name: md["package"], }, Version: version, diff --git a/updater/fetchers/ubuntu/ubuntu_test.go b/updater/fetchers/ubuntu/ubuntu_test.go index d76d457e..dfa34003 100644 --- a/updater/fetchers/ubuntu/ubuntu_test.go +++ b/updater/fetchers/ubuntu/ubuntu_test.go @@ -20,7 +20,7 @@ import ( "runtime" "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) @@ -42,24 +42,24 @@ func TestUbuntuParser(t *testing.T) { _, hasUnkownRelease := unknownReleases["unknown"] assert.True(t, hasUnkownRelease) - expectedFeatureVersions := []database.FeatureVersion{ + expectedFeatureVersions := []services.FeatureVersion{ { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:14.04"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "ubuntu:14.04"}, Name: "libmspack", }, Version: types.MaxVersion, }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:15.04"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "ubuntu:15.04"}, Name: "libmspack", }, Version: types.NewVersionUnsafe("0.4-3"), }, { - Feature: database.Feature{ - Namespace: database.Namespace{Name: "ubuntu:15.10"}, + Feature: services.Feature{ + Namespace: services.Namespace{Name: "ubuntu:15.10"}, Name: "libmspack-anotherpkg", }, Version: types.NewVersionUnsafe("0.1"), diff --git a/updater/metadata_fetchers.go b/updater/metadata_fetchers.go index 72e764af..17386416 100644 --- a/updater/metadata_fetchers.go +++ b/updater/metadata_fetchers.go @@ -17,22 +17,23 @@ package updater import ( "sync" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/vulnerabilities" ) var metadataFetchers = make(map[string]MetadataFetcher) type VulnerabilityWithLock struct { - *database.Vulnerability + *services.Vulnerability Lock sync.Mutex } // MetadataFetcher type MetadataFetcher interface { // Load runs right before the Updater calls AddMetadata for each vulnerabilities. - Load(database.Datastore) error + Load(vulnerabilities.Service) error - // AddMetadata adds metadata to the given database.Vulnerability. + // AddMetadata adds metadata to the given services.Vulnerability. // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. AddMetadata(*VulnerabilityWithLock) error diff --git a/updater/metadata_fetchers/nvd/nvd.go b/updater/metadata_fetchers/nvd/nvd.go index 4f57bb62..618a2de8 100644 --- a/updater/metadata_fetchers/nvd/nvd.go +++ b/updater/metadata_fetchers/nvd/nvd.go @@ -15,7 +15,7 @@ import ( "sync" "time" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services/vulnerabilities" "github.com/coreos/clair/updater" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/pkg/capnslog" @@ -53,7 +53,7 @@ func init() { updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{}) } -func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { +func (fetcher *NVDMetadataFetcher) Load(vulnstore vulnerabilities.Service) error { fetcher.lock.Lock() defer fetcher.lock.Unlock() diff --git a/updater/updater.go b/updater/updater.go index bfb8680b..13a3c765 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -23,7 +23,10 @@ import ( "time" "github.com/coreos/clair/config" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/keyvalue" + "github.com/coreos/clair/services/locks" + "github.com/coreos/clair/services/vulnerabilities" "github.com/coreos/clair/utils" "github.com/coreos/pkg/capnslog" "github.com/pborman/uuid" @@ -65,7 +68,7 @@ func init() { } // Run updates the vulnerability database at regular intervals. -func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.Stopper) { +func Run(config *config.UpdaterConfig, locksvc locks.Service, kvstore keyvalue.Service, vulnstore vulnerabilities.Service, st *utils.Stopper) { defer st.End() // Do not run the updater if there is no config or if the interval is 0. @@ -83,7 +86,7 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S // Determine if this is the first update and define the next update time. // The next update time is (last update time + interval) or now if this is the first update. nextUpdate := time.Now().UTC() - lastUpdate, firstUpdate, err := getLastUpdate(datastore) + lastUpdate, firstUpdate, err := getLastUpdate(kvstore) if err != nil { log.Errorf("an error occured while getting the last update time") nextUpdate = nextUpdate.Add(config.Interval) @@ -95,12 +98,12 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S if nextUpdate.Before(time.Now().UTC()) { // Attempt to get a lock on the the update. log.Debug("attempting to obtain update lock") - hasLock, hasLockUntil := datastore.Lock(lockName, whoAmI, lockDuration, false) + hasLock, hasLockUntil := locksvc.Lock(lockName, whoAmI, lockDuration, false) if hasLock { // Launch update in a new go routine. doneC := make(chan bool, 1) go func() { - Update(datastore, firstUpdate) + Update(kvstore, vulnstore, firstUpdate) doneC <- true }() @@ -110,21 +113,21 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S done = true case <-time.After(refreshLockDuration): // Refresh the lock until the update is done. - datastore.Lock(lockName, whoAmI, lockDuration, true) + locksvc.Lock(lockName, whoAmI, lockDuration, true) case <-st.Chan(): stop = true } } // Unlock the update. - datastore.Unlock(lockName, whoAmI) + locksvc.Unlock(lockName, whoAmI) if stop { break } continue } else { - lockOwner, lockExpiration, err := datastore.FindLock(lockName) + lockOwner, lockExpiration, err := locksvc.FindLock(lockName) if err != nil { log.Debug("update lock is already taken") nextUpdate = hasLockUntil @@ -159,17 +162,17 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S // Update fetches all the vulnerabilities from the registered fetchers, upserts // them into the database and then sends notifications. -func Update(datastore database.Datastore, firstUpdate bool) { +func Update(kvstore keyvalue.Service, vulnstore vulnerabilities.Service, firstUpdate bool) { defer setUpdaterDuration(time.Now()) log.Info("updating vulnerabilities") // Fetch updates. - status, vulnerabilities, flags, notes := fetch(datastore) + status, vulnerabilities, flags, notes := fetch(kvstore, vulnstore) // Insert vulnerabilities. log.Tracef("inserting %d vulnerabilities for update", len(vulnerabilities)) - err := datastore.InsertVulnerabilities(vulnerabilities, !firstUpdate) + err := vulnstore.InsertVulnerabilities(vulnerabilities, !firstUpdate) if err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when inserting vulnerabilities for update: %s", err) @@ -179,7 +182,7 @@ func Update(datastore database.Datastore, firstUpdate bool) { // Update flags. for flagName, flagValue := range flags { - datastore.InsertKeyValue(flagName, flagValue) + kvstore.InsertKeyValue(flagName, flagValue) } // Log notes. @@ -190,7 +193,7 @@ func Update(datastore database.Datastore, firstUpdate bool) { // Update last successful update if every fetchers worked properly. if status { - datastore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) + kvstore.InsertKeyValue(flagName, strconv.FormatInt(time.Now().UTC().Unix(), 10)) } log.Info("update finished") @@ -201,8 +204,8 @@ func setUpdaterDuration(start time.Time) { } // fetch get data from the registered fetchers, in parallel. -func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[string]string, []string) { - var vulnerabilities []database.Vulnerability +func fetch(kvstore keyvalue.Service, vulnstore vulnerabilities.Service) (bool, []services.Vulnerability, map[string]string, []string) { + var vulnerabilities []services.Vulnerability var notes []string status := true flags := make(map[string]string) @@ -212,7 +215,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st var responseC = make(chan *FetcherResponse, 0) for n, f := range fetchers { go func(name string, fetcher Fetcher) { - response, err := fetcher.FetchUpdate(datastore) + response, err := fetcher.FetchUpdate(kvstore) if err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when fetching update '%s': %s.", name, err) @@ -238,11 +241,11 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st } close(responseC) - return status, addMetadata(datastore, vulnerabilities), flags, notes + return status, addMetadata(vulnstore, vulnerabilities), flags, notes } // Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel. -func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability { +func addMetadata(vulnstore vulnerabilities.Service, vulnerabilities []services.Vulnerability) []services.Vulnerability { if len(metadataFetchers) == 0 { return vulnerabilities } @@ -266,7 +269,7 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner defer wg.Done() // Load the metadata fetcher. - if err := metadataFetcher.Load(datastore); err != nil { + if err := metadataFetcher.Load(vulnstore); err != nil { promUpdaterErrorsTotal.Inc() log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err) return @@ -286,8 +289,8 @@ func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulner return vulnerabilities } -func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) { - lastUpdateTSS, err := datastore.GetKeyValue(flagName) +func getLastUpdate(kvstore keyvalue.Service) (time.Time, bool, error) { + lastUpdateTSS, err := kvstore.GetKeyValue(flagName) if err != nil { return time.Time{}, false, err } @@ -311,12 +314,12 @@ func getLastUpdate(datastore database.Datastore) (time.Time, bool, error) { // // It helps simplifying the fetchers that share the same metadata about a Vulnerability regardless // of their actual namespace (ie. same vulnerability information for every version of a distro). -func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []database.Vulnerability { - vulnerabilitiesMap := make(map[string]*database.Vulnerability) +func doVulnerabilitiesNamespacing(vulnerabilities []services.Vulnerability) []services.Vulnerability { + vulnerabilitiesMap := make(map[string]*services.Vulnerability) for _, v := range vulnerabilities { featureVersions := v.FixedIn - v.FixedIn = []database.FeatureVersion{} + v.FixedIn = []services.FeatureVersion{} for _, fv := range featureVersions { index := fv.Feature.Namespace.Name + ":" + v.Name @@ -324,7 +327,7 @@ func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []da if vulnerability, ok := vulnerabilitiesMap[index]; !ok { newVulnerability := v newVulnerability.Namespace.Name = fv.Feature.Namespace.Name - newVulnerability.FixedIn = []database.FeatureVersion{fv} + newVulnerability.FixedIn = []services.FeatureVersion{fv} vulnerabilitiesMap[index] = &newVulnerability } else { @@ -334,7 +337,7 @@ func doVulnerabilitiesNamespacing(vulnerabilities []database.Vulnerability) []da } // Convert map into a slice. - var response []database.Vulnerability + var response []services.Vulnerability for _, vulnerability := range vulnerabilitiesMap { response = append(response, *vulnerability) } diff --git a/updater/updater_test.go b/updater/updater_test.go index 5b6df0f0..c67ee8fa 100644 --- a/updater/updater_test.go +++ b/updater/updater_test.go @@ -4,42 +4,42 @@ import ( "fmt" "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/stretchr/testify/assert" ) func TestDoVulnerabilitiesNamespacing(t *testing.T) { - fv1 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "Namespace1"}, + fv1 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "Namespace1"}, Name: "Feature1", }, Version: types.NewVersionUnsafe("0.1"), } - fv2 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "Namespace2"}, + fv2 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "Namespace2"}, Name: "Feature1", }, Version: types.NewVersionUnsafe("0.2"), } - fv3 := database.FeatureVersion{ - Feature: database.Feature{ - Namespace: database.Namespace{Name: "Namespace2"}, + fv3 := services.FeatureVersion{ + Feature: services.Feature{ + Namespace: services.Namespace{Name: "Namespace2"}, Name: "Feature2", }, Version: types.NewVersionUnsafe("0.3"), } - vulnerability := database.Vulnerability{ + vulnerability := services.Vulnerability{ Name: "DoVulnerabilityNamespacing", - FixedIn: []database.FeatureVersion{fv1, fv2, fv3}, + FixedIn: []services.FeatureVersion{fv1, fv2, fv3}, } - vulnerabilities := doVulnerabilitiesNamespacing([]database.Vulnerability{vulnerability}) + vulnerabilities := doVulnerabilitiesNamespacing([]services.Vulnerability{vulnerability}) for _, vulnerability := range vulnerabilities { switch vulnerability.Namespace.Name { case fv1.Feature.Namespace.Name: diff --git a/utils/http/http.go b/utils/http/http.go index 1aac81f3..f8147eaa 100644 --- a/utils/http/http.go +++ b/utils/http/http.go @@ -20,7 +20,7 @@ import ( "io" "net/http" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker" @@ -54,7 +54,7 @@ func WriteHTTPError(w http.ResponseWriter, httpStatus int, err error) { switch err { case cerrors.ErrNotFound: httpStatus = http.StatusNotFound - case database.ErrBackendException: + case services.ErrBackendException: httpStatus = http.StatusServiceUnavailable case worker.ErrParentUnknown, worker.ErrUnsupported, utils.ErrCouldNotExtract, utils.ErrExtractedFileTooBig: httpStatus = http.StatusBadRequest diff --git a/worker/detectors/feature/dpkg/dpkg.go b/worker/detectors/feature/dpkg/dpkg.go index 6154d2ce..93607791 100644 --- a/worker/detectors/feature/dpkg/dpkg.go +++ b/worker/detectors/feature/dpkg/dpkg.go @@ -19,7 +19,7 @@ import ( "regexp" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors" "github.com/coreos/pkg/capnslog" @@ -40,16 +40,16 @@ func init() { } // Detect detects packages using var/lib/dpkg/status from the input data -func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { +func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]services.FeatureVersion, error) { f, hasFile := data["var/lib/dpkg/status"] if !hasFile { - return []database.FeatureVersion{}, nil + return []services.FeatureVersion{}, nil } // Create a map to store packages and ensure their uniqueness - packagesMap := make(map[string]database.FeatureVersion) + packagesMap := make(map[string]services.FeatureVersion) - var pkg database.FeatureVersion + var pkg services.FeatureVersion var err error scanner := bufio.NewScanner(strings.NewReader(string(f))) for scanner.Scan() { @@ -100,7 +100,7 @@ func (detector *DpkgFeaturesDetector) Detect(data map[string][]byte) ([]database } // Convert the map to a slice - packages := make([]database.FeatureVersion, 0, len(packagesMap)) + packages := make([]services.FeatureVersion, 0, len(packagesMap)) for _, pkg := range packagesMap { packages = append(packages, pkg) } diff --git a/worker/detectors/feature/dpkg/dpkg_test.go b/worker/detectors/feature/dpkg/dpkg_test.go index 94fd34d4..06b008fc 100644 --- a/worker/detectors/feature/dpkg/dpkg_test.go +++ b/worker/detectors/feature/dpkg/dpkg_test.go @@ -17,7 +17,7 @@ package dpkg import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/feature" ) @@ -25,18 +25,18 @@ import ( var dpkgPackagesTests = []feature.FeatureVersionTest{ // Test an Ubuntu dpkg status file { - FeatureVersions: []database.FeatureVersion{ + FeatureVersions: []services.FeatureVersion{ // Two packages from this source are installed, it should only appear one time { - Feature: database.Feature{Name: "pam"}, + Feature: services.Feature{Name: "pam"}, Version: types.NewVersionUnsafe("1.1.8-3.1ubuntu3"), }, { - Feature: database.Feature{Name: "makedev"}, // The source name and the package name are equals + Feature: services.Feature{Name: "makedev"}, // The source name and the package name are equals Version: types.NewVersionUnsafe("2.3.1-93ubuntu1"), // The version comes from the "Version:" line }, { - Feature: database.Feature{Name: "gcc-5"}, + Feature: services.Feature{Name: "gcc-5"}, Version: types.NewVersionUnsafe("5.1.1-12ubuntu1"), // The version comes from the "Source:" line }, }, diff --git a/worker/detectors/feature/rpm/rpm.go b/worker/detectors/feature/rpm/rpm.go index b5012639..ff46b5cd 100644 --- a/worker/detectors/feature/rpm/rpm.go +++ b/worker/detectors/feature/rpm/rpm.go @@ -20,7 +20,7 @@ import ( "os" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -39,27 +39,27 @@ func init() { } // Detect detects packages using var/lib/rpm/Packages from the input data -func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database.FeatureVersion, error) { +func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]services.FeatureVersion, error) { f, hasFile := data["var/lib/rpm/Packages"] if !hasFile { - return []database.FeatureVersion{}, nil + return []services.FeatureVersion{}, nil } // Create a map to store packages and ensure their uniqueness - packagesMap := make(map[string]database.FeatureVersion) + packagesMap := make(map[string]services.FeatureVersion) // Write the required "Packages" file to disk tmpDir, err := ioutil.TempDir(os.TempDir(), "rpm") defer os.RemoveAll(tmpDir) if err != nil { log.Errorf("could not create temporary folder for RPM detection: %s", err) - return []database.FeatureVersion{}, cerrors.ErrFilesystem + return []services.FeatureVersion{}, cerrors.ErrFilesystem } err = ioutil.WriteFile(tmpDir+"/Packages", f, 0700) if err != nil { log.Errorf("could not create temporary file for RPM detection: %s", err) - return []database.FeatureVersion{}, cerrors.ErrFilesystem + return []services.FeatureVersion{}, cerrors.ErrFilesystem } // Query RPM @@ -70,7 +70,7 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. log.Errorf("could not query RPM: %s. output: %s", err, string(out)) // Do not bubble up because we probably won't be able to fix it, // the database must be corrupted - return []database.FeatureVersion{}, nil + return []services.FeatureVersion{}, nil } scanner := bufio.NewScanner(strings.NewReader(string(out))) @@ -95,8 +95,8 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. } // Add package - pkg := database.FeatureVersion{ - Feature: database.Feature{ + pkg := services.FeatureVersion{ + Feature: services.Feature{ Name: line[0], }, Version: version, @@ -105,7 +105,7 @@ func (detector *RpmFeaturesDetector) Detect(data map[string][]byte) ([]database. } // Convert the map to a slice - packages := make([]database.FeatureVersion, 0, len(packagesMap)) + packages := make([]services.FeatureVersion, 0, len(packagesMap)) for _, pkg := range packagesMap { packages = append(packages, pkg) } diff --git a/worker/detectors/feature/rpm/rpm_test.go b/worker/detectors/feature/rpm/rpm_test.go index adb5adb4..856b9fa8 100644 --- a/worker/detectors/feature/rpm/rpm_test.go +++ b/worker/detectors/feature/rpm/rpm_test.go @@ -17,7 +17,7 @@ package rpm import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/utils/types" "github.com/coreos/clair/worker/detectors/feature" ) @@ -26,15 +26,15 @@ var rpmPackagesTests = []feature.FeatureVersionTest{ // Test a CentOS 7 RPM database // Memo: Use the following command on a RPM-based system to shrink a database: rpm -qa --qf "%{NAME}\n" |tail -n +3| xargs rpm -e --justdb { - FeatureVersions: []database.FeatureVersion{ + FeatureVersions: []services.FeatureVersion{ // Two packages from this source are installed, it should only appear once { - Feature: database.Feature{Name: "centos-release"}, + Feature: services.Feature{Name: "centos-release"}, Version: types.NewVersionUnsafe("7-1.1503.el7.centos.2.8"), }, // Two packages from this source are installed, it should only appear once { - Feature: database.Feature{Name: "filesystem"}, + Feature: services.Feature{Name: "filesystem"}, Version: types.NewVersionUnsafe("3.2-18.el7"), }, }, diff --git a/worker/detectors/feature/test.go b/worker/detectors/feature/test.go index fe9f3d55..c69d5d43 100644 --- a/worker/detectors/feature/test.go +++ b/worker/detectors/feature/test.go @@ -20,13 +20,13 @@ import ( "runtime" "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors" "github.com/stretchr/testify/assert" ) type FeatureVersionTest struct { - FeatureVersions []database.FeatureVersion + FeatureVersions []services.FeatureVersion Data map[string][]byte } diff --git a/worker/detectors/features.go b/worker/detectors/features.go index da8d8c06..22c9af43 100644 --- a/worker/detectors/features.go +++ b/worker/detectors/features.go @@ -18,13 +18,13 @@ import ( "fmt" "sync" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" ) // The FeaturesDetector interface defines a way to detect packages from input data. type FeaturesDetector interface { // Detect detects a list of FeatureVersion from the input data. - Detect(map[string][]byte) ([]database.FeatureVersion, error) + Detect(map[string][]byte) ([]services.FeatureVersion, error) // GetRequiredFiles returns the list of files required for Detect, without // leading /. GetRequiredFiles() []string @@ -54,13 +54,13 @@ func RegisterFeaturesDetector(name string, f FeaturesDetector) { } // DetectFeatures detects a list of FeatureVersion using every registered FeaturesDetector. -func DetectFeatures(data map[string][]byte) ([]database.FeatureVersion, error) { - var packages []database.FeatureVersion +func DetectFeatures(data map[string][]byte) ([]services.FeatureVersion, error) { + var packages []services.FeatureVersion for _, detector := range featuresDetectors { pkgs, err := detector.Detect(data) if err != nil { - return []database.FeatureVersion{}, err + return []services.FeatureVersion{}, err } packages = append(packages, pkgs...) } diff --git a/worker/detectors/namespace.go b/worker/detectors/namespace.go index 7d00cdfc..a1dfbfee 100644 --- a/worker/detectors/namespace.go +++ b/worker/detectors/namespace.go @@ -20,14 +20,14 @@ import ( "fmt" "sync" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" ) // The NamespaceDetector interface defines a way to detect a Namespace from input data. // A namespace is usually made of an Operating System name and its version. type NamespaceDetector interface { // Detect detects a Namespace and its version from input data. - Detect(map[string][]byte) *database.Namespace + Detect(map[string][]byte) *services.Namespace // GetRequiredFiles returns the list of files required for Detect, without // leading /. GetRequiredFiles() []string @@ -61,7 +61,7 @@ func RegisterNamespaceDetector(name string, f NamespaceDetector) { } // DetectNamespace finds the OS of the layer by using every registered NamespaceDetector. -func DetectNamespace(data map[string][]byte) *database.Namespace { +func DetectNamespace(data map[string][]byte) *services.Namespace { for _, detector := range namespaceDetectors { if namespace := detector.Detect(data); namespace != nil { return namespace diff --git a/worker/detectors/namespace/aptsources/aptsources.go b/worker/detectors/namespace/aptsources/aptsources.go index 69b6e30f..061d09a9 100644 --- a/worker/detectors/namespace/aptsources/aptsources.go +++ b/worker/detectors/namespace/aptsources/aptsources.go @@ -18,7 +18,8 @@ import ( "bufio" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/namespaces" "github.com/coreos/clair/worker/detectors" ) @@ -33,7 +34,7 @@ func init() { detectors.RegisterNamespaceDetector("apt-sources", &AptSourcesNamespaceDetector{}) } -func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { +func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *services.Namespace { f, hasFile := data["etc/apt/sources.list"] if !hasFile { return nil @@ -61,12 +62,12 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat } var found bool - version, found = database.DebianReleasesMapping[line[2]] + version, found = namespaces.DebianReleasesMapping[line[2]] if found { OS = "debian" break } - version, found = database.UbuntuReleasesMapping[line[2]] + version, found = namespaces.UbuntuReleasesMapping[line[2]] if found { OS = "ubuntu" break @@ -75,7 +76,7 @@ func (detector *AptSourcesNamespaceDetector) Detect(data map[string][]byte) *dat } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &services.Namespace{Name: OS + ":" + version} } return nil } diff --git a/worker/detectors/namespace/aptsources/aptsources_test.go b/worker/detectors/namespace/aptsources/aptsources_test.go index d502dc48..9e7362eb 100644 --- a/worker/detectors/namespace/aptsources/aptsources_test.go +++ b/worker/detectors/namespace/aptsources/aptsources_test.go @@ -17,13 +17,13 @@ package aptsources import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors/namespace" ) var aptSourcesOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "debian:unstable"}, + ExpectedNamespace: services.Namespace{Name: "debian:unstable"}, Data: map[string][]byte{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux stretch/sid" diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease.go b/worker/detectors/namespace/lsbrelease/lsbrelease.go index eab19984..a991ae73 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease.go +++ b/worker/detectors/namespace/lsbrelease/lsbrelease.go @@ -19,7 +19,7 @@ import ( "regexp" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors" ) @@ -38,7 +38,7 @@ func init() { detectors.RegisterNamespaceDetector("lsb-release", &LsbReleaseNamespaceDetector{}) } -func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { +func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace { f, hasFile := data["etc/lsb-release"] if !hasFile { return nil @@ -70,7 +70,7 @@ func (detector *LsbReleaseNamespaceDetector) Detect(data map[string][]byte) *dat } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &services.Namespace{Name: OS + ":" + version} } return nil } diff --git a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go index 9aa3b64f..3c3547d9 100644 --- a/worker/detectors/namespace/lsbrelease/lsbrelease_test.go +++ b/worker/detectors/namespace/lsbrelease/lsbrelease_test.go @@ -17,13 +17,13 @@ package lsbrelease import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors/namespace" ) var lsbReleaseOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "ubuntu:12.04"}, + ExpectedNamespace: services.Namespace{Name: "ubuntu:12.04"}, Data: map[string][]byte{ "etc/lsb-release": []byte( `DISTRIB_ID=Ubuntu @@ -33,7 +33,7 @@ DISTRIB_DESCRIPTION="Ubuntu 12.04 LTS"`), }, }, { // We don't care about the minor version of Debian - ExpectedNamespace: database.Namespace{Name: "debian:7"}, + ExpectedNamespace: services.Namespace{Name: "debian:7"}, Data: map[string][]byte{ "etc/lsb-release": []byte( `DISTRIB_ID=Debian diff --git a/worker/detectors/namespace/osrelease/osrelease.go b/worker/detectors/namespace/osrelease/osrelease.go index 118fb9fd..3d036480 100644 --- a/worker/detectors/namespace/osrelease/osrelease.go +++ b/worker/detectors/namespace/osrelease/osrelease.go @@ -19,7 +19,7 @@ import ( "regexp" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors" ) @@ -39,7 +39,7 @@ func init() { // Detect tries to detect OS/Version using "/etc/os-release" and "/usr/lib/os-release" // Typically for Debian / Ubuntu // /etc/debian_version can't be used, it does not make any difference between testing and unstable, it returns stretch/sid -func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { +func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace { var OS, version string for _, filePath := range detector.GetRequiredFiles() { @@ -65,7 +65,7 @@ func (detector *OsReleaseNamespaceDetector) Detect(data map[string][]byte) *data } if OS != "" && version != "" { - return &database.Namespace{Name: OS + ":" + version} + return &services.Namespace{Name: OS + ":" + version} } return nil } diff --git a/worker/detectors/namespace/osrelease/osrelease_test.go b/worker/detectors/namespace/osrelease/osrelease_test.go index 4b08cf33..a88c9698 100644 --- a/worker/detectors/namespace/osrelease/osrelease_test.go +++ b/worker/detectors/namespace/osrelease/osrelease_test.go @@ -17,13 +17,13 @@ package osrelease import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors/namespace" ) var osReleaseOSTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "debian:8"}, + ExpectedNamespace: services.Namespace{Name: "debian:8"}, Data: map[string][]byte{ "etc/os-release": []byte( `PRETTY_NAME="Debian GNU/Linux 8 (jessie)" @@ -37,7 +37,7 @@ BUG_REPORT_URL="https://bugs.debian.org/"`), }, }, { - ExpectedNamespace: database.Namespace{Name: "ubuntu:15.10"}, + ExpectedNamespace: services.Namespace{Name: "ubuntu:15.10"}, Data: map[string][]byte{ "etc/os-release": []byte( `NAME="Ubuntu" @@ -52,7 +52,7 @@ BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/"`), }, }, { // Doesn't have quotes around VERSION_ID - ExpectedNamespace: database.Namespace{Name: "fedora:20"}, + ExpectedNamespace: services.Namespace{Name: "fedora:20"}, Data: map[string][]byte{ "etc/os-release": []byte( `NAME=Fedora diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease.go b/worker/detectors/namespace/redhatrelease/redhatrelease.go index a6569b07..578ace70 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease.go +++ b/worker/detectors/namespace/redhatrelease/redhatrelease.go @@ -18,7 +18,7 @@ import ( "regexp" "strings" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors" ) @@ -37,7 +37,7 @@ func init() { detectors.RegisterNamespaceDetector("redhat-release", &RedhatReleaseNamespaceDetector{}) } -func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *database.Namespace { +func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) *services.Namespace { for _, filePath := range detector.GetRequiredFiles() { f, hasFile := data[filePath] if !hasFile { @@ -46,7 +46,7 @@ func (detector *RedhatReleaseNamespaceDetector) Detect(data map[string][]byte) * r := redhatReleaseRegexp.FindStringSubmatch(string(f)) if len(r) == 4 { - return &database.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]} + return &services.Namespace{Name: strings.ToLower(r[1]) + ":" + r[3]} } } diff --git a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go index 25c786ac..20a8529d 100644 --- a/worker/detectors/namespace/redhatrelease/redhatrelease_test.go +++ b/worker/detectors/namespace/redhatrelease/redhatrelease_test.go @@ -17,19 +17,19 @@ package redhatrelease import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors/namespace" ) var redhatReleaseTests = []namespace.NamespaceTest{ { - ExpectedNamespace: database.Namespace{Name: "centos:6"}, + ExpectedNamespace: services.Namespace{Name: "centos:6"}, Data: map[string][]byte{ "etc/centos-release": []byte(`CentOS release 6.6 (Final)`), }, }, { - ExpectedNamespace: database.Namespace{Name: "centos:7"}, + ExpectedNamespace: services.Namespace{Name: "centos:7"}, Data: map[string][]byte{ "etc/system-release": []byte(`CentOS Linux release 7.1.1503 (Core)`), }, diff --git a/worker/detectors/namespace/test.go b/worker/detectors/namespace/test.go index 679d23db..4f73fd44 100644 --- a/worker/detectors/namespace/test.go +++ b/worker/detectors/namespace/test.go @@ -17,14 +17,14 @@ package namespace import ( "testing" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" "github.com/coreos/clair/worker/detectors" "github.com/stretchr/testify/assert" ) type NamespaceTest struct { Data map[string][]byte - ExpectedNamespace database.Namespace + ExpectedNamespace services.Namespace } func TestNamespaceDetector(t *testing.T, detector detectors.NamespaceDetector, tests []NamespaceTest) { diff --git a/worker/worker.go b/worker/worker.go index 23ee2d39..d9abbae0 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -19,7 +19,8 @@ package worker import ( "github.com/coreos/pkg/capnslog" - "github.com/coreos/clair/database" + "github.com/coreos/clair/services" + "github.com/coreos/clair/services/layers" "github.com/coreos/clair/utils" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/worker/detectors" @@ -50,7 +51,7 @@ var ( // then stores everything in the database. // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an // older engine version and that processes them. -func Process(datastore database.Datastore, imageFormat, name, parentName, path string, headers map[string]string) error { +func Process(ls layers.Service, imageFormat, name, parentName, path string, headers map[string]string) error { // Verify parameters. if name == "" { return cerrors.NewBadRequestError("could not process a layer which does not have a name") @@ -68,19 +69,19 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s name, utils.CleanURL(path), Version, parentName, imageFormat) // Check to see if the layer is already in the database. - layer, err := datastore.FindLayer(name, false, false) + layer, err := ls.FindLayer(name, false, false) if err != nil && err != cerrors.ErrNotFound { return err } if err == cerrors.ErrNotFound { // New layer case. - layer = database.Layer{Name: name, EngineVersion: Version} + layer = services.Layer{Name: name, EngineVersion: Version} // Retrieve the parent if it has one. // We need to get it with its Features in order to diff them. if parentName != "" { - parent, err := datastore.FindLayer(parentName, true, false) + parent, err := ls.FindLayer(parentName, true, false) if err != nil && err != cerrors.ErrNotFound { return err } @@ -109,11 +110,11 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s return err } - return datastore.InsertLayer(layer) + return ls.InsertLayer(layer) } // detectContent downloads a layer's archive and extracts its Namespace and Features. -func detectContent(imageFormat, name, path string, headers map[string]string, parent *database.Layer) (namespace *database.Namespace, featureVersions []database.FeatureVersion, err error) { +func detectContent(imageFormat, name, path string, headers map[string]string, parent *services.Layer) (namespace *services.Namespace, featureVersions []services.FeatureVersion, err error) { data, err := detectors.DetectData(imageFormat, path, headers, append(detectors.GetRequiredFilesFeatures(), detectors.GetRequiredFilesNamespace()...), maxFileSize) if err != nil { log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err) @@ -135,7 +136,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa return } -func detectNamespace(name string, data map[string][]byte, parent *database.Layer) (namespace *database.Namespace) { +func detectNamespace(name string, data map[string][]byte, parent *services.Layer) (namespace *services.Namespace) { // Use registered detectors to get the Namespace. namespace = detectors.DetectNamespace(data) if namespace != nil { @@ -155,7 +156,7 @@ func detectNamespace(name string, data map[string][]byte, parent *database.Layer return } -func detectFeatureVersions(name string, data map[string][]byte, namespace *database.Namespace, parent *database.Layer) (features []database.FeatureVersion, err error) { +func detectFeatureVersions(name string, data map[string][]byte, namespace *services.Namespace, parent *services.Layer) (features []services.FeatureVersion, err error) { // TODO(Quentin-M): We need to pass the parent image to DetectFeatures because it's possible that // some detectors would need it in order to produce the entire feature list (if they can only // detect a diff). Also, we should probably pass the detected namespace so detectors could @@ -175,7 +176,7 @@ func detectFeatureVersions(name string, data map[string][]byte, namespace *datab } // Build a map of the namespaces for each FeatureVersion in our parent layer. - parentFeatureNamespaces := make(map[string]database.Namespace) + parentFeatureNamespaces := make(map[string]services.Namespace) if parent != nil { for _, parentFeature := range parent.Features { parentFeatureNamespaces[parentFeature.Feature.Name+":"+parentFeature.Version.String()] = parentFeature.Feature.Namespace diff --git a/worker/worker_test.go b/worker/worker_test.go index 31829e44..e7f1d2fd 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/coreos/clair/database" + "github.com/coreos/clair/services" cerrors "github.com/coreos/clair/utils/errors" "github.com/coreos/clair/utils/types" @@ -34,12 +35,12 @@ import ( type mockDatastore struct { database.MockDatastore - layers map[string]database.Layer + layers map[string]services.Layer } func newMockDatastore() *mockDatastore { return &mockDatastore{ - layers: make(map[string]database.Layer), + layers: make(map[string]services.Layer), } } @@ -49,27 +50,27 @@ func TestProcessWithDistUpgrade(t *testing.T) { // Create a mock datastore. datastore := newMockDatastore() - datastore.FctInsertLayer = func(layer database.Layer) error { + datastore.FctInsertLayer = func(layer services.Layer) error { datastore.layers[layer.Name] = layer return nil } - datastore.FctFindLayer = func(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) { + datastore.FctFindLayer = func(name string, withFeatures, withVulnerabilities bool) (services.Layer, error) { if layer, exists := datastore.layers[name]; exists { return layer, nil } - return database.Layer{}, cerrors.ErrNotFound + return services.Layer{}, cerrors.ErrNotFound } // Create the list of FeatureVersions that should not been upgraded from one layer to another. - nonUpgradedFeatureVersions := []database.FeatureVersion{ - {Feature: database.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")}, - {Feature: database.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")}, - {Feature: database.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")}, - {Feature: database.Feature{Name: "mawk"}, Version: types.NewVersionUnsafe("1.3.3-17")}, - {Feature: database.Feature{Name: "insserv"}, Version: types.NewVersionUnsafe("1.14.0-5")}, - {Feature: database.Feature{Name: "db"}, Version: types.NewVersionUnsafe("5.1.29-5")}, - {Feature: database.Feature{Name: "ustr"}, Version: types.NewVersionUnsafe("1.0.4-3")}, - {Feature: database.Feature{Name: "xz-utils"}, Version: types.NewVersionUnsafe("5.1.1alpha+20120614-2")}, + nonUpgradedFeatureVersions := []services.FeatureVersion{ + {Feature: services.Feature{Name: "libtext-wrapi18n-perl"}, Version: types.NewVersionUnsafe("0.06-7")}, + {Feature: services.Feature{Name: "libtext-charwidth-perl"}, Version: types.NewVersionUnsafe("0.04-7")}, + {Feature: services.Feature{Name: "libtext-iconv-perl"}, Version: types.NewVersionUnsafe("1.7-5")}, + {Feature: services.Feature{Name: "mawk"}, Version: types.NewVersionUnsafe("1.3.3-17")}, + {Feature: services.Feature{Name: "insserv"}, Version: types.NewVersionUnsafe("1.14.0-5")}, + {Feature: services.Feature{Name: "db"}, Version: types.NewVersionUnsafe("5.1.29-5")}, + {Feature: services.Feature{Name: "ustr"}, Version: types.NewVersionUnsafe("1.0.4-3")}, + {Feature: services.Feature{Name: "xz-utils"}, Version: types.NewVersionUnsafe("5.1.1alpha+20120614-2")}, } // Process test layers.