From 57b146d0d808a29db9f299778fb5527cd0974b06 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 26 Jul 2017 16:20:19 -0700 Subject: [PATCH] Datastore: updated for Clair V3, decoupled interfaces and models --- database/database.go | 229 ++++++++++++++++++------------------ database/models.go | 218 +++++++++++++++++++++++++--------- database/severity.go | 14 ++- pkg/strutil/strutil.go | 55 +++++++++ pkg/strutil/strutil_test.go | 34 ++++++ 5 files changed, 382 insertions(+), 168 deletions(-) create mode 100644 pkg/strutil/strutil.go create mode 100644 pkg/strutil/strutil_test.go diff --git a/database/database.go b/database/database.go index d3f7fc0f..16925bb1 100644 --- a/database/database.go +++ b/database/database.go @@ -23,9 +23,9 @@ import ( ) var ( - // ErrBackendException is an error that occurs when the database backend does - // not work properly (ie. unreachable). - ErrBackendException = errors.New("database: an error occured when querying the backend") + // ErrBackendException is an error that occurs when the database backend + // does not work properly (ie. unreachable). + ErrBackendException = errors.New("database: an error occurred when querying the backend") // ErrInconsistent is an error that occurs when a database consistency check // fails (i.e. when an entity which is supposed to be unique is detected @@ -43,8 +43,8 @@ type RegistrableComponentConfig struct { var drivers = make(map[string]Driver) -// Driver is a function that opens a Datastore specified by its database driver type and specific -// configuration. +// Driver is a function that opens a Datastore specified by its database driver +// type and specific configuration. type Driver func(RegistrableComponentConfig) (Datastore, error) // Register makes a Constructor available by the provided name. @@ -70,130 +70,127 @@ func Open(cfg RegistrableComponentConfig) (Datastore, error) { return driver(cfg) } -// Datastore represents the required operations on a persistent data store for -// a Clair deployment. -type Datastore interface { - // ListNamespaces returns the entire list of known Namespaces. - ListNamespaces() ([]Namespace, error) - - // InsertLayer stores a Layer in the database. +// Session contains the required operations on a persistent data store for a +// Clair deployment. +// +// Session is started by Datastore.Begin and terminated with Commit or Rollback. +// Besides Commit and Rollback, other functions cannot be called after the +// session is terminated. +// Any function is not guaranteed to be called successfully if there's a session +// failure. +type Session interface { + // Commit commits changes to datastore. // - // A Layer is uniquely identified by its Name. - // The Name and EngineVersion fields are mandatory. - // If a Parent is specified, it is expected that it has been retrieved using - // FindLayer. - // If a Layer that already exists is inserted and the EngineVersion of the - // given Layer is higher than the stored one, the stored Layer should be - // updated. - // The function has to be idempotent, inserting a layer that already exists - // shouldn't return an error. - InsertLayer(Layer) error + // Commit call after Rollback does no-op. + Commit() error - // FindLayer retrieves a Layer from the database. + // Rollback drops changes to datastore. // - // When `withFeatures` is true, the Features field should be filled. - // When `withVulnerabilities` is true, the Features field should be filled - // and their AffectedBy fields should contain every vulnerabilities that - // affect them. - FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error) + // Rollback call after Commit does no-op. + Rollback() error - // DeleteLayer deletes a Layer from the database and every layers that are - // based on it, recursively. - DeleteLayer(name string) error + // UpsertAncestry inserts or replaces an ancestry and its namespaced + // features and processors used to scan the ancestry. + UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error - // ListVulnerabilities returns the list of vulnerabilities of a particular - // Namespace. + // FindAncestry retrieves an ancestry with processors used to scan the + // ancestry. If the ancestry is not found, return false. // - // The Limit and page parameters are used to paginate the return list. - // The first given page should be 0. - // The function should return the next available page. If there are no more - // pages, -1 has to be returned. - ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error) + // The ancestry's processors are returned to short cut processing ancestry + // if it has been processed by all processors in the current Clair instance. + FindAncestry(name string) (ancestry Ancestry, processedBy Processors, found bool, err error) - // InsertVulnerabilities stores the given Vulnerabilities in the database, - // updating them if necessary. + // FindAncestryFeatures retrieves an ancestry with all detected namespaced + // features. If the ancestry is not found, return false. + FindAncestryFeatures(name string) (ancestry AncestryWithFeatures, found bool, err error) + + // PersistFeatures inserts a set of features if not in the database. + PersistFeatures(features []Feature) error + + // PersistNamespacedFeatures inserts a set of namespaced features if not in + // the database. + PersistNamespacedFeatures([]NamespacedFeature) error + + // CacheAffectedNamespacedFeatures relates the namespaced features with the + // vulnerabilities affecting these features. // - // A vulnerability is uniquely identified by its Namespace and its Name. - // The FixedIn field may only contain a partial list of Features that are - // affected by the Vulnerability, along with the version in which the - // vulnerability is fixed. It is the responsibility of the implementation to - // update the list properly. - // A version equals to versionfmt.MinVersion means that the given Feature is - // not being affected by the Vulnerability at all and thus, should be removed - // from the list. - // It is important that Features should be unique in the FixedIn list. For - // example, it doesn't make sense to have two `openssl` Feature listed as a - // Vulnerability can only be fixed in one Version. This is true because - // Vulnerabilities and Features are namespaced (i.e. specific to one - // operating system). - // Each vulnerability insertion or update has to create a Notification that - // will contain the old and the updated Vulnerability, unless - // createNotification equals to true. - InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error + // NOTE(Sida): it's not necessary for every database implementation and so + // this function may have a better home. + CacheAffectedNamespacedFeatures([]NamespacedFeature) error - // FindVulnerability retrieves a Vulnerability from the database, including - // the FixedIn list. - FindVulnerability(namespaceName, name string) (Vulnerability, error) + // FindAffectedNamespacedFeatures retrieves a set of namespaced features + // with affecting vulnerabilities. + FindAffectedNamespacedFeatures(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error) - // DeleteVulnerability removes a Vulnerability from the database. + // PersistNamespaces inserts a set of namespaces if not in the database. + PersistNamespaces([]Namespace) error + + // PersistLayer inserts a layer if not in the datastore. + PersistLayer(Layer) error + + // PersistLayerContent persists a layer's content in the database. The given + // namespaces and features can be partial content of this layer. // - // It has to create a Notification that will contain the old Vulnerability. - DeleteVulnerability(namespaceName, name string) error + // The layer, namespaces and features are expected to be already existing + // in the database. + PersistLayerContent(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error - // InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions - // of existing ones to the specified Vulnerability in the database. + // FindLayer retrieves a layer and the processors scanned the layer. + FindLayer(hash string) (layer Layer, processedBy Processors, found bool, err error) + + // FindLayerWithContent returns a layer with all detected features and + // namespaces. + FindLayerWithContent(hash string) (layer LayerWithContent, found bool, err error) + + // InsertVulnerabilities inserts a set of UNIQUE vulnerabilities with + // affected features into database, assuming that all vulnerabilities + // provided are NOT in database and all vulnerabilities' namespaces are + // already in the database. + InsertVulnerabilities([]VulnerabilityWithAffected) error + + // FindVulnerability retrieves a set of Vulnerabilities with affected + // features. + FindVulnerabilities([]VulnerabilityID) ([]NullableVulnerability, error) + + // DeleteVulnerability removes a set of Vulnerabilities assuming that the + // requested vulnerabilities are in the database. + DeleteVulnerabilities([]VulnerabilityID) error + + // InsertVulnerabilityNotifications inserts a set of unique vulnerability + // notifications into datastore, assuming that they are not in the database. + InsertVulnerabilityNotifications([]VulnerabilityNotification) error + + // FindNewNotification retrieves a notification, which has never been + // notified or notified before a certain time. + FindNewNotification(notifiedBefore time.Time) (hook NotificationHook, found bool, err error) + + // FindVulnerabilityNotification retrieves a vulnerability notification with + // affected ancestries affected by old or new vulnerability. // - // It has has to create a Notification that will contain the old and the - // updated Vulnerability. - InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error - - // DeleteVulnerabilityFix removes a FixedIn Feature from the specified - // Vulnerability in the database. It can be used to store the fact that a - // Vulnerability no longer affects the given Feature in any Version. + // Because the number of affected ancestries maybe large, they are paginated + // and their pages are specified by the given encrypted PageNumbers, which, + // if empty, are always considered first page. // - // It has has to create a Notification that will contain the old and the - // updated Vulnerability. - DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error + // Session interface implementation should have encrypt and decrypt + // functions for PageNumber. + FindVulnerabilityNotification(name string, limit int, + oldVulnerabilityPage PageNumber, + newVulnerabilityPage PageNumber) ( + noti VulnerabilityNotificationWithVulnerable, + found bool, err error) - // GetAvailableNotification returns the Name, Created, Notified and Deleted - // fields of a Notification that should be handled. - // - // The renotify interval defines how much time after being marked as Notified - // by SetNotificationNotified, a Notification that hasn't been deleted should - // be returned again by this function. - // A Notification for which there is a valid Lock with the same Name should - // not be returned. - GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error) + // MarkNotificationNotified marks a Notification as notified now, assuming + // the requested notification is in the database. + MarkNotificationNotified(name string) error - // GetNotification returns a Notification, including its OldVulnerability and - // NewVulnerability fields. - // - // On these Vulnerabilities, LayersIntroducingVulnerability should be filled - // with every Layer that introduces the Vulnerability (i.e. adds at least one - // affected FeatureVersion). - // The Limit and page parameters are used to paginate - // LayersIntroducingVulnerability. The first given page should be - // VulnerabilityNotificationFirstPage. The function will then return the next - // available page. If there is no more page, NoVulnerabilityNotificationPage - // has to be returned. - GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error) - - // SetNotificationNotified marks a Notification as notified and thus, makes - // it unavailable for GetAvailableNotification, until the renotify duration - // is elapsed. - SetNotificationNotified(name string) error - - // DeleteNotification marks a Notification as deleted, and thus, makes it - // unavailable for GetAvailableNotification. + // DeleteNotification removes a Notification in the database. DeleteNotification(name string) error - // InsertKeyValue stores or updates a simple key/value pair in the database. - InsertKeyValue(key, value string) error + // UpdateKeyValue stores or updates a simple key/value pair. + UpdateKeyValue(key, value string) error - // GetKeyValue retrieves a value from the database from the given key. - // - // It returns an empty string if there is no such key. - GetKeyValue(key string) (string, error) + // FindKeyValue retrieves a value from the given key. + FindKeyValue(key string) (value string, found bool, err error) // Lock creates or renew a Lock in the database with the given name, owner // and duration. @@ -204,14 +201,20 @@ type Datastore interface { // Lock should not block, it should instead returns whether the Lock has been // successfully acquired/renewed. If it's the case, the expiration time of // that Lock is returned as well. - Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time) + Lock(name string, owner string, duration time.Duration, renew bool) (success bool, expiration time.Time, err error) // Unlock releases an existing Lock. - Unlock(name, owner string) + Unlock(name, owner string) error // FindLock returns the owner of a Lock specified by the name, and its // expiration time if it exists. - FindLock(name string) (string, time.Time, error) + FindLock(name string) (owner string, expiration time.Time, found bool, err error) +} + +// Datastore represents a persistent data store +type Datastore interface { + // Begin starts a session to change. + Begin() (Session, error) // Ping returns the health status of the database. Ping() bool diff --git a/database/models.go b/database/models.go index 608f2449..fe36fbfc 100644 --- a/database/models.go +++ b/database/models.go @@ -20,49 +20,115 @@ import ( "time" ) -// ID is only meant to be used by database implementations and should never be used for anything else. -type Model struct { - ID int +// Processors are extentions to scan layer's content. +type Processors struct { + Listers []string + Detectors []string } +// Ancestry is a manifest that keeps all layers in an image in order. +type Ancestry struct { + Name string + // Layers should be ordered and i_th layer is the parent of i+1_th layer in + // the slice. + Layers []Layer +} + +// AncestryWithFeatures is an ancestry with namespaced features detected in the +// ancestry, which is processed by `ProcessedBy`. +type AncestryWithFeatures struct { + Ancestry + + ProcessedBy Processors + Features []NamespacedFeature +} + +// Layer corresponds to a layer in an image processed by `ProcessedBy`. type Layer struct { - Model - - Name string - EngineVersion int - Parent *Layer - Namespaces []Namespace - Features []FeatureVersion + // Hash is content hash of the layer. + Hash string } -type Namespace struct { - Model +// LayerWithContent is a layer with its detected namespaces and features by +// ProcessedBy. +type LayerWithContent struct { + Layer + ProcessedBy Processors + Namespaces []Namespace + Features []Feature +} + +// Namespace is the contextual information around features. +// +// e.g. Debian:7, NodeJS. +type Namespace struct { Name string VersionFormat string } +// Feature represents a package detected in a layer but the namespace is not +// determined. +// +// e.g. Name: OpenSSL, Version: 1.0, VersionFormat: dpkg. +// dpkg implies the installer package manager but the namespace (might be +// debian:7, debian:8, ...) could not be determined. type Feature struct { - Model + Name string + Version string + VersionFormat string +} + +// NamespacedFeature is a feature with determined namespace and can be affected +// by vulnerabilities. +// +// e.g. OpenSSL 1.0 dpkg Debian:7. +type NamespacedFeature struct { + Feature - Name string Namespace Namespace } -type FeatureVersion struct { - Model +// AffectedNamespacedFeature is a namespaced feature affected by the +// vulnerabilities with fixed-in versions for this feature. +type AffectedNamespacedFeature struct { + NamespacedFeature - Feature Feature - Version string - AffectedBy []Vulnerability - - // For output purposes. Only make sense when the feature version is in the context of an image. - AddedBy Layer + AffectedBy []VulnerabilityWithFixedIn } -type Vulnerability struct { - Model +// VulnerabilityWithFixedIn is used for AffectedNamespacedFeature to retrieve +// the affecting vulnerabilities and the fixed-in versions for the feature. +type VulnerabilityWithFixedIn struct { + Vulnerability + FixedInVersion string +} + +// AffectedFeature is used to determine whether a namespaced feature is affected +// by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is +// bound to vulnerability. +type AffectedFeature struct { + Namespace Namespace + FeatureName string + // FixedInVersion is known next feature version that's not affected by the + // vulnerability. Empty FixedInVersion means the unaffected version is + // unknown. + FixedInVersion string + // AffectedVersion contains the version range to determine whether or not a + // feature is affected. + AffectedVersion string +} + +// VulnerabilityID is an identifier for every vulnerability. Every vulnerability +// has unique namespace and name. +type VulnerabilityID struct { + Name string + Namespace string +} + +// Vulnerability represents CVE or similar vulnerability reports. +type Vulnerability struct { Name string Namespace Namespace @@ -71,17 +137,85 @@ type Vulnerability struct { Severity Severity Metadata MetadataMap - - FixedIn []FeatureVersion - LayersIntroducingVulnerability []Layer - - // For output purposes. Only make sense when the vulnerability - // is already about a specific Feature/FeatureVersion. - FixedBy string `json:",omitempty"` } +// VulnerabilityWithAffected is an vulnerability with all known affected +// features. +type VulnerabilityWithAffected struct { + Vulnerability + + Affected []AffectedFeature +} + +// PagedVulnerableAncestries is a vulnerability with a page of affected +// ancestries each with a special index attached for streaming purpose. The +// current page number and next page number are for navigate. +type PagedVulnerableAncestries struct { + Vulnerability + + // Affected is a map of special indexes to Ancestries, which the pair + // should be unique in a stream. Every indexes in the map should be larger + // than previous page. + Affected map[int]string + + Limit int + Current PageNumber + Next PageNumber + + // End signals the end of the pages. + End bool +} + +// NotificationHook is a message sent to another service to inform of a change +// to a Vulnerability or the Ancestries affected by a Vulnerability. It contains +// the name of a notification that should be read and marked as read via the +// API. +type NotificationHook struct { + Name string + + Created time.Time + Notified time.Time + Deleted time.Time +} + +// VulnerabilityNotification is a notification for vulnerability changes. +type VulnerabilityNotification struct { + NotificationHook + + Old *Vulnerability + New *Vulnerability +} + +// VulnerabilityNotificationWithVulnerable is a notification for vulnerability +// changes with vulnerable ancestries. +type VulnerabilityNotificationWithVulnerable struct { + NotificationHook + + Old *PagedVulnerableAncestries + New *PagedVulnerableAncestries +} + +// PageNumber is used to do pagination. +type PageNumber string + type MetadataMap map[string]interface{} +// NullableAffectedNamespacedFeature is an affectednamespacedfeature with +// whether it's found in datastore. +type NullableAffectedNamespacedFeature struct { + AffectedNamespacedFeature + + Valid bool +} + +// NullableVulnerability is a vulnerability with whether the vulnerability is +// found in datastore. +type NullableVulnerability struct { + VulnerabilityWithAffected + + Valid bool +} + func (mm *MetadataMap) Scan(value interface{}) error { if value == nil { return nil @@ -99,25 +233,3 @@ func (mm *MetadataMap) Value() (driver.Value, error) { json, err := json.Marshal(*mm) return string(json), err } - -type VulnerabilityNotification struct { - Model - - Name string - - Created time.Time - Notified time.Time - Deleted time.Time - - OldVulnerability *Vulnerability - NewVulnerability *Vulnerability -} - -type VulnerabilityNotificationPageNumber struct { - // -1 means that we reached the end already. - OldVulnerability int - NewVulnerability int -} - -var VulnerabilityNotificationFirstPage = VulnerabilityNotificationPageNumber{0, 0} -var NoVulnerabilityNotificationPage = VulnerabilityNotificationPageNumber{-1, -1} diff --git a/database/severity.go b/database/severity.go index 58084d64..840f6afb 100644 --- a/database/severity.go +++ b/database/severity.go @@ -36,7 +36,7 @@ const ( // NegligibleSeverity is technically a security problem, but is only // theoretical in nature, requires a very special situation, has almost no // install base, or does no real damage. These tend not to get backport from - // upstreams, and will likely not be included in security updates unless + // upstream, and will likely not be included in security updates unless // there is an easy fix and some other issue causes an update. NegligibleSeverity Severity = "Negligible" @@ -93,7 +93,7 @@ func NewSeverity(s string) (Severity, error) { // Compare determines the equality of two severities. // // If the severities are equal, returns 0. -// If the receiever is less, returns -1. +// If the receiver is less, returns -1. // If the receiver is greater, returns 1. func (s Severity) Compare(s2 Severity) int { var i1, i2 int @@ -132,3 +132,13 @@ func (s *Severity) Scan(value interface{}) error { func (s Severity) Value() (driver.Value, error) { return string(s), nil } + +// Valid checks if the severity is valid or not. +func (s Severity) Valid() bool { + for _, v := range Severities { + if s == v { + return true + } + } + return false +} diff --git a/pkg/strutil/strutil.go b/pkg/strutil/strutil.go new file mode 100644 index 00000000..a8d04f21 --- /dev/null +++ b/pkg/strutil/strutil.go @@ -0,0 +1,55 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package strutil + +// CompareStringLists returns the strings that are present in X but not in Y. +func CompareStringLists(X, Y []string) []string { + m := make(map[string]bool) + + for _, y := range Y { + m[y] = true + } + + diff := []string{} + for _, x := range X { + if m[x] { + continue + } + + diff = append(diff, x) + m[x] = true + } + + return diff +} + +// CompareStringListsInBoth returns the strings that are present in both X and Y. +func CompareStringListsInBoth(X, Y []string) []string { + m := make(map[string]struct{}) + + for _, y := range Y { + m[y] = struct{}{} + } + + diff := []string{} + for _, x := range X { + if _, e := m[x]; e { + diff = append(diff, x) + delete(m, x) + } + } + + return diff +} diff --git a/pkg/strutil/strutil_test.go b/pkg/strutil/strutil_test.go new file mode 100644 index 00000000..4cbf1e90 --- /dev/null +++ b/pkg/strutil/strutil_test.go @@ -0,0 +1,34 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package strutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringComparison(t *testing.T) { + cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"}) + assert.Len(t, cmp, 1) + assert.NotContains(t, cmp, "a") + assert.Contains(t, cmp, "b") + + cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"}) + assert.Len(t, cmp, 2) + assert.NotContains(t, cmp, "b") + assert.Contains(t, cmp, "a") + assert.Contains(t, cmp, "c") +}