arno
/
clair
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Datastore: updated for Clair V3, decoupled interfaces and models

Browse Source
pull/432/head
Sida Chen 7 years ago
parent a378cb070c
commit 57b146d0d8
5 changed files with 384 additions and 170 deletions
Show Stats Download Patch File Download Diff File

241
database/database.go
Unescape View File

@ -23,9 +23,9 @@ import (
)
var (
// ErrBackendException is an error that occurs when the database backend does
// not work properly (ie. unreachable).
ErrBackendException = errors.New("database: an error occured when querying the backend")
// ErrBackendException is an error that occurs when the database backend
// does not work properly (ie. unreachable).
ErrBackendException = errors.New("database: an error occurred when querying the backend")
// ErrInconsistent is an error that occurs when a database consistency check
// fails (i.e. when an entity which is supposed to be unique is detected
@ -43,8 +43,8 @@ type RegistrableComponentConfig struct {
var drivers = make(map[string]Driver)
// Driver is a function that opens a Datastore specified by its database driver type and specific
// configuration.
// Driver is a function that opens a Datastore specified by its database driver
// type and specific configuration.
type Driver func(RegistrableComponentConfig) (Datastore, error)
// Register makes a Constructor available by the provided name.
@ -70,130 +70,127 @@ func Open(cfg RegistrableComponentConfig) (Datastore, error) {
return driver(cfg)
}
// Datastore represents the required operations on a persistent data store for
// a Clair deployment.
type Datastore interface {
// ListNamespaces returns the entire list of known Namespaces.
ListNamespaces() ([]Namespace, error)
// InsertLayer stores a Layer in the database.
//
// A Layer is uniquely identified by its Name.
// The Name and EngineVersion fields are mandatory.
// If a Parent is specified, it is expected that it has been retrieved using
// FindLayer.
// If a Layer that already exists is inserted and the EngineVersion of the
// given Layer is higher than the stored one, the stored Layer should be
// updated.
// The function has to be idempotent, inserting a layer that already exists
// shouldn't return an error.
InsertLayer(Layer) error
// FindLayer retrieves a Layer from the database.
//
// When `withFeatures` is true, the Features field should be filled.
// When `withVulnerabilities` is true, the Features field should be filled
// and their AffectedBy fields should contain every vulnerabilities that
// affect them.
FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error)
// DeleteLayer deletes a Layer from the database and every layers that are
// based on it, recursively.
DeleteLayer(name string) error
// ListVulnerabilities returns the list of vulnerabilities of a particular
// Namespace.
// Session contains the required operations on a persistent data store for a
// Clair deployment.
//
// Session is started by Datastore.Begin and terminated with Commit or Rollback.
// Besides Commit and Rollback, other functions cannot be called after the
// session is terminated.
// Any function is not guaranteed to be called successfully if there's a session
// failure.
type Session interface {
// Commit commits changes to datastore.
//
// The Limit and page parameters are used to paginate the return list.
// The first given page should be 0.
// The function should return the next available page. If there are no more
// pages, -1 has to be returned.
ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
// InsertVulnerabilities stores the given Vulnerabilities in the database,
// updating them if necessary.
// Commit call after Rollback does no-op.
Commit() error
// Rollback drops changes to datastore.
//
// A vulnerability is uniquely identified by its Namespace and its Name.
// The FixedIn field may only contain a partial list of Features that are
// affected by the Vulnerability, along with the version in which the
// vulnerability is fixed. It is the responsibility of the implementation to
// update the list properly.
// A version equals to versionfmt.MinVersion means that the given Feature is
// not being affected by the Vulnerability at all and thus, should be removed
// from the list.
// It is important that Features should be unique in the FixedIn list. For
// example, it doesn't make sense to have two `openssl` Feature listed as a
// Vulnerability can only be fixed in one Version. This is true because
// Vulnerabilities and Features are namespaced (i.e. specific to one
// operating system).
// Each vulnerability insertion or update has to create a Notification that
// will contain the old and the updated Vulnerability, unless
// createNotification equals to true.
InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
// FindVulnerability retrieves a Vulnerability from the database, including
// the FixedIn list.
FindVulnerability(namespaceName, name string) (Vulnerability, error)
// DeleteVulnerability removes a Vulnerability from the database.
// Rollback call after Commit does no-op.
Rollback() error
// UpsertAncestry inserts or replaces an ancestry and its namespaced
// features and processors used to scan the ancestry.
UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error
// FindAncestry retrieves an ancestry with processors used to scan the
// ancestry. If the ancestry is not found, return false.
//
// It has to create a Notification that will contain the old Vulnerability.
DeleteVulnerability(namespaceName, name string) error
// The ancestry's processors are returned to short cut processing ancestry
// if it has been processed by all processors in the current Clair instance.
FindAncestry(name string) (ancestry Ancestry, processedBy Processors, found bool, err error)
// FindAncestryFeatures retrieves an ancestry with all detected namespaced
// features. If the ancestry is not found, return false.
FindAncestryFeatures(name string) (ancestry AncestryWithFeatures, found bool, err error)
// PersistFeatures inserts a set of features if not in the database.
PersistFeatures(features []Feature) error
// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions
// of existing ones to the specified Vulnerability in the database.
// PersistNamespacedFeatures inserts a set of namespaced features if not in
// the database.
PersistNamespacedFeatures([]NamespacedFeature) error
// CacheAffectedNamespacedFeatures relates the namespaced features with the
// vulnerabilities affecting these features.
//
// It has has to create a Notification that will contain the old and the
// updated Vulnerability.
InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
// NOTE(Sida): it's not necessary for every database implementation and so
// this function may have a better home.
CacheAffectedNamespacedFeatures([]NamespacedFeature) error
// FindAffectedNamespacedFeatures retrieves a set of namespaced features
// with affecting vulnerabilities.
FindAffectedNamespacedFeatures(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error)
// PersistNamespaces inserts a set of namespaces if not in the database.
PersistNamespaces([]Namespace) error
// DeleteVulnerabilityFix removes a FixedIn Feature from the specified
// Vulnerability in the database. It can be used to store the fact that a
// Vulnerability no longer affects the given Feature in any Version.
// PersistLayer inserts a layer if not in the datastore.
PersistLayer(Layer) error
// PersistLayerContent persists a layer's content in the database. The given
// namespaces and features can be partial content of this layer.
//
// It has has to create a Notification that will contain the old and the
// updated Vulnerability.
DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
// The layer, namespaces and features are expected to be already existing
// in the database.
PersistLayerContent(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
// FindLayer retrieves a layer and the processors scanned the layer.
FindLayer(hash string) (layer Layer, processedBy Processors, found bool, err error)
// FindLayerWithContent returns a layer with all detected features and
// namespaces.
FindLayerWithContent(hash string) (layer LayerWithContent, found bool, err error)
// InsertVulnerabilities inserts a set of UNIQUE vulnerabilities with
// affected features into database, assuming that all vulnerabilities
// provided are NOT in database and all vulnerabilities' namespaces are
// already in the database.
InsertVulnerabilities([]VulnerabilityWithAffected) error
// FindVulnerability retrieves a set of Vulnerabilities with affected
// features.
FindVulnerabilities([]VulnerabilityID) ([]NullableVulnerability, error)
// DeleteVulnerability removes a set of Vulnerabilities assuming that the
// requested vulnerabilities are in the database.
DeleteVulnerabilities([]VulnerabilityID) error
// GetAvailableNotification returns the Name, Created, Notified and Deleted
// fields of a Notification that should be handled.
// InsertVulnerabilityNotifications inserts a set of unique vulnerability
// notifications into datastore, assuming that they are not in the database.
InsertVulnerabilityNotifications([]VulnerabilityNotification) error
// FindNewNotification retrieves a notification, which has never been
// notified or notified before a certain time.
FindNewNotification(notifiedBefore time.Time) (hook NotificationHook, found bool, err error)
// FindVulnerabilityNotification retrieves a vulnerability notification with
// affected ancestries affected by old or new vulnerability.
//
// The renotify interval defines how much time after being marked as Notified
// by SetNotificationNotified, a Notification that hasn't been deleted should
// be returned again by this function.
// A Notification for which there is a valid Lock with the same Name should
// not be returned.
GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error)
// GetNotification returns a Notification, including its OldVulnerability and
// NewVulnerability fields.
// Because the number of affected ancestries maybe large, they are paginated
// and their pages are specified by the given encrypted PageNumbers, which,
// if empty, are always considered first page.
//
// On these Vulnerabilities, LayersIntroducingVulnerability should be filled
// with every Layer that introduces the Vulnerability (i.e. adds at least one
// affected FeatureVersion).
// The Limit and page parameters are used to paginate
// LayersIntroducingVulnerability. The first given page should be
// VulnerabilityNotificationFirstPage. The function will then return the next
// available page. If there is no more page, NoVulnerabilityNotificationPage
// has to be returned.
GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
// SetNotificationNotified marks a Notification as notified and thus, makes
// it unavailable for GetAvailableNotification, until the renotify duration
// is elapsed.
SetNotificationNotified(name string) error
// DeleteNotification marks a Notification as deleted, and thus, makes it
// unavailable for GetAvailableNotification.
// Session interface implementation should have encrypt and decrypt
// functions for PageNumber.
FindVulnerabilityNotification(name string, limit int,
oldVulnerabilityPage PageNumber,
newVulnerabilityPage PageNumber) (
noti VulnerabilityNotificationWithVulnerable,
found bool, err error)
// MarkNotificationNotified marks a Notification as notified now, assuming
// the requested notification is in the database.
MarkNotificationNotified(name string) error
// DeleteNotification removes a Notification in the database.
DeleteNotification(name string) error
// InsertKeyValue stores or updates a simple key/value pair in the database.
InsertKeyValue(key, value string) error
// UpdateKeyValue stores or updates a simple key/value pair.
UpdateKeyValue(key, value string) error
// GetKeyValue retrieves a value from the database from the given key.
//
// It returns an empty string if there is no such key.
GetKeyValue(key string) (string, error)
// FindKeyValue retrieves a value from the given key.
FindKeyValue(key string) (value string, found bool, err error)
// Lock creates or renew a Lock in the database with the given name, owner
// and duration.
@ -204,14 +201,20 @@ type Datastore interface {
// Lock should not block, it should instead returns whether the Lock has been
// successfully acquired/renewed. If it's the case, the expiration time of
// that Lock is returned as well.
Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time)
Lock(name string, owner string, duration time.Duration, renew bool) (success bool, expiration time.Time, err error)
// Unlock releases an existing Lock.
Unlock(name, owner string)
Unlock(name, owner string) error
// FindLock returns the owner of a Lock specified by the name, and its
// expiration time if it exists.
FindLock(name string) (string, time.Time, error)
FindLock(name string) (owner string, expiration time.Time, found bool, err error)
}
// Datastore represents a persistent data store
type Datastore interface {
// Begin starts a session to change.
Begin() (Session, error)
// Ping returns the health status of the database.
Ping() bool

210
database/models.go
Unescape View File

@ -20,49 +20,115 @@ import (
"time"
)
// ID is only meant to be used by database implementations and should never be used for anything else.
type Model struct {
ID int
// Processors are extentions to scan layer's content.
type Processors struct {
Listers []string
Detectors []string
}
// Ancestry is a manifest that keeps all layers in an image in order.
type Ancestry struct {
Name string
// Layers should be ordered and i_th layer is the parent of i+1_th layer in
// the slice.
Layers []Layer
}
// AncestryWithFeatures is an ancestry with namespaced features detected in the
// ancestry, which is processed by `ProcessedBy`.
type AncestryWithFeatures struct {
Ancestry
ProcessedBy Processors
Features []NamespacedFeature
}
// Layer corresponds to a layer in an image processed by `ProcessedBy`.
type Layer struct {
Model
// Hash is content hash of the layer.
Hash string
}
Name string
EngineVersion int
Parent *Layer
Namespaces []Namespace
Features []FeatureVersion
// LayerWithContent is a layer with its detected namespaces and features by
// ProcessedBy.
type LayerWithContent struct {
Layer
ProcessedBy Processors
Namespaces []Namespace
Features []Feature
}
// Namespace is the contextual information around features.
//
// e.g. Debian:7, NodeJS.
type Namespace struct {
Model
Name string
VersionFormat string
}
// Feature represents a package detected in a layer but the namespace is not
// determined.
//
// e.g. Name: OpenSSL, Version: 1.0, VersionFormat: dpkg.
// dpkg implies the installer package manager but the namespace (might be
// debian:7, debian:8, ...) could not be determined.
type Feature struct {
Model
Name string
Version string
VersionFormat string
}
// NamespacedFeature is a feature with determined namespace and can be affected
// by vulnerabilities.
//
// e.g. OpenSSL 1.0 dpkg Debian:7.
type NamespacedFeature struct {
Feature
Name string
Namespace Namespace
}
type FeatureVersion struct {
Model
// AffectedNamespacedFeature is a namespaced feature affected by the
// vulnerabilities with fixed-in versions for this feature.
type AffectedNamespacedFeature struct {
NamespacedFeature
AffectedBy []VulnerabilityWithFixedIn
}
Feature Feature
Version string
AffectedBy []Vulnerability
// VulnerabilityWithFixedIn is used for AffectedNamespacedFeature to retrieve
// the affecting vulnerabilities and the fixed-in versions for the feature.
type VulnerabilityWithFixedIn struct {
Vulnerability
// For output purposes. Only make sense when the feature version is in the context of an image.
AddedBy Layer
FixedInVersion string
}
type Vulnerability struct {
Model
// AffectedFeature is used to determine whether a namespaced feature is affected
// by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is
// bound to vulnerability.
type AffectedFeature struct {
Namespace Namespace
FeatureName string
// FixedInVersion is known next feature version that's not affected by the
// vulnerability. Empty FixedInVersion means the unaffected version is
// unknown.
FixedInVersion string
// AffectedVersion contains the version range to determine whether or not a
// feature is affected.
AffectedVersion string
}
// VulnerabilityID is an identifier for every vulnerability. Every vulnerability
// has unique namespace and name.
type VulnerabilityID struct {
Name string
Namespace string
}
// Vulnerability represents CVE or similar vulnerability reports.
type Vulnerability struct {
Name string
Namespace Namespace
@ -71,17 +137,85 @@ type Vulnerability struct {
Severity Severity
Metadata MetadataMap
}
// VulnerabilityWithAffected is an vulnerability with all known affected
// features.
type VulnerabilityWithAffected struct {
Vulnerability
Affected []AffectedFeature
}
// PagedVulnerableAncestries is a vulnerability with a page of affected
// ancestries each with a special index attached for streaming purpose. The
// current page number and next page number are for navigate.
type PagedVulnerableAncestries struct {
Vulnerability
// Affected is a map of special indexes to Ancestries, which the pair
// should be unique in a stream. Every indexes in the map should be larger
// than previous page.
Affected map[int]string
Limit int
Current PageNumber
Next PageNumber
FixedIn []FeatureVersion
LayersIntroducingVulnerability []Layer
// End signals the end of the pages.
End bool
}
// NotificationHook is a message sent to another service to inform of a change
// to a Vulnerability or the Ancestries affected by a Vulnerability. It contains
// the name of a notification that should be read and marked as read via the
// API.
type NotificationHook struct {
Name string
// For output purposes. Only make sense when the vulnerability
// is already about a specific Feature/FeatureVersion.
FixedBy string `json:",omitempty"`
Created time.Time
Notified time.Time
Deleted time.Time
}
// VulnerabilityNotification is a notification for vulnerability changes.
type VulnerabilityNotification struct {
NotificationHook
Old *Vulnerability
New *Vulnerability
}
// VulnerabilityNotificationWithVulnerable is a notification for vulnerability
// changes with vulnerable ancestries.
type VulnerabilityNotificationWithVulnerable struct {
NotificationHook
Old *PagedVulnerableAncestries
New *PagedVulnerableAncestries
}
// PageNumber is used to do pagination.
type PageNumber string
type MetadataMap map[string]interface{}
// NullableAffectedNamespacedFeature is an affectednamespacedfeature with
// whether it's found in datastore.
type NullableAffectedNamespacedFeature struct {
AffectedNamespacedFeature
Valid bool
}
// NullableVulnerability is a vulnerability with whether the vulnerability is
// found in datastore.
type NullableVulnerability struct {
VulnerabilityWithAffected
Valid bool
}
func (mm *MetadataMap) Scan(value interface{}) error {
if value == nil {
return nil
@ -99,25 +233,3 @@ func (mm *MetadataMap) Value() (driver.Value, error) {
json, err := json.Marshal(*mm)
return string(json), err
}
type VulnerabilityNotification struct {
Model
Name string
Created time.Time
Notified time.Time
Deleted time.Time
OldVulnerability *Vulnerability
NewVulnerability *Vulnerability
}
type VulnerabilityNotificationPageNumber struct {
// -1 means that we reached the end already.
OldVulnerability int
NewVulnerability int
}
var VulnerabilityNotificationFirstPage = VulnerabilityNotificationPageNumber{0, 0}
var NoVulnerabilityNotificationPage = VulnerabilityNotificationPageNumber{-1, -1}

14
database/severity.go
Unescape View File

@ -36,7 +36,7 @@ const (
// NegligibleSeverity is technically a security problem, but is only
// theoretical in nature, requires a very special situation, has almost no
// install base, or does no real damage. These tend not to get backport from
// upstreams, and will likely not be included in security updates unless
// upstream, and will likely not be included in security updates unless
// there is an easy fix and some other issue causes an update.
NegligibleSeverity Severity = "Negligible"
@ -93,7 +93,7 @@ func NewSeverity(s string) (Severity, error) {
// Compare determines the equality of two severities.
//
// If the severities are equal, returns 0.
// If the receiever is less, returns -1.
// If the receiver is less, returns -1.
// If the receiver is greater, returns 1.
func (s Severity) Compare(s2 Severity) int {
var i1, i2 int
@ -132,3 +132,13 @@ func (s *Severity) Scan(value interface{}) error {
func (s Severity) Value() (driver.Value, error) {
return string(s), nil
}
// Valid checks if the severity is valid or not.
func (s Severity) Valid() bool {
for _, v := range Severities {
if s == v {
return true
}
}
return false
}

55
pkg/strutil/strutil.go
Unescape View File

@ -0,0 +1,55 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package strutil
// CompareStringLists returns the strings that are present in X but not in Y.
func CompareStringLists(X, Y []string) []string {
m := make(map[string]bool)
for _, y := range Y {
m[y] = true
}
diff := []string{}
for _, x := range X {
if m[x] {
continue
}
diff = append(diff, x)
m[x] = true
}
return diff
}
// CompareStringListsInBoth returns the strings that are present in both X and Y.
func CompareStringListsInBoth(X, Y []string) []string {
m := make(map[string]struct{})
for _, y := range Y {
m[y] = struct{}{}
}
diff := []string{}
for _, x := range X {
if _, e := m[x]; e {
diff = append(diff, x)
delete(m, x)
}
}
return diff
}

34
pkg/strutil/strutil_test.go
Unescape View File

@ -0,0 +1,34 @@
// Copyright 2017 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package strutil
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestStringComparison(t *testing.T) {
cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"})
assert.Len(t, cmp, 1)
assert.NotContains(t, cmp, "a")
assert.Contains(t, cmp, "b")
cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"})
assert.Len(t, cmp, 2)
assert.NotContains(t, cmp, "b")
assert.Contains(t, cmp, "a")
assert.Contains(t, cmp, "c")
}
Write Preview
Loading…
Cancel
Save