Datastore: updated for Clair V3, decoupled interfaces and models

database/database.go

@@ -23,9 +23,9 @@ import (
 )
 
 var (
-	// ErrBackendException is an error that occurs when the database backend does
+	// ErrBackendException is an error that occurs when the database backend
-	// not work properly (ie. unreachable).
+	// does not work properly (ie. unreachable).
-	ErrBackendException = errors.New("database: an error occured when querying the backend")
+	ErrBackendException = errors.New("database: an error occurred when querying the backend")
 
 	// ErrInconsistent is an error that occurs when a database consistency check
 	// fails (i.e. when an entity which is supposed to be unique is detected
@@ -43,8 +43,8 @@ type RegistrableComponentConfig struct {
 
 var drivers = make(map[string]Driver)
 
-// Driver is a function that opens a Datastore specified by its database driver type and specific
+// Driver is a function that opens a Datastore specified by its database driver
-// configuration.
+// type and specific configuration.
 type Driver func(RegistrableComponentConfig) (Datastore, error)
 
 // Register makes a Constructor available by the provided name.
@@ -70,130 +70,127 @@ func Open(cfg RegistrableComponentConfig) (Datastore, error) {
 	return driver(cfg)
 }
 
-// Datastore represents the required operations on a persistent data store for
+// Session contains the required operations on a persistent data store for a
-// a Clair deployment.
+// Clair deployment.
-type Datastore interface {
+//
-	// ListNamespaces returns the entire list of known Namespaces.
+// Session is started by Datastore.Begin and terminated with Commit or Rollback.
-	ListNamespaces() ([]Namespace, error)
+// Besides Commit and Rollback, other functions cannot be called after the
-
+// session is terminated.
-	// InsertLayer stores a Layer in the database.
+// Any function is not guaranteed to be called successfully if there's a session
+// failure.
+type Session interface {
+	// Commit commits changes to datastore.
 	//
-	// A Layer is uniquely identified by its Name.
+	// Commit call after Rollback does no-op.
-	// The Name and EngineVersion fields are mandatory.
+	Commit() error
-	// If a Parent is specified, it is expected that it has been retrieved using
-	// FindLayer.
-	// If a Layer that already exists is inserted and the EngineVersion of the
-	// given Layer is higher than the stored one, the stored Layer should be
-	// updated.
-	// The function has to be idempotent, inserting a layer that already exists
-	// shouldn't return an error.
-	InsertLayer(Layer) error
 
-	// FindLayer retrieves a Layer from the database.
+	// Rollback drops changes to datastore.
 	//
-	// When `withFeatures` is true, the Features field should be filled.
+	// Rollback call after Commit does no-op.
-	// When `withVulnerabilities` is true, the Features field should be filled
+	Rollback() error
-	// and their AffectedBy fields should contain every vulnerabilities that
-	// affect them.
-	FindLayer(name string, withFeatures, withVulnerabilities bool) (Layer, error)
 
-	// DeleteLayer deletes a Layer from the database and every layers that are
+	// UpsertAncestry inserts or replaces an ancestry and its namespaced
-	// based on it, recursively.
+	// features and processors used to scan the ancestry.
-	DeleteLayer(name string) error
+	UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error
 
-	// ListVulnerabilities returns the list of vulnerabilities of a particular
+	// FindAncestry retrieves an ancestry with processors used to scan the
-	// Namespace.
+	// ancestry. If the ancestry is not found, return false.
 	//
-	// The Limit and page parameters are used to paginate the return list.
+	// The ancestry's processors are returned to short cut processing ancestry
-	// The first given page should be 0.
+	// if it has been processed by all processors in the current Clair instance.
-	// The function should return the next available page. If there are no more
+	FindAncestry(name string) (ancestry Ancestry, processedBy Processors, found bool, err error)
-	// pages, -1 has to be returned.
-	ListVulnerabilities(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
 
-	// InsertVulnerabilities stores the given Vulnerabilities in the database,
+	// FindAncestryFeatures retrieves an ancestry with all detected namespaced
-	// updating them if necessary.
+	// features. If the ancestry is not found, return false.
+	FindAncestryFeatures(name string) (ancestry AncestryWithFeatures, found bool, err error)
+
+	// PersistFeatures inserts a set of features if not in the database.
+	PersistFeatures(features []Feature) error
+
+	// PersistNamespacedFeatures inserts a set of namespaced features if not in
+	// the database.
+	PersistNamespacedFeatures([]NamespacedFeature) error
+
+	// CacheAffectedNamespacedFeatures relates the namespaced features with the
+	// vulnerabilities affecting these features.
 	//
-	// A vulnerability is uniquely identified by its Namespace and its Name.
+	// NOTE(Sida): it's not necessary for every database implementation and so
-	// The FixedIn field may only contain a partial list of Features that are
+	// this function may have a better home.
-	// affected by the Vulnerability, along with the version in which the
+	CacheAffectedNamespacedFeatures([]NamespacedFeature) error
-	// vulnerability is fixed. It is the responsibility of the implementation to
-	// update the list properly.
-	// A version equals to versionfmt.MinVersion means that the given Feature is
-	// not being affected by the Vulnerability at all and thus, should be removed
-	// from the list.
-	// It is important that Features should be unique in the FixedIn list. For
-	// example, it doesn't make sense to have two `openssl` Feature listed as a
-	// Vulnerability can only be fixed in one Version. This is true because
-	// Vulnerabilities and Features are namespaced (i.e. specific to one
-	// operating system).
-	// Each vulnerability insertion or update has to create a Notification that
-	// will contain the old and the updated Vulnerability, unless
-	// createNotification equals to true.
-	InsertVulnerabilities(vulnerabilities []Vulnerability, createNotification bool) error
 
-	// FindVulnerability retrieves a Vulnerability from the database, including
+	// FindAffectedNamespacedFeatures retrieves a set of namespaced features
-	// the FixedIn list.
+	// with affecting vulnerabilities.
-	FindVulnerability(namespaceName, name string) (Vulnerability, error)
+	FindAffectedNamespacedFeatures(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error)
 
-	// DeleteVulnerability removes a Vulnerability from the database.
+	// PersistNamespaces inserts a set of namespaces if not in the database.
+	PersistNamespaces([]Namespace) error
+
+	// PersistLayer inserts a layer if not in the datastore.
+	PersistLayer(Layer) error
+
+	// PersistLayerContent persists a layer's content in the database. The given
+	// namespaces and features can be partial content of this layer.
 	//
-	// It has to create a Notification that will contain the old Vulnerability.
+	// The layer, namespaces and features are expected to be already existing
-	DeleteVulnerability(namespaceName, name string) error
+	// in the database.
+	PersistLayerContent(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
 
-	// InsertVulnerabilityFixes adds new FixedIn Feature or update the Versions
+	// FindLayer retrieves a layer and the processors scanned the layer.
-	// of existing ones to the specified Vulnerability in the database.
+	FindLayer(hash string) (layer Layer, processedBy Processors, found bool, err error)
+
+	// FindLayerWithContent returns a layer with all detected features and
+	// namespaces.
+	FindLayerWithContent(hash string) (layer LayerWithContent, found bool, err error)
+
+	// InsertVulnerabilities inserts a set of UNIQUE vulnerabilities with
+	// affected features into database, assuming that all vulnerabilities
+	// provided are NOT in database and all vulnerabilities' namespaces are
+	// already in the database.
+	InsertVulnerabilities([]VulnerabilityWithAffected) error
+
+	// FindVulnerability retrieves a set of Vulnerabilities with affected
+	// features.
+	FindVulnerabilities([]VulnerabilityID) ([]NullableVulnerability, error)
+
+	// DeleteVulnerability removes a set of Vulnerabilities assuming that the
+	// requested vulnerabilities are in the database.
+	DeleteVulnerabilities([]VulnerabilityID) error
+
+	// InsertVulnerabilityNotifications inserts a set of unique vulnerability
+	// notifications into datastore, assuming that they are not in the database.
+	InsertVulnerabilityNotifications([]VulnerabilityNotification) error
+
+	// FindNewNotification retrieves a notification, which has never been
+	// notified or notified before a certain time.
+	FindNewNotification(notifiedBefore time.Time) (hook NotificationHook, found bool, err error)
+
+	// FindVulnerabilityNotification retrieves a vulnerability notification with
+	// affected ancestries affected by old or new vulnerability.
 	//
-	// It has has to create a Notification that will contain the old and the
+	// Because the number of affected ancestries maybe large, they are paginated
-	// updated Vulnerability.
+	// and their pages are specified by the given encrypted PageNumbers, which,
-	InsertVulnerabilityFixes(vulnerabilityNamespace, vulnerabilityName string, fixes []FeatureVersion) error
+	// if empty, are always considered first page.
-
-	// DeleteVulnerabilityFix removes a FixedIn Feature from the specified
-	// Vulnerability in the database. It can be used to store the fact that a
-	// Vulnerability no longer affects the given Feature in any Version.
 	//
-	// It has has to create a Notification that will contain the old and the
+	// Session interface implementation should have encrypt and decrypt
-	// updated Vulnerability.
+	// functions for PageNumber.
-	DeleteVulnerabilityFix(vulnerabilityNamespace, vulnerabilityName, featureName string) error
+	FindVulnerabilityNotification(name string, limit int,
+		oldVulnerabilityPage PageNumber,
+		newVulnerabilityPage PageNumber) (
+		noti VulnerabilityNotificationWithVulnerable,
+		found bool, err error)
 
-	// GetAvailableNotification returns the Name, Created, Notified and Deleted
+	// MarkNotificationNotified marks a Notification as notified now, assuming
-	// fields of a Notification that should be handled.
+	// the requested notification is in the database.
-	//
+	MarkNotificationNotified(name string) error
-	// The renotify interval defines how much time after being marked as Notified
-	// by SetNotificationNotified, a Notification that hasn't been deleted should
-	// be returned again by this function.
-	// A Notification for which there is a valid Lock with the same Name should
-	// not be returned.
-	GetAvailableNotification(renotifyInterval time.Duration) (VulnerabilityNotification, error)
 
-	// GetNotification returns a Notification, including its OldVulnerability and
+	// DeleteNotification removes a Notification in the database.
-	// NewVulnerability fields.
-	//
-	// On these Vulnerabilities, LayersIntroducingVulnerability should be filled
-	// with every Layer that introduces the Vulnerability (i.e. adds at least one
-	// affected FeatureVersion).
-	// The Limit and page parameters are used to paginate
-	// LayersIntroducingVulnerability. The first given page should be
-	// VulnerabilityNotificationFirstPage. The function will then return the next
-	// available page. If there is no more page, NoVulnerabilityNotificationPage
-	// has to be returned.
-	GetNotification(name string, limit int, page VulnerabilityNotificationPageNumber) (VulnerabilityNotification, VulnerabilityNotificationPageNumber, error)
-
-	// SetNotificationNotified marks a Notification as notified and thus, makes
-	// it unavailable for GetAvailableNotification, until the renotify duration
-	// is elapsed.
-	SetNotificationNotified(name string) error
-
-	// DeleteNotification marks a Notification as deleted, and thus, makes it
-	// unavailable for GetAvailableNotification.
 	DeleteNotification(name string) error
 
-	// InsertKeyValue stores or updates a simple key/value pair in the database.
+	// UpdateKeyValue stores or updates a simple key/value pair.
-	InsertKeyValue(key, value string) error
+	UpdateKeyValue(key, value string) error
 
-	// GetKeyValue retrieves a value from the database from the given key.
+	// FindKeyValue retrieves a value from the given key.
-	//
+	FindKeyValue(key string) (value string, found bool, err error)
-	// It returns an empty string if there is no such key.
-	GetKeyValue(key string) (string, error)
 
 	// Lock creates or renew a Lock in the database with the given name, owner
 	// and duration.
@@ -204,14 +201,20 @@ type Datastore interface {
 	// Lock should not block, it should instead returns whether the Lock has been
 	// successfully acquired/renewed. If it's the case, the expiration time of
 	// that Lock is returned as well.
-	Lock(name string, owner string, duration time.Duration, renew bool) (bool, time.Time)
+	Lock(name string, owner string, duration time.Duration, renew bool) (success bool, expiration time.Time, err error)
 
 	// Unlock releases an existing Lock.
-	Unlock(name, owner string)
+	Unlock(name, owner string) error
 
 	// FindLock returns the owner of a Lock specified by the name, and its
 	// expiration time if it exists.
-	FindLock(name string) (string, time.Time, error)
+	FindLock(name string) (owner string, expiration time.Time, found bool, err error)
+}
+
+// Datastore represents a persistent data store
+type Datastore interface {
+	// Begin starts a session to change.
+	Begin() (Session, error)
 
 	// Ping returns the health status of the database.
 	Ping() bool

database/models.go

@@ -20,49 +20,115 @@ import (
 	"time"
 )
 
-// ID is only meant to be used by database implementations and should never be used for anything else.
+// Processors are extentions to scan layer's content.
-type Model struct {
+type Processors struct {
-	ID int
+	Listers   []string
+	Detectors []string
 }
 
-type Layer struct {
+// Ancestry is a manifest that keeps all layers in an image in order.
-	Model
+type Ancestry struct {
-
 	Name string
-	EngineVersion int
+	// Layers should be ordered and i_th layer is the parent of i+1_th layer in
-	Parent        *Layer
+	// the slice.
-	Namespaces    []Namespace
+	Layers []Layer
-	Features      []FeatureVersion
 }
 
-type Namespace struct {
+// AncestryWithFeatures is an ancestry with namespaced features detected in the
-	Model
+// ancestry, which is processed by `ProcessedBy`.
+type AncestryWithFeatures struct {
+	Ancestry
 
+	ProcessedBy Processors
+	Features    []NamespacedFeature
+}
+
+// Layer corresponds to a layer in an image processed by `ProcessedBy`.
+type Layer struct {
+	// Hash is content hash of the layer.
+	Hash string
+}
+
+// LayerWithContent is a layer with its detected namespaces and features by
+// ProcessedBy.
+type LayerWithContent struct {
+	Layer
+
+	ProcessedBy Processors
+	Namespaces  []Namespace
+	Features    []Feature
+}
+
+// Namespace is the contextual information around features.
+//
+// e.g. Debian:7, NodeJS.
+type Namespace struct {
 	Name          string
 	VersionFormat string
 }
 
+// Feature represents a package detected in a layer but the namespace is not
+// determined.
+//
+// e.g. Name: OpenSSL, Version: 1.0, VersionFormat: dpkg.
+// dpkg implies the installer package manager but the namespace (might be
+// debian:7, debian:8, ...) could not be determined.
 type Feature struct {
-	Model
-
 	Name          string
+	Version       string
+	VersionFormat string
+}
+
+// NamespacedFeature is a feature with determined namespace and can be affected
+// by vulnerabilities.
+//
+// e.g. OpenSSL 1.0 dpkg Debian:7.
+type NamespacedFeature struct {
+	Feature
+
 	Namespace Namespace
 }
 
-type FeatureVersion struct {
+// AffectedNamespacedFeature is a namespaced feature affected by the
-	Model
+// vulnerabilities with fixed-in versions for this feature.
+type AffectedNamespacedFeature struct {
+	NamespacedFeature
 
-	Feature    Feature
+	AffectedBy []VulnerabilityWithFixedIn
-	Version    string
-	AffectedBy []Vulnerability
-
-	// For output purposes. Only make sense when the feature version is in the context of an image.
-	AddedBy Layer
 }
 
-type Vulnerability struct {
+// VulnerabilityWithFixedIn is used for AffectedNamespacedFeature to retrieve
-	Model
+// the affecting vulnerabilities and the fixed-in versions for the feature.
+type VulnerabilityWithFixedIn struct {
+	Vulnerability
 
+	FixedInVersion string
+}
+
+// AffectedFeature is used to determine whether a namespaced feature is affected
+// by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is
+// bound to vulnerability.
+type AffectedFeature struct {
+	Namespace   Namespace
+	FeatureName string
+	// FixedInVersion is known next feature version that's not affected by the
+	// vulnerability. Empty FixedInVersion means the unaffected version is
+	// unknown.
+	FixedInVersion string
+	// AffectedVersion contains the version range to determine whether or not a
+	// feature is affected.
+	AffectedVersion string
+}
+
+// VulnerabilityID is an identifier for every vulnerability. Every vulnerability
+// has unique namespace and name.
+type VulnerabilityID struct {
+	Name      string
+	Namespace string
+}
+
+// Vulnerability represents CVE or similar vulnerability reports.
+type Vulnerability struct {
 	Name      string
 	Namespace Namespace
 
@@ -71,17 +137,85 @@ type Vulnerability struct {
 	Severity    Severity
 
 	Metadata MetadataMap
-
-	FixedIn                        []FeatureVersion
-	LayersIntroducingVulnerability []Layer
-
-	// For output purposes. Only make sense when the vulnerability
-	// is already about a specific Feature/FeatureVersion.
-	FixedBy string `json:",omitempty"`
 }
 
+// VulnerabilityWithAffected is an vulnerability with all known affected
+// features.
+type VulnerabilityWithAffected struct {
+	Vulnerability
+
+	Affected []AffectedFeature
+}
+
+// PagedVulnerableAncestries is a vulnerability with a page of affected
+// ancestries each with a special index attached for streaming purpose. The
+// current page number and next page number are for navigate.
+type PagedVulnerableAncestries struct {
+	Vulnerability
+
+	// Affected is a map of special indexes to Ancestries, which the pair
+	// should be unique in a stream. Every indexes in the map should be larger
+	// than previous page.
+	Affected map[int]string
+
+	Limit   int
+	Current PageNumber
+	Next    PageNumber
+
+	// End signals the end of the pages.
+	End bool
+}
+
+// NotificationHook is a message sent to another service to inform of a change
+// to a Vulnerability or the Ancestries affected by a Vulnerability. It contains
+// the name of a notification that should be read and marked as read via the
+// API.
+type NotificationHook struct {
+	Name string
+
+	Created  time.Time
+	Notified time.Time
+	Deleted  time.Time
+}
+
+// VulnerabilityNotification is a notification for vulnerability changes.
+type VulnerabilityNotification struct {
+	NotificationHook
+
+	Old *Vulnerability
+	New *Vulnerability
+}
+
+// VulnerabilityNotificationWithVulnerable is a notification for vulnerability
+// changes with vulnerable ancestries.
+type VulnerabilityNotificationWithVulnerable struct {
+	NotificationHook
+
+	Old *PagedVulnerableAncestries
+	New *PagedVulnerableAncestries
+}
+
+// PageNumber is used to do pagination.
+type PageNumber string
+
 type MetadataMap map[string]interface{}
 
+// NullableAffectedNamespacedFeature is an affectednamespacedfeature with
+// whether it's found in datastore.
+type NullableAffectedNamespacedFeature struct {
+	AffectedNamespacedFeature
+
+	Valid bool
+}
+
+// NullableVulnerability is a vulnerability with whether the vulnerability is
+// found in datastore.
+type NullableVulnerability struct {
+	VulnerabilityWithAffected
+
+	Valid bool
+}
+
 func (mm *MetadataMap) Scan(value interface{}) error {
 	if value == nil {
 		return nil
@@ -99,25 +233,3 @@ func (mm *MetadataMap) Value() (driver.Value, error) {
 	json, err := json.Marshal(*mm)
 	return string(json), err
 }
-
-type VulnerabilityNotification struct {
-	Model
-
-	Name string
-
-	Created  time.Time
-	Notified time.Time
-	Deleted  time.Time
-
-	OldVulnerability *Vulnerability
-	NewVulnerability *Vulnerability
-}
-
-type VulnerabilityNotificationPageNumber struct {
-	// -1 means that we reached the end already.
-	OldVulnerability int
-	NewVulnerability int
-}
-
-var VulnerabilityNotificationFirstPage = VulnerabilityNotificationPageNumber{0, 0}
-var NoVulnerabilityNotificationPage = VulnerabilityNotificationPageNumber{-1, -1}

database/severity.go

@@ -36,7 +36,7 @@ const (
 	// NegligibleSeverity is technically a security problem, but is only
 	// theoretical in nature, requires a very special situation, has almost no
 	// install base, or does no real damage. These tend not to get backport from
-	// upstreams, and will likely not be included in security updates unless
+	// upstream, and will likely not be included in security updates unless
 	// there is an easy fix and some other issue causes an update.
 	NegligibleSeverity Severity = "Negligible"
 
@@ -93,7 +93,7 @@ func NewSeverity(s string) (Severity, error) {
 // Compare determines the equality of two severities.
 //
 // If the severities are equal, returns 0.
-// If the receiever is less, returns -1.
+// If the receiver is less, returns -1.
 // If the receiver is greater, returns 1.
 func (s Severity) Compare(s2 Severity) int {
 	var i1, i2 int
@@ -132,3 +132,13 @@ func (s *Severity) Scan(value interface{}) error {
 func (s Severity) Value() (driver.Value, error) {
 	return string(s), nil
 }
+
+// Valid checks if the severity is valid or not.
+func (s Severity) Valid() bool {
+	for _, v := range Severities {
+		if s == v {
+			return true
+		}
+	}
+	return false
+}

pkg/strutil/strutil.go

@@ -0,0 +1,55 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package strutil
+
+// CompareStringLists returns the strings that are present in X but not in Y.
+func CompareStringLists(X, Y []string) []string {
+	m := make(map[string]bool)
+
+	for _, y := range Y {
+		m[y] = true
+	}
+
+	diff := []string{}
+	for _, x := range X {
+		if m[x] {
+			continue
+		}
+
+		diff = append(diff, x)
+		m[x] = true
+	}
+
+	return diff
+}
+
+// CompareStringListsInBoth returns the strings that are present in both X and Y.
+func CompareStringListsInBoth(X, Y []string) []string {
+	m := make(map[string]struct{})
+
+	for _, y := range Y {
+		m[y] = struct{}{}
+	}
+
+	diff := []string{}
+	for _, x := range X {
+		if _, e := m[x]; e {
+			diff = append(diff, x)
+			delete(m, x)
+		}
+	}
+
+	return diff
+}

pkg/strutil/strutil_test.go

@@ -0,0 +1,34 @@
+// Copyright 2017 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package strutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStringComparison(t *testing.T) {
+	cmp := CompareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"})
+	assert.Len(t, cmp, 1)
+	assert.NotContains(t, cmp, "a")
+	assert.Contains(t, cmp, "b")
+
+	cmp = CompareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"})
+	assert.Len(t, cmp, 2)
+	assert.NotContains(t, cmp, "b")
+	assert.Contains(t, cmp, "a")
+	assert.Contains(t, cmp, "c")
+}