database: Update database model and interface for detectors

All detected features and namespaces under the context of Layer and
Ancestry will now have the detectors associated, so that the API can
provide the detection information to the Client.

database/database.go

@@ -33,6 +33,14 @@ var (
 	// fails (i.e. when an entity which is supposed to be unique is detected
 	// twice)
 	ErrInconsistent = errors.New("database: inconsistent database")
+
+	// ErrInvalidParameters is an error that occurs when the parameters are not valid.
+	ErrInvalidParameters = errors.New("database: parameters are not valid")
+
+	// ErrMissingEntities is an error that occurs when an associated immutable
+	// entity doesn't exist in the database. This error can indicate a wrong
+	// implementation or corrupted database.
+	ErrMissingEntities = errors.New("database: associated immutable entities are missing in the database")
 )
 
 // RegistrableComponentConfig is a configuration block that can be used to
@@ -99,6 +107,9 @@ type Session interface {
 	// namespaced features. If the ancestry is not found, return false.
 	FindAncestry(name string) (ancestry Ancestry, found bool, err error)
 
+	// PersistDetector inserts a slice of detectors if not in the database.
+	PersistDetectors(detectors []Detector) error
+
 	// PersistFeatures inserts a set of features if not in the database.
 	PersistFeatures(features []Feature) error
 
@@ -120,12 +131,10 @@ type Session interface {
 	// PersistNamespaces inserts a set of namespaces if not in the database.
 	PersistNamespaces([]Namespace) error
 
-	// PersistLayer persists a layer's content in the database. The given
-	// namespaces and features can be partial content of this layer.
+	// PersistLayer appends a layer's content in the database.
 	//
-	// The layer, namespaces and features are expected to be already existing
-	// in the database.
-	PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
+	// If any feature, namespace, or detector is not in the database, it returns not found error.
+	PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectedBy []Detector) error
 
 	// FindLayer returns a layer with all detected features and
 	// namespaces.
@@ -157,8 +166,8 @@ type Session interface {
 	// affected ancestries affected by old or new vulnerability.
 	//
 	// Because the number of affected ancestries maybe large, they are paginated
-	// and their pages are specified by the paination token, which, if empty, are
-	// always considered first page.
+	// and their pages are specified by the pagination token, which should be
+	// considered first page when it's empty.
 	FindVulnerabilityNotification(name string, limit int, oldVulnerabilityPage pagination.Token, newVulnerabilityPage pagination.Token) (noti VulnerabilityNotificationWithVulnerable, found bool, err error)
 
 	// MarkNotificationAsRead marks a Notification as notified now, assuming

database/detector.go

@@ -0,0 +1,144 @@
+// Copyright 2018 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package database
+
+import (
+	"database/sql/driver"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+const (
+	// NamespaceDetectorType is a type of detector that extracts the namespaces.
+	NamespaceDetectorType DetectorType = "namespace"
+	// FeatureDetectorType is a type of detector that extracts the features.
+	FeatureDetectorType DetectorType = "feature"
+)
+
+// DetectorTypes contains all detector types.
+var (
+	DetectorTypes = []DetectorType{
+		NamespaceDetectorType,
+		FeatureDetectorType,
+	}
+	// ErrFailedToParseDetectorType is the error returned when a detector type could
+	// not be parsed from a string.
+	ErrFailedToParseDetectorType = errors.New("failed to parse DetectorType from input")
+	// ErrInvalidDetector is the error returned when a detector from database has
+	// invalid name or version or type.
+	ErrInvalidDetector = errors.New("the detector has invalid metadata")
+)
+
+// DetectorType is the type of a detector.
+type DetectorType string
+
+// Value implements the database/sql/driver.Valuer interface.
+func (s DetectorType) Value() (driver.Value, error) {
+	return string(s), nil
+}
+
+// Scan implements the database/sql.Scanner interface.
+func (s *DetectorType) Scan(value interface{}) error {
+	val, ok := value.([]byte)
+	if !ok {
+		return errors.New("could not scan a Severity from a non-string input")
+	}
+
+	var err error
+	*s, err = NewDetectorType(string(val))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NewDetectorType attempts to parse a string into a standard DetectorType
+// value.
+func NewDetectorType(s string) (DetectorType, error) {
+	for _, ss := range DetectorTypes {
+		if strings.EqualFold(s, string(ss)) {
+			return ss, nil
+		}
+	}
+
+	return "", ErrFailedToParseDetectorType
+}
+
+// Valid checks if a detector type is defined.
+func (s DetectorType) Valid() bool {
+	for _, t := range DetectorTypes {
+		if s == t {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Detector is an versioned Clair extension.
+type Detector struct {
+	// Name of an extension should be non-empty and uniquely identifies the
+	// extension.
+	Name string
+	// Version of an extension should be non-empty.
+	Version string
+	// DType is the type of the extension and should be one of the types in
+	// DetectorTypes.
+	DType DetectorType
+}
+
+// Valid checks if all fields in the detector satisfies the spec.
+func (d Detector) Valid() bool {
+	if d.Name == "" || d.Version == "" || !d.DType.Valid() {
+		return false
+	}
+
+	return true
+}
+
+// String returns a unique string representation of the detector.
+func (d Detector) String() string {
+	return fmt.Sprintf("%s:%s", d.Name, d.Version)
+}
+
+// NewNamespaceDetector returns a new namespace detector.
+func NewNamespaceDetector(name, version string) Detector {
+	return Detector{
+		Name:    name,
+		Version: version,
+		DType:   NamespaceDetectorType,
+	}
+}
+
+// NewFeatureDetector returns a new feature detector.
+func NewFeatureDetector(name, version string) Detector {
+	return Detector{
+		Name:    name,
+		Version: version,
+		DType:   FeatureDetectorType,
+	}
+}
+
+// SerializeDetectors returns the string representation of given detectors.
+func SerializeDetectors(detectors []Detector) []string {
+	strDetectors := []string{}
+	for _, d := range detectors {
+		strDetectors = append(strDetectors, d.String())
+	}
+
+	return strDetectors
+}

database/mock.go

@@ -30,9 +30,10 @@ type MockSession struct {
 	FctFindAffectedNamespacedFeatures   func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error)
 	FctPersistNamespaces                func([]Namespace) error
 	FctPersistFeatures                  func([]Feature) error
+	FctPersistDetectors                 func(detectors []Detector) error
 	FctPersistNamespacedFeatures        func([]NamespacedFeature) error
 	FctCacheAffectedNamespacedFeatures  func([]NamespacedFeature) error
-	FctPersistLayer                     func(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
+	FctPersistLayer                     func(hash string, features []LayerFeature, namespaces []LayerNamespace, by []Detector) error
 	FctFindLayer                        func(name string) (Layer, bool, error)
 	FctInsertVulnerabilities            func([]VulnerabilityWithAffected) error
 	FctFindVulnerabilities              func([]VulnerabilityID) ([]NullableVulnerability, error)
@@ -85,6 +86,13 @@ func (ms *MockSession) FindAffectedNamespacedFeatures(features []NamespacedFeatu
 	panic("required mock function not implemented")
 }
 
+func (ms *MockSession) PersistDetectors(detectors []Detector) error {
+	if ms.FctPersistDetectors != nil {
+		return ms.FctPersistDetectors(detectors)
+	}
+	panic("required mock function not implemented")
+}
+
 func (ms *MockSession) PersistNamespaces(namespaces []Namespace) error {
 	if ms.FctPersistNamespaces != nil {
 		return ms.FctPersistNamespaces(namespaces)
@@ -113,9 +121,9 @@ func (ms *MockSession) CacheAffectedNamespacedFeatures(namespacedFeatures []Name
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) PersistLayer(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error {
+func (ms *MockSession) PersistLayer(hash string, features []LayerFeature, namespaces []LayerNamespace, detectors []Detector) error {
 	if ms.FctPersistLayer != nil {
-		return ms.FctPersistLayer(hash, namespaces, features, processedBy)
+		return ms.FctPersistLayer(hash, features, namespaces, detectors)
 	}
 	panic("required mock function not implemented")
 }

database/models.go

@@ -22,47 +22,129 @@ import (
 	"github.com/coreos/clair/pkg/pagination"
 )
 
-// Processors are extentions to scan a layer's content.
-type Processors struct {
-	Listers   []string
-	Detectors []string
-}
-
 // Ancestry is a manifest that keeps all layers in an image in order.
 type Ancestry struct {
+	// Name is a globally unique value for a set of layers. This is often the
+	// sha256 digest of an OCI/Docker manifest.
 	Name string
-	// ProcessedBy contains the processors that are used when computing the
+	// By contains the processors that are used when computing the
 	// content of this ancestry.
-	ProcessedBy Processors
+	By []Detector
 	// Layers should be ordered and i_th layer is the parent of i+1_th layer in
 	// the slice.
 	Layers []AncestryLayer
 }
 
+// Valid checks if the ancestry is compliant to spec.
+func (a *Ancestry) Valid() bool {
+	if a == nil {
+		return false
+	}
+
+	if a.Name == "" {
+		return false
+	}
+
+	for _, d := range a.By {
+		if !d.Valid() {
+			return false
+		}
+	}
+
+	for _, l := range a.Layers {
+		if !l.Valid() {
+			return false
+		}
+	}
+
+	return true
+}
+
 // AncestryLayer is a layer with all detected namespaced features.
 type AncestryLayer struct {
-	LayerMetadata
-
-	// DetectedFeatures are the features introduced by this layer when it was
-	// processed.
-	DetectedFeatures []NamespacedFeature
-}
-
-// LayerMetadata contains the metadata of a layer.
-type LayerMetadata struct {
-	// Hash is content hash of the layer.
+	// Hash is the sha-256 tarsum on the layer's blob content.
 	Hash string
-	// ProcessedBy contains the processors that processed this layer.
-	ProcessedBy Processors
+	// Features are the features introduced by this layer when it was
+	// processed.
+	Features []AncestryFeature
 }
 
-// Layer is a layer with its detected namespaces and features by
-// ProcessedBy.
-type Layer struct {
-	LayerMetadata
+// Valid checks if the Ancestry Layer is compliant to the spec.
+func (l *AncestryLayer) Valid() bool {
+	if l == nil {
+		return false
+	}
 
-	Namespaces []Namespace
-	Features   []Feature
+	if l.Hash == "" {
+		return false
+	}
+
+	return true
+}
+
+// GetFeatures returns the Ancestry's features.
+func (l *AncestryLayer) GetFeatures() []NamespacedFeature {
+	nsf := make([]NamespacedFeature, 0, len(l.Features))
+	for _, f := range l.Features {
+		nsf = append(nsf, f.NamespacedFeature)
+	}
+
+	return nsf
+}
+
+// AncestryFeature is a namespaced feature with the detectors used to
+// find this feature.
+type AncestryFeature struct {
+	NamespacedFeature
+
+	// FeatureBy is the detector that detected the feature.
+	FeatureBy Detector
+	// NamespaceBy is the detector that detected the namespace.
+	NamespaceBy Detector
+}
+
+// Layer is a layer with all the detected features and namespaces.
+type Layer struct {
+	// Hash is the sha-256 tarsum on the layer's blob content.
+	Hash string
+	// By contains a list of detectors scanned this Layer.
+	By         []Detector
+	Namespaces []LayerNamespace
+	Features   []LayerFeature
+}
+
+func (l *Layer) GetFeatures() []Feature {
+	features := make([]Feature, 0, len(l.Features))
+	for _, f := range l.Features {
+		features = append(features, f.Feature)
+	}
+
+	return features
+}
+
+func (l *Layer) GetNamespaces() []Namespace {
+	namespaces := make([]Namespace, 0, len(l.Namespaces))
+	for _, ns := range l.Namespaces {
+		namespaces = append(namespaces, ns.Namespace)
+	}
+
+	return namespaces
+}
+
+// LayerNamespace is a namespace with detection information.
+type LayerNamespace struct {
+	Namespace
+
+	// By is the detector found the namespace.
+	By Detector
+}
+
+// LayerFeature is a feature with detection information.
+type LayerFeature struct {
+	Feature
+
+	// By is the detector found the feature.
+	By Detector
 }
 
 // Namespace is the contextual information around features.